import numpy as np
import pandas as pd
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.impute import SimpleImputer

# Naive Bayes
X = np.array([[1, 'A'], [2, 'B'], [3, 'A'], [4, 'B']])
y = np.array(['Y', 'N', 'Y', 'N'])

naive_bayes_model = GaussianNB()
naive_bayes_model.fit(X, y)

# Random Forest
X = np.array([[1, 2], [2, 4], [3, 6], [4, 8]])
y = np.array(['Y', 'N', 'Y', 'N'])

random_forest_model = RandomForestClassifier(n_estimators=100)
random_forest_model.fit(X, y)

# Missing Value Analysis
data = pd.DataFrame({'A': [1, 2, np.nan, 4], 'B': [5, np.nan, 7, 8], 'C': [9, 10, 11, np.nan]})
missing_values_count = data.isnull().sum()
print("Missing Values Count:")
print(missing_values_count)

imputer = SimpleImputer(strategy='mean')
imputed_data = imputer.fit_transform(data)
imputed_data = pd.DataFrame(imputed_data, columns=data.columns)
print("Imputed Data:")
print(imputed_data)

# Unique Value Analysis
data = pd.DataFrame({'A': [1, 2, 3, 4, 4], 'B': ['X', 'Y', 'X', 'Y', 'Z']})
unique_values = data.nunique()
print("Unique Values Count:")
print(unique_values)

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import seaborn as sns
from matplotlib import pyplot as plt
from matplotlib.colors import rgb2hex
import matplotlib.cm as cm
from wordcloud import WordCloud
from PIL import Image

import nltk
import re
nltk.download('stopwords')
from nltk.corpus import stopwords
nltk.download('wordnet')
from nltk.stem.porter import PorterStemmer

from sklearn import metrics
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline
from sklearn.feature_extraction.text import CountVectorizer

from sklearn.naive_bayes import MultinomialNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import confusion_matrix, accuracy_score, classification_report

import matplotlib.colors
from collections import Counter
cmap2 = cm.get_cmap('twilight',13)
colors1= []
for i in range(cmap2.N):
    rgb= cmap2(i)[:4]
    colors1.append(rgb2hex(rgb))
    #print(rgb2hex(rgb))
# Set style
sns.set(style='whitegrid')

df = pd.read_csv('/path to file/Data_file.csv',low_memory=False,index_col=0)

# Drop duplicates and Null Values

df.drop_duplicates(inplace=True)
df.dropna()

This dataset includes 23486 rows and 10 feature variables. Each row corresponds to a customer review, and includes the variables:

# Get to know your data

df.info()

<class 'pandas.core.frame.DataFrame'>
Int64Index: 23486 entries, 0 to 23485
Data columns (total 10 columns):
 #   Column                   Non-Null Count  Dtype 
---  ------                   --------------  ----- 
 0   Clothing ID              23486 non-null  int64 
 1   Age                      23486 non-null  int64 
 2   Title                    19676 non-null  object
 3   Review Text              22641 non-null  object
 4   Rating                   23486 non-null  int64 
 5   Recommended IND          23486 non-null  int64 
 6   Positive Feedback Count  23486 non-null  int64 
 7   Division Name            23472 non-null  object
 8   Department Name          23472 non-null  object
 9   Class Name               23472 non-null  object
dtypes: int64(5), object(5)
memory usage: 2.0+ MB

# Missing Values
df.isna().sum()

Clothing ID                   0
Age                           0
Title                      3810
Review Text                 845
Rating                        0
Recommended IND               0
Positive Feedback Count       0
Division Name                14
Department Name              14
Class Name                   14
dtype: int64

# See the stats

df.describe().T

# Get unique Values

df.Rating.value_counts()

5    13131
4     5077
3     2871
2     1565
1      842
Name: Rating, dtype: int64

# Get Class name Counts

df['Class Name'].value_counts()

Dresses           6319
Knits             4843
Blouses           3097
Sweaters          1428
Pants             1388
Jeans             1147
Fine gauge        1100
Skirts             945
Jackets            704
Lounge             691
Swim               350
Outerwear          328
Shorts             317
Sleep              228
Legwear            165
Intimates          154
Layering           146
Trend              119
Casual bottoms       2
Chemises             1
Name: Class Name, dtype: int64

# Get Count of Department Name

df['Department Name'].value_counts()

Tops        10468
Dresses      6319
Bottoms      3799
Intimate     1735
Jackets      1032
Trend         119
Name: Department Name, dtype: int64

# Cloth Department Analysis

plt.figure(figsize=(10,10))
sns.countplot(x='Department Name',data=df,palette='mako',order=df['Department Name'].value_counts().index,edgecolor='black',linewidth=1)
plt.xlabel('Dress Departments')
plt.ylabel('Count')
plt.xticks(rotation=45)
plt.title('Cloth Department Analysis')
plt.grid(False)

plt.show()

# Cloth Department by Ratings

plt.figure(figsize=(12,10))
sns.countplot(x='Department Name',data=df,palette='mako',order=df['Department Name'].value_counts().index,hue='Rating',edgecolor='black',linewidth=1)
plt.xlabel('Cloth Departments')
plt.ylabel('Count')
plt.xticks(rotation=45)
plt.title('Cloth Department Analysis')
plt.grid(False)

plt.show()

# Cloth Department by Age, Department and Recommendation

plt.figure(figsize=(12,10))
sns.boxplot(x = 'Age', y = 'Department Name', data = df,palette=colors1,hue='Recommended IND')
plt.grid(False)

plt.title('Cloth Department by Age and Recommendation ')
plt.show()

# Cloth Department Distribution

plt.figure(figsize=(12,10))
plt.pie(x=df['Department Name'].value_counts().values,data=df,colors=colors1,labels=df['Department Name'].value_counts().index,autopct='%.0f%%',explode=[0.02 for i in df['Department Name'].value_counts().index],startangle=45,wedgeprops={'linewidth':0.8,'edgecolor':'black'})
plt.title('Cloth Department Distribution')
plt.legend(loc='lower left')

plt.show()

# Cloth Class by Age, Department and Recommendation

plt.figure(figsize=(12,10))
sns.violinplot(x = 'Department Name', y = 'Age', data = df,palette='mako',hue='Recommended IND',orient='v')
plt.grid(False)

plt.title('Cloth Department by Age and Recommendation ')
plt.show()

# Cloth Class Analysis

plt.figure(figsize=(12,10))
sns.countplot(x='Class Name',data=df,palette=colors1,order=df['Class Name'].value_counts().index,edgecolor='black',linewidth=1)
plt.xlabel('Cloth Class')
plt.ylabel('Count')
plt.xticks(rotation=45)
plt.title('Cloth Class Analysis')
plt.grid(False)

plt.show()

# Cloth Class Analysis by Rating

plt.figure(figsize=(22,12))
sns.countplot(x=df['Class Name'],data=df,palette=colors1,order=df['Class Name'].value_counts().index,edgecolor='black',linewidth=1,hue='Rating')
plt.xlabel('Cloth Class')
plt.ylabel('Count')
plt.xticks(rotation=45)
plt.title('Cloth Class Analysis by Ratings')
plt.grid(False)
plt.legend(loc='right')

plt.show()

# Cloth Class Distribution

plt.figure(figsize=(18,15))
plt.pie(x=df['Class Name'].value_counts().values,data=df,colors=colors1,labels=df['Class Name'].value_counts().index,autopct='%.0f%%',explode=[0.07 for i in df['Class Name'].value_counts().index],startangle=180,wedgeprops={'linewidth':0.8,'edgecolor':'black'})
plt.title('Cloth Class Distribution')
#plt.grid(False)
plt.legend(loc='lower left')

plt.show()

# Cloth Class by Age, Department and Recommendation

plt.figure(figsize=(12,10))
sns.boxplot(x = 'Age', y = 'Class Name', data = df,palette=colors1,hue='Recommended IND')
plt.grid(False)

plt.title('Cloth Class by Age and Recommendation ')
plt.show()

# Division Value Counts

df['Division Name'].value_counts()

General           13839
General Petite     8110
Initmates          1502
Name: Division Name, dtype: int64

# Cloth Division Analysis by Department

plt.figure(figsize=(12,10))
sns.countplot(x='Division Name',data=df,palette=colors1,order=df['Division Name'].value_counts().index,edgecolor='black',linewidth=1,hue='Department Name')
plt.xlabel('Cloth Division')
plt.ylabel('Count')
plt.xticks(rotation=45)
plt.title('Cloth Division Analysis by Department')
plt.grid(False)
plt.legend(loc='upper right')

plt.show()

# Cloth Division by Rating

plt.figure(figsize=(12,10))
sns.countplot(x='Division Name',data=df,palette=colors1,order=df['Division Name'].value_counts().index,edgecolor='black',linewidth=1,hue='Rating')
plt.xlabel('Cloth Division')
plt.ylabel('Count')
plt.xticks(rotation=45)
plt.title('Cloth Division Analysis by Rating')
plt.grid(False)
plt.legend(loc='upper right')

plt.show()

# Cloth Division Percentage

plt.figure(figsize=(12,10))
plt.pie(x=df['Division Name'].value_counts().values,data=df,colors=colors1,labels=df['Division Name'].value_counts().index,autopct='%.0f%%',explode=[0.02 for i in df['Division Name'].value_counts().index],startangle=45,wedgeprops={'linewidth':0.8,'edgecolor':'black'})

plt.title('Cloth Division Percentage')

plt.legend()
plt.show()

# Cloth Division Name by Age

plt.figure(figsize=(12,10))
sns.boxplot(x = 'Age', y = 'Division Name', data = df,palette=colors1,hue='Recommended IND')
plt.grid(False)

plt.title('Cloth Division by Age and Recommendation ')
plt.show()

# Rating by Age

plt.figure(figsize=(12,10))
sns.barplot(x=df['Age'].head(10),y='Rating',data=df,palette=colors1,edgecolor='black',linewidth=1)
plt.title('Cloth Rating By Age')
plt.grid(False)

plt.show()

# Rating Distribution

plt.figure(figsize=(12,10))
sns.countplot(x='Rating',data=df,palette=colors1,order=df['Rating'].value_counts().index,edgecolor='black',linewidth=1)
plt.xlabel('Rating Class')
plt.ylabel('Count')

plt.title('Rating Distribution')
plt.grid(False)

plt.show()

# Rating Percentage

plt.figure(figsize=(12,10))
plt.pie(x=df['Rating'].value_counts().values,data=df,colors=colors1,labels=df['Rating'].value_counts().index,autopct='%.0f%%',explode=[0.02 for i in df['Rating'].value_counts().index],startangle=45,wedgeprops={'linewidth':0.8,'edgecolor':'black'})
plt.title('Rating Percentage')
plt.legend()

plt.show()

# Rating Distribution by Age

plt.figure(figsize=(12,10))
sns.boxplot(x = 'Rating', y = 'Age', data = df,palette='mako')
plt.grid(False)

plt.title('Rating Distribution by Age')
plt.show()

# Age Distribution

plt.figure(figsize=(12,10))

plt.hist(df['Age'], bins=40,color='#CBC3E3',edgecolor='black')
plt.xlabel('Age')
plt.ylabel('Reviews Count')
plt.grid(False)
plt.title('Number of Reviews by Age')

plt.show()

# Cloth Recommendation Analysis

df['Recommended IND'].value_counts()

1    19293
0     4172
Name: Recommended IND, dtype: int64

# Cloth Recommendation Count

plt.figure(figsize=(8,5))
sns.countplot(x='Recommended IND',data=df,palette=colors1,order=df['Recommended IND'].value_counts().index,edgecolor='black',linewidth=1)
plt.xlabel('Cloth Recommendation ( 1: Recommended, 0: Not Recommended)')
plt.ylabel('Count')

plt.title('Cloth Recommendation Count')
plt.grid(False)

plt.show()

# Recommendation Distribution ( 1 means recommended and 0 means not # Recommended by the Customer)

plt.figure(figsize=(12,10))
plt.pie(x=df['Recommended IND'].value_counts().values,data=df,colors=colors1,labels=df['Recommended IND'].value_counts().index,autopct='%.0f%%',explode=[0.02 for i in df['Recommended IND'].value_counts().index],startangle=45,wedgeprops={'linewidth':0.8,'edgecolor':'black'})
plt.title('Cloth Recommendation Distribution')
plt.legend()

plt.show()

# recommendation Analysis ( 1 means recommended and 0 means not Recommended by the Customer)

r = df[df['Recommended IND']==1]
not_r= df[df['Recommended IND']==0]

# Plot Cloth Recommendation by Cloth Department, Division, Class

fig = plt.figure(figsize=(20, 15))
ax1 = plt.subplot2grid((2, 2), (0, 0))
ax1 = sns.countplot(r['Division Name'], palette ='mako', alpha = 0.8, label = "Recommended",edgecolor='black',linewidth=1)
ax1 = sns.countplot(not_r['Division Name'], palette = colors1, alpha = 0.8, label = "Not Recommended",edgecolor='black',linewidth=1)
ax1 = plt.title("Recommended Items by Cloth Division")
ax1 = plt.legend()

ax2 = plt.subplot2grid((2, 2), (0, 1))
ax2 = sns.countplot(r['Department Name'], palette ='mako', alpha = 0.8, label = "Recommended",edgecolor='black',linewidth=1)
ax2 = sns.countplot(not_r['Department Name'], palette =colors1, alpha = 0.8, label = "Not Recommended",edgecolor='black',linewidth=1)
ax2 = plt.title("Recommended Items by Cloth Department")
ax2 = plt.legend()

ax3 = plt.subplot2grid((2, 2), (1, 0), colspan=2)
ax3 = plt.xticks(rotation=45)
ax3 = sns.countplot(r['Class Name'], palette ='mako', alpha = 0.8, label = "Recommended",edgecolor='black',linewidth=1)
ax3 = sns.countplot(not_r['Class Name'], palette =colors1, alpha = 0.8, label = "Not Recommended",edgecolor='black',linewidth=1)
ax3 = plt.title("Recommended Items by Cloth Class")
ax3 = plt.legend()

plt.show()

# heatmap

plt.figure(figsize=(8,6))
h = df.drop('Clothing ID',axis=1).corr()
sns.heatmap(h,annot=True,cmap='mako')
plt.show()

# Tokenizing the reviews

def tokens(words):
    words = re.sub("[^a-zA-Z]"," ", words)
    text = words.lower().split()                   
    return " ".join(text)
df['Review Text'] = df['Review Text'].astype(str)
df['Final_Reviews'] = df['Review Text'].apply(tokens)

# Use the Stop words

sw = stopwords.words('english')

clothes =['skirt','pants','white','black','fabric','silky','leather','blouse','sleeve','even','jacket','dress','color','wear','top','sweater','material','shirt','jeans','pant']
def stopwords(review):
    text = [word.lower() for word in review.split() if word.lower() not in sw and word.lower() not in clothes]
    return " ".join(text)
df['Final_Reviews'] = df['Final_Reviews'].apply(stopwords)

# Lemmatize
from nltk.stem.wordnet import WordNetLemmatizer
lm = WordNetLemmatizer()

def lemma(text):
    lem_text = [lm.lemmatize(word) for word in text.split()]
    return " ".join(lem_text)
df['Final_Reviews'] = df['Final_Reviews'].apply(lemma)

# Seperating Positive and Negative Reviews

nw = []
pw =[]
pos = df[df['Recommended IND']== 1]
neg = df[df['Recommended IND']== 0]

for r in neg.Final_Reviews:
    nw.append(r)
nw = ' '.join(nw)

for r in pos.Final_Reviews:
    pw.append(r) 
pw = ' '.join(pw)

# Wordcloud of Negative Reivews

wordcloud = WordCloud(background_color="white", max_words=len(nw),width=500, height=480, max_font_size=60, min_font_size=10,colormap='rocket')

wordcloud.generate(nw)

plt.figure(figsize=(20,17))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.margins(x=0, y=0)
plt.show()

# Word Cloud of positive Reviews

wordcloud = WordCloud(background_color="white", max_words=len(pw),width=500, height=480, max_font_size=60, min_font_size=10,colormap='mako')

wordcloud.generate(pw)

plt.figure(figsize=(20,17))
plt.imshow(wordcloud, interpolation="bilinear")
plt.axis("off")
plt.margins(x=0, y=0)

plt.show()

# Building Model

X = pos['Final_Reviews']
y = pos['Recommended IND']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 42)

# Count Vectorizer

v = CountVectorizer(min_df=7, ngram_range=(1,2)).fit(X_train)
X_tv = v.transform(X_train)

# Naive Bayes

model_nb = Pipeline([('v', CountVectorizer(min_df=7, ngram_range=(1,2))),('tfidf', TfidfTransformer()),('clf',MultinomialNB())])

model_nb.fit(X_train, y_train)

ytest = np.array(y_test)
pred_nb = model_nb.predict(X_test)
print('accuracy %s' % accuracy_score(pred_nb, y_test))
print('Confusion Matrix:',confusion_matrix(y_test, pred_nb))
print(classification_report(ytest, pred_y))

accuracy 1.0
Confusion Matrix: [[5788]]
              precision    recall  f1-score   support

           1       1.00      1.00      1.00      5788

    accuracy                           1.00      5788
   macro avg       1.00      1.00      1.00      5788
weighted avg       1.00      1.00      1.00      5788

# Random Forest
model_rf = Pipeline([('v', CountVectorizer(min_df=7, ngram_range=(1,2))),('tfidf', TfidfTransformer()),('clf-rf', RandomForestClassifier(n_estimators=30)),])
model_rf.fit(X_train, y_train)
ytest = np.array(y_test)
pred_rf = model_rf.predict(X_test)
print('accuracy %s' % accuracy_score(pred_rf, y_test))
print('Confusion Matrix:', confusion_matrix(y_test, pred_rf))
print(classification_report(ytest, pred))

accuracy 1.0
Confusion Matrix: [[5788]]
              precision    recall  f1-score   support

           1       1.00      1.00      1.00      5788

    accuracy                           1.00      5788
   macro avg       1.00      1.00      1.00      5788
weighted avg       1.00      1.00      1.00      5788