|
import pandas as pd |
|
import numpy as np |
|
import matplotlib.pyplot as plt |
|
import seaborn as sns |
|
from scipy.stats.mstats import winsorize |
|
from sklearn.preprocessing import StandardScaler,MinMaxScaler |
|
from sklearn.metrics import root_mean_squared_error |
|
from scipy.stats import yeojohnson |
|
import evaluationer |
|
from sklearn.model_selection import train_test_split as tts |
|
def detect_outliers(df,num_cols): |
|
global outlier_df,zscore_cols,outlier_indexes,iqr_cols |
|
outlier_df = pd.DataFrame({"method" :[],"columns name":[],"upper limit":[], |
|
"lower limit":[],"no of Rows":[],"percentage outlier":[]}) |
|
if type(num_cols) == list: |
|
if len(num_cols)!=0: |
|
num_cols = num_cols |
|
else: |
|
num_cols = df.select_dtypes(exclude = "object").columns.tolist() |
|
else: |
|
if num_cols.tolist() != None: |
|
num_cols = num_cols |
|
else: |
|
num_cols = df.select_dtypes(exclude = "object").columns.tolist() |
|
zscore_cols = [] |
|
iqr_cols = [] |
|
outlier_indexes =[] |
|
for col in num_cols: |
|
skewness = df[col].skew() |
|
if -0.5 <= skewness <= 0.5: |
|
method = "zscore" |
|
zscore_cols.append(col) |
|
|
|
else: |
|
method = "iqr" |
|
iqr_cols.append(col) |
|
if len(zscore_cols) >0: |
|
for col in zscore_cols: |
|
mean = df[col].mean() |
|
std = df[col].std() |
|
ul = mean + (3*std) |
|
ll = mean - (3*std) |
|
mask = (df[col] < ll) | (df[col] > ul) |
|
temp = df[mask] |
|
|
|
Zscore_index = temp.index.tolist() |
|
outlier_indexes.extend(Zscore_index) |
|
|
|
if len(temp)>0: |
|
|
|
temp_df = pd.DataFrame({"method" : ["ZScore"], |
|
"columns name" : [col], |
|
"upper limit" : [round(ul,2)], |
|
"lower limit" :[ round(ll,2)], |
|
"no of Rows" : [len(temp)], |
|
"percentage outlier" : [round(len(temp)*100/len(df),2)]}) |
|
|
|
outlier_df = pd.concat([outlier_df,temp_df]).reset_index(drop = True) |
|
|
|
else: |
|
print("No columns for Zscore method") |
|
|
|
|
|
if len(iqr_cols) >0: |
|
for col in iqr_cols: |
|
q3 = df[col].quantile(.75) |
|
q1 = df[col].quantile(.25) |
|
IQR = q3 -q1 |
|
ul = q3 + 1.5*IQR |
|
ll = q1 - 1.5*IQR |
|
mask = (df[col] < ll) | (df[col] > ul) |
|
temp = df[mask] |
|
|
|
IQR_index = temp.index.tolist() |
|
outlier_indexes.extend(IQR_index) |
|
|
|
if len(temp)>0: |
|
list(outlier_indexes).append(list(IQR_index)) |
|
|
|
temp_df1 = pd.DataFrame({"method" : ["IQR"], |
|
"columns name" : [col], |
|
"upper limit" : [round(ul,2)], |
|
"lower limit" : [round(ll,2)], |
|
"no of Rows": [len(temp)], |
|
"percentage outlier" : [round((len(temp)*100/len(df)),2)] |
|
}) |
|
|
|
outlier_df = pd.concat([outlier_df,temp_df1]).reset_index(drop = True) |
|
|
|
else: |
|
print("No columns for IQR method") |
|
|
|
|
|
outlier_indexes = list(set(outlier_indexes)) |
|
|
|
return outlier_df,outlier_indexes |
|
|
|
|
|
def outlier_handling(df,y,model,outlier_indexes = [],outlier_cols = None ,method = root_mean_squared_error,test_size = 0.2, random_state = 42,eva = "reg"): |
|
num_col = df.select_dtypes(exclude = "O").columns |
|
|
|
global outliers_dropped_df,log_transformed_df,sqrt_transformed_df,yeo_johnson_transformed_df,rank_transformed_df |
|
global std_scaler_df,winsorize_transformed_df,inverse_log_transformed_winsorize_df,inverse_sqrt_transformed_winsorize_df,minmaxscaler_df |
|
if eva == "reg": |
|
if len(outlier_indexes) ==0: |
|
print("no outlier indexes passed") |
|
outliers_dropped_df = df.copy() |
|
else: |
|
outliers_dropped_df = df.drop(index =outlier_indexes) |
|
|
|
if outlier_cols != None: |
|
|
|
if df[outlier_cols][df[outlier_cols] <0].sum().sum() == 0: |
|
log_transformed_df = df.copy() |
|
log_transformed_df[outlier_cols] = np.log(log_transformed_df[outlier_cols] + 1e-5) |
|
sqrt_transformed_df = df.copy() |
|
sqrt_transformed_df[outlier_cols] = np.sqrt(sqrt_transformed_df[outlier_cols] + 1e-5) |
|
inverse_log_transformed_winsorize_df = log_transformed_df.copy() |
|
inverse_sqrt_transformed_winsorize_df = sqrt_transformed_df.copy() |
|
for column in outlier_cols: |
|
inverse_log_transformed_winsorize_df[column] = np.exp(winsorize(inverse_log_transformed_winsorize_df[column], limits=[0.05, 0.05])) |
|
inverse_sqrt_transformed_winsorize_df[column] = (winsorize(inverse_sqrt_transformed_winsorize_df[column], limits=[0.05, 0.05]))**2 |
|
else: |
|
print("df have values less than zero") |
|
std_scaler_df = df.copy() |
|
std_scaler_df[outlier_cols] = StandardScaler().fit_transform(std_scaler_df[outlier_cols]) |
|
|
|
minmaxscaler_df = df.copy() |
|
minmaxscaler_df[outlier_cols] = MinMaxScaler().fit_transform(minmaxscaler_df[outlier_cols]) |
|
|
|
yeo_johnson_transformed_df = df.copy() |
|
for column in outlier_cols: |
|
try: |
|
yeo_johnson_transformed_df[column], lambda_ = yeojohnson(yeo_johnson_transformed_df[column]) |
|
|
|
except : |
|
yeo_johnson_transformed_df[column] = yeo_johnson_transformed_df[column] |
|
|
|
print(f"Yeo-Johnson transformation failed for column '{column}'. Original data used.") |
|
|
|
rank_transformed_df = df.copy() |
|
rank_transformed_df[outlier_cols] = rank_transformed_df[outlier_cols].rank() |
|
winsorize_transformed_df = df.copy() |
|
for column in outlier_cols: |
|
winsorize_transformed_df[column] = winsorize(winsorize_transformed_df[column], limits=[0.05, 0.05]) |
|
|
|
|
|
|
|
else: |
|
|
|
|
|
if df[num_col][df[num_col] <0].sum().sum() == 0: |
|
log_transformed_df = df.copy() |
|
log_transformed_df[num_col] = np.log(log_transformed_df[num_col] + 1e-5) |
|
sqrt_transformed_df = df.copy() |
|
sqrt_transformed_df[num_col] = np.sqrt(sqrt_transformed_df[num_col] + 1e-5) |
|
inverse_log_transformed_winsorize_df = log_transformed_df.copy() |
|
inverse_sqrt_transformed_winsorize_df = sqrt_transformed_df.copy() |
|
for column in num_col: |
|
inverse_log_transformed_winsorize_df[column] = np.exp(winsorize(inverse_log_transformed_winsorize_df[column], limits=[0.05, 0.05])) |
|
inverse_sqrt_transformed_winsorize_df[column] = (winsorize(inverse_sqrt_transformed_winsorize_df[column], limits=[0.05, 0.05]))**2 |
|
else: |
|
|
|
print("df have values less than zero") |
|
|
|
std_scaler_df = df.copy() |
|
std_scaler_df[outlier_cols] = StandardScaler().fit_transform(std_scaler_df[outlier_cols]) |
|
|
|
minmaxscaler_df = df.copy() |
|
minmaxscaler_df[outlier_cols] = MinMaxScaler().fit_transform(minmaxscaler_df[outlier_cols]) |
|
|
|
yeo_johnson_transformed_df = df.copy() |
|
for column in num_col: |
|
try: |
|
yeo_johnson_transformed_df[column], lambda_ = yeojohnson(yeo_johnson_transformed_df[column]) |
|
|
|
except : |
|
yeo_johnson_transformed_df[column] = yeo_johnson_transformed_df[column] |
|
|
|
print(f"Yeo-Johnson transformation failed for column '{column}'. Original data used.") |
|
|
|
rank_transformed_df = df.copy() |
|
rank_transformed_df[num_col] = rank_transformed_df[num_col].rank() |
|
winsorize_transformed_df = df.copy() |
|
for column in num_col: |
|
winsorize_transformed_df[column] = winsorize(winsorize_transformed_df[column], limits=[0.05, 0.05]) |
|
|
|
if (df[num_col][df[num_col] <0].sum().sum() == 0): |
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,outliers_dropped_df,log_transformed_df,sqrt_transformed_df,yeo_johnson_transformed_df, |
|
rank_transformed_df,winsorize_transformed_df,inverse_log_transformed_winsorize_df,inverse_sqrt_transformed_winsorize_df] |
|
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","outliers_dropped_df", "log_transformed_df","sqrt_transformed_df", "yeo_johnson_transformed_df","rank_transformed_df","winsorize_transformed_df", |
|
"inverse_log_transformed_winsorize_df", "inverse_sqrt_transformed_winsorize_df"] |
|
elif df[outlier_cols][df[outlier_cols] <0].sum().sum() == 0: |
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,outliers_dropped_df,log_transformed_df,sqrt_transformed_df,yeo_johnson_transformed_df, |
|
rank_transformed_df,winsorize_transformed_df,inverse_log_transformed_winsorize_df,inverse_sqrt_transformed_winsorize_df] |
|
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","outliers_dropped_df","log_transformed_df", "sqrt_transformed_df","yeo_johnson_transformed_df","rank_transformed_df", |
|
"winsorize_transformed_df","inverse_log_transformed_winsorize_df","inverse_sqrt_transformed_winsorize_df"] |
|
|
|
else: |
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,outliers_dropped_df,yeo_johnson_transformed_df,rank_transformed_df,winsorize_transformed_df] |
|
|
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","outliers_dropped_df","yeo_johnson_transformed_df","rank_transformed_df","winsorize_transformed_df"] |
|
|
|
for j,i in enumerate(outlier_handled_df): |
|
X_train, X_test, y_train, y_test = tts(i,y[i.index],test_size = test_size, random_state = random_state) |
|
evaluationer.evaluation(f"{outlier_handled_df_name[j]}",X_train,X_test,y_train,y_test,model,root_mean_squared_error,eva) |
|
|
|
|
|
return evaluationer.reg_evaluation_df , outlier_handled_df,outlier_handled_df_name |
|
elif eva =="class": |
|
|
|
std_scaler_df = df.copy() |
|
|
|
std_scaler_df.loc[:,:] = StandardScaler().fit_transform(std_scaler_df.loc[:,:]) |
|
|
|
minmaxscaler_df = df.copy() |
|
minmaxscaler_df.loc[:,:] = MinMaxScaler().fit_transform(minmaxscaler_df.loc[:,:]) |
|
|
|
rank_transformed_df = df.copy() |
|
rank_transformed_df = rank_transformed_df.rank() |
|
|
|
outlier_handled_df = [std_scaler_df,minmaxscaler_df,rank_transformed_df] |
|
outlier_handled_df_name = ["std_scaler_df","minmaxscaler_df","rank_transformed_df"] |
|
|
|
for j,i in enumerate(outlier_handled_df): |
|
|
|
X_train, X_test, y_train, y_test = tts(i,y[i.index],test_size = test_size, random_state = random_state) |
|
evaluationer.evaluation(f"{outlier_handled_df_name[j]}", X_train,X_test,y_train,y_test,model,root_mean_squared_error,eva = "class") |
|
return evaluationer.classification_evaluation_df, outlier_handled_df,outlier_handled_df_name |
|
|
|
|
|
|
