Add files via upload

Forest ML.py

@@ -0,0 +1,141 @@
+# -*- coding: utf-8 -*-
+"""
+Created on Thu Sep 12 21:21:12 2019
+
+@author: jpick
+"""
+
+import pandas as pd
+import numpy as np
+from sklearn.preprocessing import StandardScaler, LabelEncoder, MinMaxScaler
+from sklearn.model_selection import train_test_split, GridSearchCV, StratifiedKFold
+from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, AdaBoostClassifier
+from sklearn.linear_model import LogisticRegression, RidgeClassifier
+from sklearn.gaussian_process import GaussianProcessClassifier
+from sklearn.gaussian_process.kernels import RBF
+from sklearn.feature_selection import SelectKBest, chi2, VarianceThreshold
+from sklearn.svm import SVC
+from sklearn.metrics import accuracy_score, make_scorer
+from sklearn.compose import ColumnTransformer
+from boruta import BorutaPy
+import pdpipe as pdp
+import tensorflow as tf
+import kerastuner as kt
+
+
+
+
+np.random.seed(123)
+# download data and look for columns with null values
+data = pd.read_csv('csv files for training/train.csv')
+X=data.iloc[:,:55]
+y=data.iloc[:,-1]
+
+count_NA=[sum(X[col].isna()) for col in X.columns]
+print(count_NA)
+
+print(X.columns)
+print(X.transpose()[11:55])
+
+#encoding and scaling the variable with pdpipe
+X_new=pdp.OneHotEncode().apply(X)
+y_new=LabelEncoder().fit_transform(y)
+print(y_new.dtype)
+numeric_cols=X.columns[1:11]
+X[numeric_cols].head()
+X_new=pdp.Scale('StandardScaler').apply(X_new)
+X_new=pdp.ColDrop('Id').apply(X_new)
+print(X_new.shape)
+
+# Using selectors to pluck out most important features
+selector_thres=VarianceThreshold()
+
+selector_feat=selector_thres.fit(X_new, 0.0)
+print(selector_feat.get_support())
+
+K_selector=SelectKBest(k=13)
+K_selector_feat=K_selector.fit(X_new,y_new)
+print(K_selector_feat.get_support())
+X_k=K_selector.transform(X_new)
+
+
+# using boruta to select features
+X_bor=X_new.values
+y_bor=y_new
+
+
+rf= RandomForestClassifier()
+gbc=GradientBoostingClassifier()
+
+feat_selector = BorutaPy(rf, n_estimators='auto', verbose=2, random_state=1)
+feat_selector.fit(X_bor,y_bor)
+
+print(feat_selector.support_)
+X_filtered=feat_selector.transform(X_bor)
+print(X_filtered.shape)
+
+# breaking the filtered datasets into train and validation sets
+X_train, X_val, y_train, y_val=train_test_split(X_filtered,y_new,test_size=0.1)
+X_k_train, X_k_val, y_K_train, y_k_val=train_test_split(X_k,y_new,test_size=0.1)
+
+m1=rf.fit(X_train,y_train)
+m2=gbc.fit(X_train,y_train)
+
+p1=m1.predict(X_val)
+p2=m2.predict(X_val)
+
+print(accuracy_score(y_val,p1))
+print(accuracy_score(y_val,p2))
+
+def build_model(hp):
+  model_type = hp.Choice('model_type', ['random_forest', 'ridge', 'gbc'])
+  if model_type == 'random_forest':
+    model = RandomForestClassifier(
+        n_estimators=hp.Int('n_estimators', 70, 120, step=10),
+        max_depth=hp.Int('max_depth', 15, 25)
+        )
+  elif model_type== 'gbc':
+      model = GradientBoostingClassifier(
+          n_estimators=hp.Int('n_estimators', 90, 180, step=10),
+          max_depth=hp.Int('max_depth', 1, 8),
+          learning_rate=hp.Float('lr', 1e-3, 1, sampling='log')
+          )
+  else:
+    model = RidgeClassifier(
+        alpha=hp.Float('alpha', 1e-3, 1, sampling='log'))
+  return model
+
+tuner = kt.tuners.Sklearn(
+    oracle=kt.oracles.BayesianOptimization(
+        objective=kt.Objective('score', 'max'),
+        max_trials=10),
+    hypermodel=build_model,
+    scoring= make_scorer(accuracy_score),
+    cv= StratifiedKFold(5),
+    directory='.',
+    project_name='my_proj')
+
+
+
+tuner.search(X_train, y_train)
+tuner.search(X_k_train, y_K_train)
+
+best_model = tuner.get_best_models(num_models=1)[0]
+
+best_model_2 = tuner.get_best_models(num_models=1)[0]
+
+print(best_model)
+print(best_model_2)
+
+pbest=best_model.predict(X_val)
+pbest_2=best_model_2.predict(X_k_val)
+
+print(accuracy_score(y_val,pbest))
+
+m1=RandomForestClassifier(n_estimators=120, max_depth=19).fit(X_k_train,y_K_train)
+
+p1=m1.predict(X_k_val)
+
+print(accuracy_score(y_k_val,p1))
+
+print(accuracy_score(y_k_val,pbest_2))