Supervised_Learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\JH\\Anaconda3\\envs\\farm\\lib\\site-packages\\konlpy\\tag\\_okt.py:16: UserWarning: \"Twitter\" has changed to \"Okt\" since KoNLPy v0.4.5.\n",
      "  warn('\"Twitter\" has changed to \"Okt\" since KoNLPy v0.4.5.')\n"
     ]
    }
   ],
   "source": [
    "#FilterReview\n",
    "from sklearn.model_selection import train_test_split \n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn import metrics \n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from konlpy.tag import Twitter\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import pickle \n",
    "import os.path\n",
    "\n",
    "'''\n",
    "readCSV :CSV 파일을 읽는다. 고정포맷이 정해짐 \n",
    "trainModel(ts =0.25): 입력된 모델에서 학습시킨다. ts 테스트 케이스 \n",
    "predict(): 학습시킨 모델에 test데이터가 입력되면 예측한다. \n",
    "writePredictionResult(): 예측 결과를 파일로 출력한다. \n",
    "'''\n",
    "twitter = Twitter()\n",
    "def tw_tokenizer(text):\n",
    "    # 입력 인자로 들어온 text 를 형태소 단어로 토큰화 하여 list 객체 반환\n",
    "    tokens_ko = twitter.morphs(text)\n",
    "    return tokens_ko\n",
    "\n",
    "class FilterReviewSV:\n",
    "    def __init__(self):\n",
    "        self.model_lr_clf_name = 'model/model_FR_lr_clf.sav'\n",
    "        self.model_tfidf_vect_name = 'model/model_FR_tfidf_vect.sav'\n",
    "        \n",
    "        self.isSetModel = False\n",
    "        if not(self.isSetModel) and os.path.isfile(self.model_lr_clf_name) and os.path.isfile(self.model_tfidf_vect_name):\n",
    "            print('모델을 불러오는중...')\n",
    "            self.model_lr_clf = pickle.load(open(self.model_lr_clf_name, 'rb'))\n",
    "            self.model_tfidf_vect = pickle.load(open(self.model_tfidf_vect_name, 'rb'))\n",
    "            isSetModel = True\n",
    "            print('모델 불러옴')\n",
    "        else:\n",
    "            print('모델이 없습니다..')\n",
    "            \n",
    "\n",
    "    def trainModel(self,filename,ts):\n",
    "        data_df = pd.read_csv(filename,engine='python')\n",
    "        X_train,X_test,y_train,y_test=train_test_split(data_df['sentence'],data_df['label'],test_size=0.25)\n",
    "        self.model_tfidf_vect = TfidfVectorizer(tokenizer=tw_tokenizer, ngram_range=(1,2), min_df=3, max_df=0.9)\n",
    "        self.model_tfidf_vect.fit(X_train)\n",
    "        \n",
    "        X_train_tfidf_vect = self.model_tfidf_vect.transform(X_train)\n",
    "        X_test_tfidf_vect = self.model_tfidf_vect.transform(X_test)\n",
    "\n",
    "        self.model_lr_clf = LogisticRegression(random_state=156)\n",
    "        self.model_lr_clf.fit(X_train_tfidf_vect , y_train)\n",
    "        pred = self.model_lr_clf.predict(X_test_tfidf_vect)\n",
    "        print('TF-IDF Vectorized Logistic Regression 의 예측 정확도는 {0:.3f}'.format(accuracy_score(y_test ,pred)))\n",
    "\n",
    "        pickle.dump(self.model_lr_clf, open(self.model_lr_clf_name, 'wb'))\n",
    "        pickle.dump(self.model_tfidf_vect, open(self.model_tfidf_vect_name, 'wb'))\n",
    "        isSetModel = True\n",
    "    def readCSV(self,filename):\n",
    "        self.data_df = pd.read_csv(filename,engine='python')\n",
    "        self.test_df_tfidf_vect = self.model_tfidf_vect.transform(self.data_df['sentence'])\n",
    "        \n",
    "    def predict(self):\n",
    "        self.y_pred = self.model_lr_clf.predict(self.test_df_tfidf_vect)\n",
    "        self.df = pd.DataFrame({'pr_label':self.y_pred, 'sentence':self.data_df['sentence']})\n",
    "        print('데이터 예측 완료')\n",
    "        print(self.df[:5])\n",
    "        return self.y_pred\n",
    "    def writePredictionResult(self, filename):\n",
    "        self.df.to_csv(filename,encoding='utf-8-sig',index=False)\n",
    "    def writeReviewResult(self, filename):\n",
    "        self.df_review = self.df[self.df['pr_label']==1]\n",
    "        self.df_review.to_csv(filename,encoding='utf-8-sig',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import glob\n",
    "import os\n",
    "\n",
    "folderlist = glob.glob('./data/*')\n",
    "for fol in folderlist:\n",
    "    if os.path.isdir(fol):\n",
    "        print('folder:',fol)\n",
    "      \n",
    "        filelist = glob.glob(fol+'/*.csv')\n",
    "        for file in filelist:\n",
    "            input_file = file.replace('\\\\','/')\n",
    "            input_path = input_file[2:]\n",
    "            print('input_file: ',input_path)\n",
    "            print('\\\\n Readed\\\\n')\n",
    "            \n",
    "\n",
    "            fr = FilterReviewSV()\n",
    "            fr.readCSV(filename=input_path)\n",
    "            fr.predict()\n",
    "            \n",
    "            output_file = 'labeled_'+input_path[11:]\n",
    "            print('output_file:', output_file)\n",
    "            fr.writePredictionResult(output_file)\n",
    "#print(fr.y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "모델을 불러오는중...\n",
      "TF-IDF Vectorized Logistic Regression 의 예측 정확도는 0.987\n"
     ]
    }
   ],
   "source": [
    "fr = FilterReviewSV()\n",
    "fr.trainModel(filename='Data_ReviewFilter/TrainData.csv',ts=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "모델을 불러오는중...\n",
      "모델 불러옴\n",
      "데이터 예측 완료\n",
      "   pr_label                                           sentence\n",
      "0         1                                아 더빙.. 진짜 짜증나네요 목소리\n",
      "1         1                  흠...포스터보고 초딩영화줄....오버연기조차 가볍지 않구나\n",
      "2         1                                  너무재밓었다그래서보는것을추천한다\n",
      "3         1                      교도소 이야기구먼 ..솔직히 재미는 없다..평점 조정\n",
      "4         1  사이몬페그의 익살스런 연기가 돋보였던 영화!스파이더맨에서 늙어보이기만 했던 커스틴 ...\n"
     ]
    }
   ],
   "source": [
    "fr = FilterReviewSV()\n",
    "fr.readCSV(filename='MovieReview/MR_test.csv')\n",
    "fr.predict()\n",
    "\n",
    "fr.writePredictionResult('sample.csv')\n",
    "fr.writeReviewResult('sample2.csv')\n",
    "#print(fr.y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}