week7_framework_pytorch_assignment_최혜빈.ipynb

{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"name":"week7_framework_pytorch_assignment_최혜빈.ipynb","provenance":[],"collapsed_sections":[],"mount_file_id":"1VoaAwPVMgxQLp0YJEh3QNZpS7UNBRNa2","authorship_tag":"ABX9TyN2jv8CGiIgwSZHsl1XU0qy"},"kernelspec":{"name":"python3","display_name":"Python 3"},"accelerator":"GPU"},"cells":[{"cell_type":"code","metadata":{"id":"WUanNgmDVK_Q","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"d84f81e2-4f8a-4d96-be5c-cf94b9bf17bd","executionInfo":{"status":"ok","timestamp":1584442999685,"user_tz":-540,"elapsed":1277,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["cd /content/drive/My Drive/tobigs"],"execution_count":1,"outputs":[{"output_type":"stream","text":["/content/drive/My Drive/tobigs\n"],"name":"stdout"}]},{"cell_type":"markdown","metadata":{"id":"cp1atypfcfF0","colab_type":"text"},"source":["### 데이터 전처리"]},{"cell_type":"code","metadata":{"id":"cPh0oYmHVhn_","colab_type":"code","colab":{}},"source":["import numpy as np\n","import pandas as pd\n","\n","sample_submission = pd.read_csv(\"sample_submission.csv\")\n","train = pd.read_csv(\"train_df.csv\")\n","test = pd.read_csv(\"test_df.csv\")"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"7eG8VY4JVkco","colab_type":"code","colab":{}},"source":["X = train.iloc[:, 1:].values / 255\n","y = train.iloc[:, 0].values\n","test = test.iloc[:, 1:].values / 255"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"x19ptbIQcbsv","colab_type":"text"},"source":["### Dataset, DataLoader"]},{"cell_type":"code","metadata":{"id":"_NPF6lizZTkg","colab_type":"code","colab":{}},"source":["from torch.utils.data import Dataset, DataLoader\n","import torch #필요한 라이브러리 import \n","\n","class TrainDataset(Dataset): #trainset dataloader 정의\n","    def __init__(self, X, y, transform=None): #데이터셋의 전처리를 해주는 부분\n","        self.X = X\n","        self.y = y\n","        self.transform = transform\n","    def __len__(self): #데이터셋의 길이. 즉 총 샘플의 수를 적어주는 부분\n","        return len(self.X)\n","    def __getitem__(self, idx): #데이터셋에서 특정 1개의 샘플을 가져오는 함수\n","        X, y = self.X[idx], self.y[idx]\n","        x2d = X.reshape(28, 28)\n","        x2d = X.reshape(784)\n","        return torch.tensor(X).float(), torch.tensor(y)\n","\n","class TestDataset(Dataset): #위의 학습데이터와 동일한 방식으로 testdataloader 생성\n","    def __init__(self, X, y=None, transform=None): #데이터셋의 전처리를 해주는 부분\n","        self.X = X\n","        self.y = y\n","        self.transform = transform\n","    def __len__(self): #데이터셋의 길이. 즉 총 샘플의 수를 적어주는 부분\n","        return len(self.X)\n","\n","    def __getitem__(self, idx): #데이터셋에서 특정 1개의 샘플을 가져오는 함수\n","        X = self.X[idx]\n","        return torch.tensor(X).float()"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"Oet6ZthkERxS","colab_type":"code","colab":{}},"source":["traindataset = TrainDataset(X, y)\n","testdataset = TestDataset(test)"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"G-Q3KFJlZedZ","colab_type":"code","colab":{}},"source":["trainloader = DataLoader(traindataset, batch_size=256, shuffle=True, num_workers=2)\n","testloader = DataLoader(testdataset, batch_size=256, shuffle=False, num_workers=2)"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"pfdfKEMgczl0","colab_type":"text"},"source":["### Model"]},{"cell_type":"markdown","metadata":{"id":"BsOUp19xdcnZ","colab_type":"text"},"source":["Model\n","\n","-Activation : 활성화 함수, 신경망의 출력을 결정하는 식 elu, selu, relu, leakyrelu, sigmoid 를 사용해보았고, loss가 가장 작게 나온 elu를 사용하였습니다 elu는 relu의 특성을 공유하면서, 그래디언트가 죽지 않는다는 장점이 있다고합니다 \n","\n","-Batch normalization: 각 배치에서 이전 레이어의 activations을 정규화하는 batchnormalization을 진행했고 이동평균 및 이동 분산에 대한 모멘텀을 의미하는 momentum을 따로 설정해주었습니다\n","\n","-Regularization: overfitting을 막기위해 필요한 규제로 dropout사용했습니다 neural net 중간중간에 넣었습니다! -> 학습을 하면서 weight들이 서로 동조하는 현상 피하기 위해"]},{"cell_type":"code","metadata":{"id":"Fs9X5fEFZgAw","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":230},"outputId":"2e4f889d-f7b8-4da6-b111-9802936d11ff","executionInfo":{"status":"ok","timestamp":1584443003993,"user_tz":-540,"elapsed":5477,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["import torch.nn as nn\n","import torch.nn.functional as F\n","\n","class Net(nn.Module):\n","    def __init__(self):\n","        super(Net, self).__init__() #총 4개의 layer 설정\n","        self.fc1 = nn.Linear(784, 512) #첫번재 layer\n","        self.bn1 = nn.BatchNorm1d(512,momentum=0.2) #batch normalization\n","        self.fc2 = nn.Linear(512, 256) #두번째 layer\n","        self.bn2 = nn.BatchNorm1d(256,momentum=0.2)#batch normalization\n","        self.fc3 = nn.Linear(256, 128)#세번째 layer\n","        self.bn3 = nn.BatchNorm1d(128,momentum=0.2)\n","        self.fc4 = nn.Linear(128, 10)#네번째 layer\n","        self.elu = nn.ELU() #activation function은 elu사용!\n","        self.dropout1 = torch.nn.Dropout(p=0.5)#dropout\n","        self.dropout2 = torch.nn.Dropout(p=0.5) #regularization위한 dropout설정\n","\n","    def forward(self, x):\n","        x = self.fc1(x) #첫번째 선형결합후\n","        x = self.fc2(self.bn1(F.relu(x)))#활성화함수relu를 지나고 bactch normalization을 거친후 두번째 layer로 들어가 선형결합\n","        x = self.dropout1(x) #regularization을 위한 dropout과정을 지나고\n","        x = self.fc3(self.bn2(self.elu(x))) #다시 활성화함수 relu를 거친후 batchnormalization 후 세번째 layer로 들어간다\n","        x = self.dropout2(x) #두번째 dropout\n","        x = self.fc4(self.bn3(x)) #또 batchnormalization후 마지막 선형결합\n","        return x #결과값 리턴받음\n","\n","net = Net()\n","\n","net"],"execution_count":7,"outputs":[{"output_type":"execute_result","data":{"text/plain":["Net(\n","  (fc1): Linear(in_features=784, out_features=512, bias=True)\n","  (bn1): BatchNorm1d(512, eps=1e-05, momentum=0.2, affine=True, track_running_stats=True)\n","  (fc2): Linear(in_features=512, out_features=256, bias=True)\n","  (bn2): BatchNorm1d(256, eps=1e-05, momentum=0.2, affine=True, track_running_stats=True)\n","  (fc3): Linear(in_features=256, out_features=128, bias=True)\n","  (bn3): BatchNorm1d(128, eps=1e-05, momentum=0.2, affine=True, track_running_stats=True)\n","  (fc4): Linear(in_features=128, out_features=10, bias=True)\n","  (elu): ELU(alpha=1.0)\n","  (dropout1): Dropout(p=0.5, inplace=False)\n","  (dropout2): Dropout(p=0.5, inplace=False)\n",")"]},"metadata":{"tags":[]},"execution_count":7}]},{"cell_type":"markdown","metadata":{"id":"To5Rd7ofEvbc","colab_type":"text"},"source":["\n","\n","*   Optimization: optimizer로 rmsprop, adam, adagrad 사용해보았는데 사실 거의 비슷하게 나왔는데 가장 loss 작게 나온 adam 선택했습니다! adam은 adagrad와 rmsprop의 학습 파라미터의 개별 원소마다 학습률을 달리 적용하는 기법을 합친것!\n","\n"]},{"cell_type":"code","metadata":{"id":"eqA3rYMzZhYS","colab_type":"code","colab":{}},"source":["import torch.optim as optim\n","\n","criterion = nn.CrossEntropyLoss()#loss는 crossentropy로 확인\n","optimizer = optim.Adam(net.parameters(), lr=0.0001)# optimizer사용 #learningrate 따로 설정함"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"qYJjvy8eZi7o","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"3b52060a-7675-4dda-ad4f-b6ff837ff759","executionInfo":{"status":"ok","timestamp":1584443003995,"user_tz":-540,"elapsed":5454,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["if torch.cuda.is_available():\n","    device = torch.device(\"cuda\") \n","print(device)"],"execution_count":9,"outputs":[{"output_type":"stream","text":["cuda\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"cl9pqBjUZknd","colab_type":"code","colab":{}},"source":["model = net.to(device) #모델 적용"],"execution_count":0,"outputs":[]},{"cell_type":"markdown","metadata":{"id":"r_d6H010-ri1","colab_type":"text"},"source":["-Weight Initialization: 각 층의 가중치(weights)와 편항(bias)등 학습 파라미터는 초기값 설정이 매우 중요하기 때문에,nn.init.kaiming_normal_(m.weight)을 사용했습니다!"]},{"cell_type":"code","metadata":{"id":"H8JqTgbV9f3y","colab_type":"code","colab":{}},"source":["def weights_init(m): #weight initialization\n","    if isinstance(m, nn.Linear):\n","        nn.init.kaiming_normal_(m.weight) #kaiming_normal_이 he_normal이랑 같음\n"," \n","model = model.apply(weights_init)#가중치 초기화 함수 모델에 적용"],"execution_count":0,"outputs":[]},{"cell_type":"code","metadata":{"id":"2AoAU9aRZleI","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":1000},"outputId":"80754ec2-acfe-4761-f1dd-9331569545a3","executionInfo":{"status":"ok","timestamp":1584443317777,"user_tz":-540,"elapsed":319008,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["num_epochs = 150 #10번정도 더 돌려볼려고 10쓴거고 사실상 150번정도 돌렸습니다\n","for epoch in range(num_epochs):  # epoch수만큼 데이터 학습\n","\n","    running_loss = 0.0\n","    for i, data in enumerate(trainloader):\n","        # get the inputs; data is a list of [inputs, labels]\n","      #앞서 정의한 trainloader에서 학습데이터를 불러오고\n","        inputs, labels = data\n","        inputs, labels = inputs.to(device), labels.to(device)\n","        # zero the parameter gradients\n","        optimizer.zero_grad() \n","\n","        # forward + backward + optimize\n","        outputs = model(inputs) #미리 정의해둔 모델을 적용한다\n","        loss = criterion(outputs, labels) #loss를 구하면서, forward로 구한 gradient를 이용해 back propagation을 이용해 가중치를 update해간다\n","        loss.backward()#backpropagation과정\n","        optimizer.step() #optimizer을 이용해 loss최소화시키는 최적화 단계 진행\n","\n","        # print statistics\n","        running_loss += loss.item()\n","    print('[%d, %5d] loss: %.9f' %(epoch + 1, i + 1, running_loss / 20)) #epoch당 loss출력\n","    running_loss = 0.0\n","\n","print('Finished Training')\n"],"execution_count":12,"outputs":[{"output_type":"stream","text":["[1,   165] loss: 9.677899079\n","[2,   165] loss: 3.636336997\n","[3,   165] loss: 2.496489234\n","[4,   165] loss: 1.986027213\n","[5,   165] loss: 1.664159213\n","[6,   165] loss: 1.421426158\n","[7,   165] loss: 1.257371542\n","[8,   165] loss: 1.089751801\n","[9,   165] loss: 0.994440640\n","[10,   165] loss: 0.877749930\n","[11,   165] loss: 0.788473132\n","[12,   165] loss: 0.721951818\n","[13,   165] loss: 0.677319052\n","[14,   165] loss: 0.599663174\n","[15,   165] loss: 0.537133723\n","[16,   165] loss: 0.488392512\n","[17,   165] loss: 0.444956257\n","[18,   165] loss: 0.410242757\n","[19,   165] loss: 0.352312817\n","[20,   165] loss: 0.324893426\n","[21,   165] loss: 0.298647388\n","[22,   165] loss: 0.271673234\n","[23,   165] loss: 0.246311311\n","[24,   165] loss: 0.247004716\n","[25,   165] loss: 0.211561663\n","[26,   165] loss: 0.184503947\n","[27,   165] loss: 0.172806886\n","[28,   165] loss: 0.151773550\n","[29,   165] loss: 0.142830145\n","[30,   165] loss: 0.125021947\n","[31,   165] loss: 0.118961240\n","[32,   165] loss: 0.119305011\n","[33,   165] loss: 0.108435356\n","[34,   165] loss: 0.096556581\n","[35,   165] loss: 0.091455624\n","[36,   165] loss: 0.087024666\n","[37,   165] loss: 0.075912879\n","[38,   165] loss: 0.071173291\n","[39,   165] loss: 0.068736494\n","[40,   165] loss: 0.063245644\n","[41,   165] loss: 0.060454418\n","[42,   165] loss: 0.057143256\n","[43,   165] loss: 0.069055116\n","[44,   165] loss: 0.054834181\n","[45,   165] loss: 0.056897850\n","[46,   165] loss: 0.061029473\n","[47,   165] loss: 0.050553605\n","[48,   165] loss: 0.038536673\n","[49,   165] loss: 0.040765072\n","[50,   165] loss: 0.034192811\n","[51,   165] loss: 0.033507141\n","[52,   165] loss: 0.030310147\n","[53,   165] loss: 0.027950081\n","[54,   165] loss: 0.026811665\n","[55,   165] loss: 0.029693601\n","[56,   165] loss: 0.026273207\n","[57,   165] loss: 0.021986946\n","[58,   165] loss: 0.020863978\n","[59,   165] loss: 0.019660427\n","[60,   165] loss: 0.018765071\n","[61,   165] loss: 0.025435926\n","[62,   165] loss: 0.030600123\n","[63,   165] loss: 0.022432258\n","[64,   165] loss: 0.017345722\n","[65,   165] loss: 0.015785896\n","[66,   165] loss: 0.017104174\n","[67,   165] loss: 0.015323682\n","[68,   165] loss: 0.014785357\n","[69,   165] loss: 0.013862962\n","[70,   165] loss: 0.014672265\n","[71,   165] loss: 0.014236915\n","[72,   165] loss: 0.011204040\n","[73,   165] loss: 0.018077896\n","[74,   165] loss: 0.013900888\n","[75,   165] loss: 0.010166602\n","[76,   165] loss: 0.012185527\n","[77,   165] loss: 0.023963590\n","[78,   165] loss: 0.019275363\n","[79,   165] loss: 0.014896578\n","[80,   165] loss: 0.021317524\n","[81,   165] loss: 0.011133533\n","[82,   165] loss: 0.032700557\n","[83,   165] loss: 0.011480669\n","[84,   165] loss: 0.010365167\n","[85,   165] loss: 0.008020556\n","[86,   165] loss: 0.009508041\n","[87,   165] loss: 0.011619415\n","[88,   165] loss: 0.008669929\n","[89,   165] loss: 0.009912092\n","[90,   165] loss: 0.006854510\n","[91,   165] loss: 0.008239018\n","[92,   165] loss: 0.007400966\n","[93,   165] loss: 0.007591069\n","[94,   165] loss: 0.007827834\n","[95,   165] loss: 0.008016302\n","[96,   165] loss: 0.006437934\n","[97,   165] loss: 0.007547082\n","[98,   165] loss: 0.010078678\n","[99,   165] loss: 0.010983028\n","[100,   165] loss: 0.007034623\n","[101,   165] loss: 0.023458839\n","[102,   165] loss: 0.010743214\n","[103,   165] loss: 0.007265451\n","[104,   165] loss: 0.007143929\n","[105,   165] loss: 0.005107326\n","[106,   165] loss: 0.006189424\n","[107,   165] loss: 0.008018999\n","[108,   165] loss: 0.009003462\n","[109,   165] loss: 0.018382239\n","[110,   165] loss: 0.008029519\n","[111,   165] loss: 0.006655484\n","[112,   165] loss: 0.005973045\n","[113,   165] loss: 0.004497823\n","[114,   165] loss: 0.004944321\n","[115,   165] loss: 0.005839259\n","[116,   165] loss: 0.003983940\n","[117,   165] loss: 0.004395675\n","[118,   165] loss: 0.004291807\n","[119,   165] loss: 0.004668604\n","[120,   165] loss: 0.004118343\n","[121,   165] loss: 0.011402829\n","[122,   165] loss: 0.005863982\n","[123,   165] loss: 0.004975653\n","[124,   165] loss: 0.004388541\n","[125,   165] loss: 0.003754508\n","[126,   165] loss: 0.004610024\n","[127,   165] loss: 0.003592117\n","[128,   165] loss: 0.006278616\n","[129,   165] loss: 0.005085667\n","[130,   165] loss: 0.033803671\n","[131,   165] loss: 0.009175894\n","[132,   165] loss: 0.003832480\n","[133,   165] loss: 0.005095198\n","[134,   165] loss: 0.005308532\n","[135,   165] loss: 0.004810586\n","[136,   165] loss: 0.020871967\n","[137,   165] loss: 0.006926751\n","[138,   165] loss: 0.005254343\n","[139,   165] loss: 0.006355038\n","[140,   165] loss: 0.004808970\n","[141,   165] loss: 0.004438467\n","[142,   165] loss: 0.004397666\n","[143,   165] loss: 0.004157418\n","[144,   165] loss: 0.003100745\n","[145,   165] loss: 0.003006418\n","[146,   165] loss: 0.002912585\n","[147,   165] loss: 0.002201740\n","[148,   165] loss: 0.002501429\n","[149,   165] loss: 0.005225292\n","[150,   165] loss: 0.009864649\n","Finished Training\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"bnZflwBUZoQV","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"5fdad30e-595f-4b55-ad09-1ed832d83b16","executionInfo":{"status":"ok","timestamp":1584443318932,"user_tz":-540,"elapsed":320146,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["correct = 0\n","total = 0\n","preds = []\n","with torch.no_grad():\n","    for data in testloader: #미리 정의해둔 testloader로 test데이터 불러온다\n","        images = data\n","        images = images.to(device)\n","        outputs = net(images)\n","        _, predicted = torch.max(outputs.data, 1)\n","        for pred in predicted: #만들어둔 모델을 이용해 test에대한 y값을 예측한 prediction data를 만든다\n","          preds.append(pred.cpu().numpy())\n","print(preds[:10])"],"execution_count":13,"outputs":[{"output_type":"stream","text":["[array(8), array(0), array(5), array(3), array(8), array(1), array(9), array(6), array(6), array(0)]\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"hajKAQgscIZW","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":195},"outputId":"a97ce131-d11a-4e9c-82a7-3124309115bf","executionInfo":{"status":"ok","timestamp":1584443318934,"user_tz":-540,"elapsed":320130,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["preds[:10]"],"execution_count":14,"outputs":[{"output_type":"execute_result","data":{"text/plain":["[array(8),\n"," array(0),\n"," array(5),\n"," array(3),\n"," array(8),\n"," array(1),\n"," array(9),\n"," array(6),\n"," array(6),\n"," array(0)]"]},"metadata":{"tags":[]},"execution_count":14}]},{"cell_type":"code","metadata":{"id":"zRa8AW3vDxkH","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":35},"outputId":"7d643abf-aa14-4166-9c71-13dc0d86ee46","executionInfo":{"status":"ok","timestamp":1584443318935,"user_tz":-540,"elapsed":320116,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["import numpy as np\n","preds = np.array(preds)\n","print(preds.shape)"],"execution_count":15,"outputs":[{"output_type":"stream","text":["(18000,)\n"],"name":"stdout"}]},{"cell_type":"code","metadata":{"id":"dBNbI1qjDyn8","colab_type":"code","colab":{"base_uri":"https://localhost:8080/","height":194},"outputId":"c1fbc912-bc45-45e3-e607-d996b3949e1c","executionInfo":{"status":"ok","timestamp":1584443318938,"user_tz":-540,"elapsed":320103,"user":{"displayName":"­최혜빈(엘텍공과대학 휴먼기계바이오공학부)","photoUrl":"","userId":"12605805000524052897"}}},"source":["sample_submission[\"Category\"] = pd.Series(preds)\n","sample_submission.head()"],"execution_count":16,"outputs":[{"output_type":"execute_result","data":{"text/html":["<div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Id</th>\n","      <th>Category</th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0</th>\n","      <td>57808</td>\n","      <td>8</td>\n","    </tr>\n","    <tr>\n","      <th>1</th>\n","      <td>4960</td>\n","      <td>0</td>\n","    </tr>\n","    <tr>\n","      <th>2</th>\n","      <td>35755</td>\n","      <td>5</td>\n","    </tr>\n","    <tr>\n","      <th>3</th>\n","      <td>15543</td>\n","      <td>3</td>\n","    </tr>\n","    <tr>\n","      <th>4</th>\n","      <td>48968</td>\n","      <td>8</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>"],"text/plain":["      Id  Category\n","0  57808         8\n","1   4960         0\n","2  35755         5\n","3  15543         3\n","4  48968         8"]},"metadata":{"tags":[]},"execution_count":16}]},{"cell_type":"code","metadata":{"id":"Q-eA6PCIDzSi","colab_type":"code","colab":{}},"source":["sample_submission.to_csv(\"torch_submission.csv\", index=False) #제출해보니 accuracy 0.985정도 나왔습니당.."],"execution_count":0,"outputs":[]}]}