GPT_Neo_Fine-tune.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "eefe4b59",
   "metadata": {},
   "source": [
    "# First we must install Pytorch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6825812b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting package metadata (current_repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "\n",
      "==> WARNING: A newer version of conda exists. <==\n",
      "  current version: 4.8.3\n",
      "  latest version: 4.10.1\n",
      "\n",
      "Please update conda by running\n",
      "\n",
      "    $ conda update -n base -c defaults conda\n",
      "\n",
      "\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!conda install pytorch torchvision torchaudio cudatoolkit=11.1 -c pytorch -c nvidia -y"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c2b689c",
   "metadata": {},
   "source": [
    "# Lets quickly look at a more naive approach to fine-tuning, one I originally explored. "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fe1ca66f",
   "metadata": {},
   "source": [
    "## One easy way to fine-tune small transformers is to use a library called Happytransformer. Happytransformer is a package built on top of the Hugging face transformer library. Using it, you can normally fine-tune small transformers very easiliy. Lets install it."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e04081ef",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (4.6.1)\n",
      "Requirement already satisfied: happytransformer in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (2.2.3)\n",
      "Requirement already satisfied: huggingface-hub==0.0.8 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (0.0.8)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (0.10.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (1.20.2)\n",
      "Requirement already satisfied: sacremoses in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (0.0.45)\n",
      "Requirement already satisfied: requests in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (2.25.1)\n",
      "Requirement already satisfied: importlib-metadata in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (3.10.0)\n",
      "Requirement already satisfied: filelock in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: packaging in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: tqdm>=4.27 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (4.49.0)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from transformers) (2021.4.4)\n",
      "Requirement already satisfied: protobuf in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from happytransformer) (3.17.3)\n",
      "Requirement already satisfied: datasets>=1.6.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from happytransformer) (1.8.0)\n",
      "Requirement already satisfied: torch>=1.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from happytransformer) (1.8.1)\n",
      "Requirement already satisfied: sentencepiece in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from happytransformer) (0.1.95)\n",
      "Requirement already satisfied: multiprocess in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets>=1.6.0->happytransformer) (0.70.12)\n",
      "Requirement already satisfied: pyarrow<4.0.0,>=1.0.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets>=1.6.0->happytransformer) (3.0.0)\n",
      "Requirement already satisfied: xxhash in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets>=1.6.0->happytransformer) (2.0.2)\n",
      "Requirement already satisfied: pandas in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets>=1.6.0->happytransformer) (1.2.4)\n",
      "Requirement already satisfied: fsspec in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets>=1.6.0->happytransformer) (2021.6.0)\n",
      "Requirement already satisfied: dill in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets>=1.6.0->happytransformer) (0.3.4)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests->transformers) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests->transformers) (2021.5.30)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests->transformers) (1.26.5)\n",
      "Requirement already satisfied: typing_extensions in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from torch>=1.0->happytransformer) (3.7.4.3)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from importlib-metadata->transformers) (3.4.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from pandas->datasets>=1.6.0->happytransformer) (2.8.1)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from pandas->datasets>=1.6.0->happytransformer) (2021.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas->datasets>=1.6.0->happytransformer) (1.15.0)\n",
      "Requirement already satisfied: joblib in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: click in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from sacremoses->transformers) (8.0.1)\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers happytransformer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f2a45d21",
   "metadata": {},
   "source": [
    "## We will be testing this method by fine-tunning models on Shakespeare on my RTX 3090."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d85ec83f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env: CUDA_VISIBLE_DEVICES=0\n"
     ]
    }
   ],
   "source": [
    "%env CUDA_VISIBLE_DEVICES=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b7b71e17",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "06/14/2021 04:17:22 - INFO - happytransformer.happy_transformer -   Using model: cuda\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-20e1d7930c9d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mmodel_to_use\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"EleutherAI/gpt-neo-1.3B\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mhappy_gen\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mHappyGeneration\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"GPT-NEO\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_to_use\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0mhappy_gen\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"train.csv\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/happytransformer/happy_generation.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model_type, model_name, load_path)\u001b[0m\n\u001b[1;32m     55\u001b[0m         \u001b[0mdevice_number\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdetect_cuda_device_number\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 57\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pipeline\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextGenerationPipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mdevice_number\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     58\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_trainer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mGENTrainer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodel_type\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_device\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlogger\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/transformers/pipelines/text_generation.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, return_full_text, *args, **kwargs)\u001b[0m\n\u001b[1;32m     47\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     48\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreturn_full_text\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 49\u001b[0;31m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     50\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     51\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_model_type\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mALLOWED_MODELS\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/transformers/pipelines/base.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, model, tokenizer, feature_extractor, modelcard, framework, task, args_parser, device, binary_output)\u001b[0m\n\u001b[1;32m    548\u001b[0m         \u001b[0;31m# Special handling\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    549\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"pt\"\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"cuda\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 550\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    551\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    552\u001b[0m         \u001b[0;31m# Update config with task specific parameters\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mto\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    671\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_complex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 673\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    674\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    675\u001b[0m     def register_backward_hook(\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    385\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    386\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mmodule\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchildren\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 387\u001b[0;31m             \u001b[0mmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    388\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    389\u001b[0m         \u001b[0;32mdef\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtensor_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_apply\u001b[0;34m(self, fn)\u001b[0m\n\u001b[1;32m    407\u001b[0m                 \u001b[0;31m# `with torch.no_grad():`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    408\u001b[0m                 \u001b[0;32mwith\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mno_grad\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 409\u001b[0;31m                     \u001b[0mparam_applied\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    410\u001b[0m                 \u001b[0mshould_use_set_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompute_should_use_set_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mparam\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparam_applied\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    411\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mshould_use_set_data\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36mconvert\u001b[0;34m(t)\u001b[0m\n\u001b[1;32m    669\u001b[0m                 return t.to(device, dtype if t.is_floating_point() or t.is_complex() else None,\n\u001b[1;32m    670\u001b[0m                             non_blocking, memory_format=convert_to_format)\n\u001b[0;32m--> 671\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdtype\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_floating_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mt\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mis_complex\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_blocking\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    672\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_apply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mconvert\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "from happytransformer import HappyGeneration \n",
    "model_to_use = \"EleutherAI/gpt-neo-125M\"\n",
    "#1.3B Won't work on 24GB or less cards\n",
    "# model_to_use = \"EleutherAI/gpt-neo-1.3B\"\n",
    "\n",
    "happy_gen = HappyGeneration(\"GPT-NEO\", model_to_use)\n",
    "happy_gen.train(\"train.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "56f9a5c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#this is clean up free up GPU VRAM\n",
    "try:\n",
    "    del happy_gen\n",
    "except:\n",
    "    pass\n",
    "import gc \n",
    "import torch\n",
    "gc.collect()\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7046caaa",
   "metadata": {},
   "source": [
    "## We can see that even the relatively small 125M model, that it takes 10GB of VRAM to fine-tune the model, and the 1.3B parameter model can't fit on 24GB, let alone the 2.7B model.  This was even with a batch size of 1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a388286d",
   "metadata": {},
   "source": [
    "# Is there hope to fine-tune these larger models then with consumer grade hardware then?  Yes, but we need to use a library called DeepSpeed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e40835c",
   "metadata": {},
   "source": [
    "# First we need to clone the DeepSpeed Repo, as we must build some optional items in the package from the source"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22a7db7e",
   "metadata": {},
   "source": [
    "## DeepSpeed is a Deep Learning optimization library by Microsoft that allows researchers to more easily run and train larger models that they otherwise would not be able to."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "5be6acfe",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'DeepSpeed'...\n",
      "remote: Enumerating objects: 9447, done.\u001b[K\n",
      "remote: Counting objects: 100% (207/207), done.\u001b[K\n",
      "remote: Compressing objects: 100% (151/151), done.\u001b[K\n",
      "remote: Total 9447 (delta 103), reused 109 (delta 53), pack-reused 9240\u001b[K\n",
      "Receiving objects: 100% (9447/9447), 18.52 MiB | 10.32 MiB/s, done.\n",
      "Resolving deltas: 100% (6412/6412), done.\n",
      "Note: switching to '2d302d6abb2cfa181f63320da3ed1be45e34ded3'.\n",
      "\n",
      "You are in 'detached HEAD' state. You can look around, make experimental\n",
      "changes and commit them, and you can discard any commits you make in this\n",
      "state without impacting any branches by switching back to a branch.\n",
      "\n",
      "If you want to create a new branch to retain commits you create, you may\n",
      "do so (now or later) by using -c with the switch command. Example:\n",
      "\n",
      "  git switch -c <new-branch-name>\n",
      "\n",
      "Or undo this operation with:\n",
      "\n",
      "  git switch -\n",
      "\n",
      "Turn off this advice by setting config variable advice.detachedHead to false\n",
      "\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/microsoft/DeepSpeed -b v0.4.0"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e8bc2a9f",
   "metadata": {},
   "source": [
    "## We are now going to run ls to see our folder structure, we should now see DeepSpeed, we are going to cd into that folder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "39432859",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DeepSpeed  GPT_Neo_Fine-tune.ipynb  test  train.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "4f239c76",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/shared_drive/projects/personal/finetune_vid/DeepSpeed\n"
     ]
    }
   ],
   "source": [
    "%cd DeepSpeed"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "059634e1",
   "metadata": {},
   "source": [
    "## If we run ls again, we will see the contents of the DeepSpeed repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "b4b492a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "azure\t\t    csrc\t       install.sh   requirements  version.txt\r\n",
      "bin\t\t    deepspeed\t       LICENSE\t    SECURITY.md\r\n",
      "CODE_OF_CONDUCT.md  DeepSpeedExamples  MANIFEST.in  setup.cfg\r\n",
      "CODEOWNERS\t    docker\t       op_builder   setup.py\r\n",
      "CONTRIBUTING.md     docs\t       README.md    tests\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ef9f586",
   "metadata": {},
   "source": [
    "## We are now going to install DeepSpeed from source using a flag to insure that all the needed ops are installed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6e90edc2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing /mnt/shared_drive/projects/personal/finetune_vid/DeepSpeed\n",
      "\u001b[33m  DEPRECATION: A future pip version will change local packages to be built in-place without first copying to a temporary directory. We recommend you use --use-feature=in-tree-build to test your packages with this new behavior before it becomes the default.\n",
      "   pip 21.3 will remove support for this functionality. You can find discussion regarding this at https://github.com/pypa/pip/issues/7555.\u001b[0m\n",
      "Requirement already satisfied: torch>=1.2 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (1.8.1)\n",
      "Requirement already satisfied: torchvision>=0.4.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (0.9.1)\n",
      "Requirement already satisfied: tqdm in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (4.49.0)\n",
      "Requirement already satisfied: tensorboardX==1.8 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (1.8)\n",
      "Requirement already satisfied: ninja in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (1.10.0.post2)\n",
      "Requirement already satisfied: numpy in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (1.20.2)\n",
      "Requirement already satisfied: psutil in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (5.8.0)\n",
      "Requirement already satisfied: packaging in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (20.9)\n",
      "Requirement already satisfied: triton in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from deepspeed==0.4.0+2d302d6) (0.4.2)\n",
      "Requirement already satisfied: protobuf>=3.2.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from tensorboardX==1.8->deepspeed==0.4.0+2d302d6) (3.17.3)\n",
      "Requirement already satisfied: six in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from tensorboardX==1.8->deepspeed==0.4.0+2d302d6) (1.15.0)\n",
      "Requirement already satisfied: typing_extensions in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from torch>=1.2->deepspeed==0.4.0+2d302d6) (3.7.4.3)\n",
      "Requirement already satisfied: pillow>=4.1.1 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from torchvision>=0.4.0->deepspeed==0.4.0+2d302d6) (8.2.0)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from packaging->deepspeed==0.4.0+2d302d6) (2.4.7)\n",
      "Building wheels for collected packages: deepspeed\n",
      "  Building wheel for deepspeed (setup.py) ... \u001b[?25l/^C\n",
      "\u001b[?25canceled\n",
      "\u001b[31mERROR: Operation cancelled by user\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!DS_BUILD_OPS=1 pip install ."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "eec9fdcd",
   "metadata": {},
   "source": [
    "## We can now make sure that DeepSpeed is properly installed by running the snippet below.  All of the compatible ops should be installed.  Some are required that like cpu_adam and transformer.  Others may not be, such as async_io.  Personally all were compatible with my system but async.io"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fd5c921e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------------------------------\n",
      "DeepSpeed C++/CUDA extension op report\n",
      "--------------------------------------------------\n",
      "NOTE: Ops not installed will be just-in-time (JIT) compiled at\n",
      "      runtime if needed. Op compatibility means that your system\n",
      "      meet the required dependencies to JIT install the op.\n",
      "--------------------------------------------------\n",
      "JIT compiled ops requires ninja\n",
      "ninja .................. \u001b[92m[OKAY]\u001b[0m\n",
      "--------------------------------------------------\n",
      "op name ................ installed .. compatible\n",
      "--------------------------------------------------\n",
      "cpu_adam ............... \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "fused_adam ............. \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "fused_lamb ............. \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "sparse_attn ............ \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "transformer ............ \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "stochastic_transformer . \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "\u001b[93m [WARNING] \u001b[0m async_io requires the libraries: ['libaio-dev'] but are missing. Can be fixed by: `apt install libaio-dev`.\n",
      "async_io ............... \u001b[93m[NO]\u001b[0m ....... \u001b[93m[NO]\u001b[0m\n",
      "transformer_inference .. \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "utils .................. \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "quantizer .............. \u001b[92m[YES]\u001b[0m ...... \u001b[92m[OKAY]\u001b[0m\n",
      "--------------------------------------------------\n",
      "DeepSpeed general environment info:\n",
      "torch install path ............... ['/home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/torch']\n",
      "torch version .................... 1.8.1\n",
      "torch cuda version ............... 11.1\n",
      "nvcc version ..................... 11.1\n",
      "deepspeed install path ........... ['/home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages/deepspeed']\n",
      "deepspeed info ................... 0.4.0+2d302d6, 2d302d6, HEAD\n",
      "deepspeed wheel compiled w. ...... torch 1.8, cuda 11.1\n"
     ]
    }
   ],
   "source": [
    "!ds_report"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "19f11ab6",
   "metadata": {},
   "source": [
    "# Next we need to download the repo that will be actually finetuning the GPT Neo model using DeepSpeed."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ce1c313c",
   "metadata": {},
   "source": [
    "## First we need to exit the DeepSpeed repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "a244d69a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/shared_drive/projects/personal/finetune_vid\n"
     ]
    }
   ],
   "source": [
    "%cd .."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "5ab4bfae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DeepSpeed  GPT_Neo_Fine-tune.ipynb  test  train.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbd17a97",
   "metadata": {},
   "source": [
    "## Now we clone the finetuning repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "41a37c89",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cloning into 'finetune-gpt2xl'...\n",
      "remote: Enumerating objects: 354, done.\u001b[K\n",
      "remote: Counting objects: 100% (50/50), done.\u001b[K\n",
      "remote: Compressing objects: 100% (50/50), done.\u001b[K\n",
      "remote: Total 354 (delta 2), reused 0 (delta 0), pack-reused 304\u001b[K\n",
      "Receiving objects: 100% (354/354), 3.41 MiB | 3.23 MiB/s, done.\n",
      "Resolving deltas: 100% (221/221), done.\n"
     ]
    }
   ],
   "source": [
    "!git clone https://github.com/Xirider/finetune-gpt2xl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fec84cb9",
   "metadata": {},
   "source": [
    "## Now we need enter the pulled repo"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "68d7f110",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DeepSpeed  finetune-gpt2xl  GPT_Neo_Fine-tune.ipynb  train.csv\r\n"
     ]
    }
   ],
   "source": [
    "!ls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "60e7ca1b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/mnt/shared_drive/projects/personal/finetune_vid/finetune-gpt2xl\n"
     ]
    }
   ],
   "source": [
    "%cd finetune-gpt2xl"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8d67d015",
   "metadata": {},
   "source": [
    "## Lastly, we need to download the datasets library that this repo uses."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "14162e69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: datasets in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (1.8.0)\n",
      "Requirement already satisfied: requests>=2.19.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (2.25.1)\n",
      "Requirement already satisfied: huggingface-hub<0.1.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (0.0.8)\n",
      "Requirement already satisfied: pyarrow<4.0.0,>=1.0.0 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (3.0.0)\n",
      "Requirement already satisfied: importlib-metadata in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (3.10.0)\n",
      "Requirement already satisfied: xxhash in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (2.0.2)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (1.20.2)\n",
      "Requirement already satisfied: tqdm<4.50.0,>=4.27 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (4.49.0)\n",
      "Requirement already satisfied: fsspec in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (2021.6.0)\n",
      "Requirement already satisfied: multiprocess in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (0.70.12)\n",
      "Requirement already satisfied: dill in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (0.3.4)\n",
      "Requirement already satisfied: pandas in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (1.2.4)\n",
      "Requirement already satisfied: packaging in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from datasets) (20.9)\n",
      "Requirement already satisfied: filelock in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from huggingface-hub<0.1.0->datasets) (3.0.12)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (1.26.5)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from requests>=2.19.0->datasets) (2021.5.30)\n",
      "Requirement already satisfied: zipp>=0.5 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from importlib-metadata->datasets) (3.4.1)\n",
      "Requirement already satisfied: typing-extensions>=3.6.4 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from importlib-metadata->datasets) (3.7.4.3)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from packaging->datasets) (2.4.7)\n",
      "Requirement already satisfied: pytz>=2017.3 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from pandas->datasets) (2021.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7.3 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from pandas->datasets) (2.8.1)\n",
      "Requirement already satisfied: six>=1.5 in /home/blake/anaconda3/envs/gptneo_finetuned/lib/python3.7/site-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install datasets"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91547c7e",
   "metadata": {},
   "source": [
    "# At this point we are able to finetune GPT Neo(including 2.7B) and other GPT models"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c811f9a8",
   "metadata": {},
   "source": [
    "## For GPT NEO 2.7B parameters, we need a high end machine.  Roughly 70GB of RAM is the minimum required for it, along with roughly 16GB of VRAM. GPT Neo 1.3B and other smaller GPT2 models don't have as high of requirements. This can rented for an ok price from a cloud provider if you dont have a powerful enough machine."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ba89aec0",
   "metadata": {},
   "source": [
    "## Lets now finetune model with the provided Shakespeare dataset with the example flags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d6b1af35",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2021-06-14 15:33:34,602] [WARNING] [runner.py:122:fetch_hostfile] Unable to find hostfile, will proceed with training with local resources only.\n",
      "[2021-06-14 15:33:34,610] [INFO] [runner.py:360:main] cmd = /home/blake/anaconda3/envs/gptneo_finetuned/bin/python -u -m deepspeed.launcher.launch --world_info=eyJsb2NhbGhvc3QiOiBbMF19 --master_addr=127.0.0.1 --master_port=29500 run_clm.py --deepspeed ds_config_gptneo.json --model_name_or_path EleutherAI/gpt-neo-1.3B --train_file train.csv --validation_file validation.csv --do_train --do_eval --fp16 --overwrite_cache --evaluation_strategy=steps --output_dir finetuned --num_train_epochs 1 --eval_steps 15 --gradient_accumulation_steps 2 --per_device_train_batch_size 4 --use_fast_tokenizer False --learning_rate 5e-06 --warmup_steps 10\n",
      "[2021-06-14 15:33:35,151] [INFO] [launch.py:80:main] WORLD INFO DICT: {'localhost': [0]}\n",
      "[2021-06-14 15:33:35,152] [INFO] [launch.py:89:main] nnodes=1, num_local_procs=1, node_rank=0\n",
      "[2021-06-14 15:33:35,152] [INFO] [launch.py:101:main] global_rank_mapping=defaultdict(<class 'list'>, {'localhost': [0]})\n",
      "[2021-06-14 15:33:35,152] [INFO] [launch.py:102:main] dist_world_size=1\n",
      "[2021-06-14 15:33:35,152] [INFO] [launch.py:105:main] Setting CUDA_VISIBLE_DEVICES=0\n",
      "[2021-06-14 15:33:36,237] [INFO] [distributed.py:47:init_distributed] Initializing torch distributed with backend: nccl\n",
      "WARNING:__main__:Process rank: 0, device: cuda:0, n_gpu: 1distributed training: True, 16-bits training: True\n",
      "INFO:__main__:Training/evaluation parameters TrainingArguments(output_dir=finetuned, overwrite_output_dir=False, do_train=True, do_eval=True, do_predict=False, evaluation_strategy=IntervalStrategy.STEPS, prediction_loss_only=False, per_device_train_batch_size=4, per_device_eval_batch_size=8, gradient_accumulation_steps=2, eval_accumulation_steps=None, learning_rate=5e-06, weight_decay=0.0, adam_beta1=0.9, adam_beta2=0.999, adam_epsilon=1e-08, max_grad_norm=1.0, num_train_epochs=1.0, max_steps=-1, lr_scheduler_type=SchedulerType.LINEAR, warmup_ratio=0.0, warmup_steps=10, logging_dir=runs/Jun14_15-33-36_blake-desktop, logging_strategy=IntervalStrategy.STEPS, logging_first_step=False, logging_steps=500, save_strategy=IntervalStrategy.STEPS, save_steps=500, save_total_limit=None, no_cuda=False, seed=42, fp16=True, fp16_opt_level=O1, fp16_backend=auto, fp16_full_eval=False, local_rank=0, tpu_num_cores=None, tpu_metrics_debug=False, debug=[], dataloader_drop_last=False, eval_steps=15, dataloader_num_workers=0, past_index=-1, run_name=finetuned, disable_tqdm=False, remove_unused_columns=True, label_names=None, load_best_model_at_end=False, metric_for_best_model=None, greater_is_better=None, ignore_data_skip=False, sharded_ddp=[], deepspeed=ds_config_gptneo.json, label_smoothing_factor=0.0, adafactor=False, group_by_length=False, length_column_name=length, report_to=['tensorboard'], ddp_find_unused_parameters=None, dataloader_pin_memory=True, skip_memory_metrics=False, use_legacy_prediction_loop=False, push_to_hub=False, resume_from_checkpoint=None, _n_gpu=1, mp_parameters=)\n",
      "WARNING:datasets.builder:Using custom data configuration default-03765c38df4609a9\n",
      "WARNING:datasets.builder:Reusing dataset csv (/home/blake/.cache/huggingface/datasets/csv/default-03765c38df4609a9/0.0.0/2dc6629a9ff6b5697d82c25b73731dd440507a69cbce8b425db50b751e8fcfd0)\n",
      "[INFO|configuration_utils.py:517] 2021-06-14 15:33:36,882 >> loading configuration file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json from cache at /home/blake/.cache/huggingface/transformers/42252c2220ae3f9f1ea86a994b63e1dcab20953ba8982117c2384587f7c01c5d.0481f2d6b709486a0897fbfb2477d85f33e6a2843fd79a2261982c19c5b42624\n",
      "[INFO|configuration_utils.py:553] 2021-06-14 15:33:36,883 >> Model config GPTNeoConfig {\n",
      "  \"activation_function\": \"gelu_new\",\n",
      "  \"architectures\": [\n",
      "    \"GPTNeoForCausalLM\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0,\n",
      "  \"attention_layers\": [\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\"\n",
      "  ],\n",
      "  \"attention_types\": [\n",
      "    [\n",
      "      [\n",
      "        \"global\",\n",
      "        \"local\"\n",
      "      ],\n",
      "      12\n",
      "    ]\n",
      "  ],\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"embed_dropout\": 0,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": null,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"max_position_embeddings\": 2048,\n",
      "  \"model_type\": \"gpt_neo\",\n",
      "  \"num_heads\": 16,\n",
      "  \"num_layers\": 24,\n",
      "  \"resid_dropout\": 0,\n",
      "  \"summary_activation\": null,\n",
      "  \"summary_first_dropout\": 0.1,\n",
      "  \"summary_proj_to_labels\": true,\n",
      "  \"summary_type\": \"cls_index\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"text-generation\": {\n",
      "      \"do_sample\": true,\n",
      "      \"max_length\": 50,\n",
      "      \"temperature\": 0.9\n",
      "    }\n",
      "  },\n",
      "  \"tokenizer_class\": \"GPT2Tokenizer\",\n",
      "  \"transformers_version\": \"4.6.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50257,\n",
      "  \"window_size\": 256\n",
      "}\n",
      "\n",
      "[INFO|configuration_utils.py:517] 2021-06-14 15:33:37,107 >> loading configuration file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json from cache at /home/blake/.cache/huggingface/transformers/42252c2220ae3f9f1ea86a994b63e1dcab20953ba8982117c2384587f7c01c5d.0481f2d6b709486a0897fbfb2477d85f33e6a2843fd79a2261982c19c5b42624\n",
      "[INFO|configuration_utils.py:553] 2021-06-14 15:33:37,107 >> Model config GPTNeoConfig {\n",
      "  \"activation_function\": \"gelu_new\",\n",
      "  \"architectures\": [\n",
      "    \"GPTNeoForCausalLM\"\n",
      "  ],\n",
      "  \"attention_dropout\": 0,\n",
      "  \"attention_layers\": [\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\",\n",
      "    \"global\",\n",
      "    \"local\"\n",
      "  ],\n",
      "  \"attention_types\": [\n",
      "    [\n",
      "      [\n",
      "        \"global\",\n",
      "        \"local\"\n",
      "      ],\n",
      "      12\n",
      "    ]\n",
      "  ],\n",
      "  \"bos_token_id\": 50256,\n",
      "  \"embed_dropout\": 0,\n",
      "  \"eos_token_id\": 50256,\n",
      "  \"gradient_checkpointing\": false,\n",
      "  \"hidden_size\": 2048,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": null,\n",
      "  \"layer_norm_epsilon\": 1e-05,\n",
      "  \"max_position_embeddings\": 2048,\n",
      "  \"model_type\": \"gpt_neo\",\n",
      "  \"num_heads\": 16,\n",
      "  \"num_layers\": 24,\n",
      "  \"resid_dropout\": 0,\n",
      "  \"summary_activation\": null,\n",
      "  \"summary_first_dropout\": 0.1,\n",
      "  \"summary_proj_to_labels\": true,\n",
      "  \"summary_type\": \"cls_index\",\n",
      "  \"summary_use_proj\": true,\n",
      "  \"task_specific_params\": {\n",
      "    \"text-generation\": {\n",
      "      \"do_sample\": true,\n",
      "      \"max_length\": 50,\n",
      "      \"temperature\": 0.9\n",
      "    }\n",
      "  },\n",
      "  \"tokenizer_class\": \"GPT2Tokenizer\",\n",
      "  \"transformers_version\": \"4.6.1\",\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 50257,\n",
      "  \"window_size\": 256\n",
      "}\n",
      "\n",
      "[INFO|tokenization_utils_base.py:1717] 2021-06-14 15:33:38,677 >> loading file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/vocab.json from cache at /home/blake/.cache/huggingface/transformers/6111bc9bbed617156dc5c0b9fa9d6793147619aad08053f03b3697f1a5027973.a1b97b074a5ac71fad0544c8abc1b3581803d73832476184bde6cff06a67b6bb\n",
      "[INFO|tokenization_utils_base.py:1717] 2021-06-14 15:33:38,677 >> loading file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/merges.txt from cache at /home/blake/.cache/huggingface/transformers/ec80888cdc98108f625f7ec7a29ec449eb361ae1325aa1e7e63006ce962c071c.f5b91da9e34259b8f4d88dbc97c740667a0e8430b96314460cdb04e86d4fc435\n",
      "[INFO|tokenization_utils_base.py:1717] 2021-06-14 15:33:38,677 >> loading file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/added_tokens.json from cache at None\n",
      "[INFO|tokenization_utils_base.py:1717] 2021-06-14 15:33:38,677 >> loading file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/special_tokens_map.json from cache at /home/blake/.cache/huggingface/transformers/1ae5a53fe395100a9213705940d92cc94554a2269777c062d951d1b710c39bb8.3ae9ae72462581d20e36bc528e9c47bb30cd671bb21add40ca0b24a0be9fac22\n",
      "[INFO|tokenization_utils_base.py:1717] 2021-06-14 15:33:38,677 >> loading file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/tokenizer_config.json from cache at /home/blake/.cache/huggingface/transformers/5fe35a59019a6fb05bfa29a31b59d407cd81ae59da93e6953772a783b740b4c0.c31b6b7d3225be0c43bc0f8e5d84d03a8b49fdb6b9f6009bbfff1f9cc5ec18bc\n",
      "[INFO|tokenization_utils_base.py:1717] 2021-06-14 15:33:38,677 >> loading file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/tokenizer.json from cache at None\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO|modeling_utils.py:1155] 2021-06-14 15:33:39,062 >> loading weights file https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/pytorch_model.bin from cache at /home/blake/.cache/huggingface/transformers/7c5fac9d60b015cbc7c007ab8fe6d0512787fbaef81968922959898c49468d73.4c6a483fbfb5a25ac384bfcd71a1ff15245f06583a00c4ab4c44ed0f761f0b08\n",
      "[INFO|modeling_utils.py:1339] 2021-06-14 15:33:52,868 >> All model checkpoint weights were used when initializing GPTNeoForCausalLM.\n",
      "\n",
      "[INFO|modeling_utils.py:1348] 2021-06-14 15:33:52,868 >> All the weights of GPTNeoForCausalLM were initialized from the model checkpoint at EleutherAI/gpt-neo-1.3B.\n",
      "If your task is similar to the task the model of the checkpoint was trained on, you can already use GPTNeoForCausalLM for predictions without further training.\n",
      "  0%|                                                     | 0/1 [00:00<?, ?ba/s][WARNING|tokenization_utils_base.py:3171] 2021-06-14 15:33:57,946 >> Token indices sequence length is longer than the specified maximum sequence length for this model (1462828 > 2048). Running this sequence through the model will result in indexing errors\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:04<00:00,  4.50s/ba]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00, 60.93ba/s]\n",
      "run_clm.py:361: DeprecationWarning: The 'warn' method is deprecated, use 'warning' instead\n",
      "  f\"The tokenizer picked seems to have a very large `model_max_length` ({tokenizer.model_max_length}). \"\n",
      "WARNING:__main__:The tokenizer picked seems to have a very large `model_max_length` (2048). Picking 1024 instead. You can change that default value by passing --block_size xxx.\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  1.09ba/s]\n",
      "100%|████████████████████████████████████████████| 1/1 [00:00<00:00, 255.55ba/s]\n",
      "[INFO|trainer.py:415] 2021-06-14 15:34:00,185 >> Using amp fp16 backend\n",
      "[2021-06-14 15:34:00,273] [INFO] [logging.py:60:log_dist] [Rank 0] DeepSpeed info: version=0.4.0+2d302d6, git-hash=2d302d6, git-branch=HEAD\n",
      "[2021-06-14 15:34:03,947] [INFO] [utils.py:13:_initialize_parameter_parallel_groups] data_parallel_size: 1, parameter_parallel_size: 1\n",
      "[2021-06-14 15:34:04,004] [INFO] [engine.py:173:__init__] DeepSpeed Flops Profiler Enabled: False\n",
      "Adam Optimizer #0 is created with scalar arithmetic capability.\n",
      "Config: alpha=0.000005, betas=(0.900000, 0.999000), weight_decay=0.000000, adam_w=1\n",
      "[2021-06-14 15:34:04,852] [INFO] [engine.py:693:_configure_optimizer] Using DeepSpeed Optimizer param name adamw as basic optimizer\n",
      "[2021-06-14 15:34:04,852] [INFO] [engine.py:697:_configure_optimizer] DeepSpeed Basic Optimizer = DeepSpeedCPUAdam\n",
      "Checking ZeRO support for optimizer=DeepSpeedCPUAdam type=<class 'deepspeed.ops.adam.cpu_adam.DeepSpeedCPUAdam'>\n",
      "[2021-06-14 15:34:04,852] [INFO] [logging.py:60:log_dist] [Rank 0] Creating fp16 ZeRO stage 2 optimizer\n",
      "[2021-06-14 15:34:04,852] [INFO] [stage2.py:105:__init__] Reduce bucket size 200000000.0\n",
      "[2021-06-14 15:34:04,852] [INFO] [stage2.py:106:__init__] Allgather bucket size 200000000.0\n",
      "[2021-06-14 15:34:04,852] [INFO] [stage2.py:107:__init__] CPU Offload: True\n",
      "[2021-06-14 15:34:15,977] [INFO] [stage2.py:409:__init__] optimizer state initialized\n",
      "[2021-06-14 15:34:15,977] [INFO] [logging.py:60:log_dist] [Rank 0] DeepSpeed Final Optimizer = adamw\n",
      "[2021-06-14 15:34:15,977] [INFO] [engine.py:505:_configure_lr_scheduler] DeepSpeed using configured LR scheduler = WarmupLR\n",
      "[2021-06-14 15:34:15,978] [INFO] [logging.py:60:log_dist] [Rank 0] DeepSpeed LR Scheduler = <deepspeed.runtime.lr_schedules.WarmupLR object at 0x7f1c530a7b10>\n",
      "[2021-06-14 15:34:15,978] [INFO] [logging.py:60:log_dist] [Rank 0] step=0, skipped=0, lr=[5e-06], mom=[[0.9, 0.999]]\n",
      "[2021-06-14 15:34:15,978] [INFO] [config.py:900:print] DeepSpeedEngine configuration:\n",
      "[2021-06-14 15:34:15,978] [INFO] [config.py:904:print]   activation_checkpointing_config  {\n",
      "    \"partition_activations\": false, \n",
      "    \"contiguous_memory_optimization\": false, \n",
      "    \"cpu_checkpointing\": false, \n",
      "    \"number_checkpoints\": null, \n",
      "    \"synchronize_checkpoint_boundary\": false, \n",
      "    \"profile\": false\n",
      "}\n",
      "[2021-06-14 15:34:15,978] [INFO] [config.py:904:print]   aio_config ................... {'block_size': 1048576, 'queue_depth': 8, 'thread_count': 1, 'single_submit': False, 'overlap_events': True}\n",
      "[2021-06-14 15:34:15,978] [INFO] [config.py:904:print]   allreduce_always_fp32 ........ False\n",
      "[2021-06-14 15:34:15,978] [INFO] [config.py:904:print]   amp_enabled .................. False\n",
      "[2021-06-14 15:34:15,978] [INFO] [config.py:904:print]   amp_params ................... False\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   checkpoint_tag_validation_enabled  True\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   checkpoint_tag_validation_fail  False\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   disable_allgather ............ False\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   dump_state ................... False\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   dynamic_loss_scale_args ...... {'init_scale': 65536, 'scale_window': 1000, 'delayed_shift': 2, 'min_scale': 1}\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_enabled ........... False\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_gas_boundary_resolution  1\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_layer_name ........ bert.encoder.layer\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_layer_num ......... 0\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_max_iter .......... 100\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_stability ......... 1e-06\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_tol ............... 0.01\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   eigenvalue_verbose ........... False\n",
      "[2021-06-14 15:34:15,979] [INFO] [config.py:904:print]   elasticity_enabled ........... False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   flops_profiler_config ........ {\n",
      "    \"enabled\": false, \n",
      "    \"profile_step\": 1, \n",
      "    \"module_depth\": -1, \n",
      "    \"top_modules\": 1, \n",
      "    \"detailed\": true, \n",
      "    \"output_file\": null\n",
      "}\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   fp16_enabled ................. True\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   fp16_mixed_quantize .......... False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   global_rank .................. 0\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   gradient_accumulation_steps .. 2\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   gradient_clipping ............ 1.0\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   gradient_predivide_factor .... 1.0\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   initial_dynamic_scale ........ 65536\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   loss_scale ................... 0\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   memory_breakdown ............. False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   optimizer_legacy_fusion ...... False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   optimizer_name ............... adamw\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   optimizer_params ............. {'lr': 5e-06, 'betas': [0.9, 0.999], 'eps': 1e-08, 'weight_decay': 0.0}\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   pipeline ..................... {'stages': 'auto', 'partition': 'best', 'seed_layers': False, 'activation_checkpoint_interval': 0}\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   pld_enabled .................. False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   pld_params ................... False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   prescale_gradients ........... False\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_change_rate ......... 0.001\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_groups .............. 1\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_offset .............. 1000\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_period .............. 1000\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_rounding ............ 0\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_start_bits .......... 16\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_target_bits ......... 8\n",
      "[2021-06-14 15:34:15,980] [INFO] [config.py:904:print]   quantize_training_enabled .... False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   quantize_type ................ 0\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   quantize_verbose ............. False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   scheduler_name ............... WarmupLR\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   scheduler_params ............. {'warmup_min_lr': 0, 'warmup_max_lr': 5e-06, 'warmup_num_steps': 10}\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   sparse_attention ............. None\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   sparse_gradients_enabled ..... False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   steps_per_print .............. 2000\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   tensorboard_enabled .......... False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   tensorboard_job_name ......... DeepSpeedJobName\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   tensorboard_output_path ...... \n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   train_batch_size ............. 8\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   train_micro_batch_size_per_gpu  4\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   use_quantizer_kernel ......... False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   wall_clock_breakdown ......... False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   world_size ................... 1\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   zero_allow_untested_optimizer  False\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   zero_config .................. {\n",
      "    \"stage\": 2, \n",
      "    \"contiguous_gradients\": true, \n",
      "    \"reduce_scatter\": true, \n",
      "    \"reduce_bucket_size\": 2.000000e+08, \n",
      "    \"allgather_partitions\": true, \n",
      "    \"allgather_bucket_size\": 2.000000e+08, \n",
      "    \"overlap_comm\": true, \n",
      "    \"load_from_fp32_weights\": true, \n",
      "    \"elastic_checkpoint\": true, \n",
      "    \"offload_param\": null, \n",
      "    \"offload_optimizer\": {\n",
      "        \"device\": \"cpu\", \n",
      "        \"nvme_path\": null, \n",
      "        \"buffer_count\": 4, \n",
      "        \"pin_memory\": false, \n",
      "        \"pipeline_read\": false, \n",
      "        \"pipeline_write\": false, \n",
      "        \"fast_init\": false\n",
      "    }, \n",
      "    \"sub_group_size\": 1.000000e+12, \n",
      "    \"prefetch_bucket_size\": 5.000000e+07, \n",
      "    \"param_persistence_threshold\": 1.000000e+05, \n",
      "    \"max_live_parameters\": 1.000000e+09, \n",
      "    \"max_reuse_distance\": 1.000000e+09, \n",
      "    \"gather_fp16_weights_on_model_save\": false, \n",
      "    \"ignore_unused_parameters\": true, \n",
      "    \"legacy_stage1\": false\n",
      "}\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   zero_enabled ................. True\n",
      "[2021-06-14 15:34:15,981] [INFO] [config.py:904:print]   zero_optimization_stage ...... 2\n",
      "[2021-06-14 15:34:15,982] [INFO] [config.py:911:print]   json = {\n",
      "    \"fp16\": {\n",
      "        \"enabled\": true, \n",
      "        \"loss_scale\": 0, \n",
      "        \"loss_scale_window\": 1000, \n",
      "        \"initial_scale_power\": 16, \n",
      "        \"hysteresis\": 2, \n",
      "        \"min_loss_scale\": 1\n",
      "    }, \n",
      "    \"optimizer\": {\n",
      "        \"type\": \"AdamW\", \n",
      "        \"params\": {\n",
      "            \"lr\": 5e-06, \n",
      "            \"betas\": [0.9, 0.999], \n",
      "            \"eps\": 1e-08, \n",
      "            \"weight_decay\": 0.0\n",
      "        }\n",
      "    }, \n",
      "    \"scheduler\": {\n",
      "        \"type\": \"WarmupLR\", \n",
      "        \"params\": {\n",
      "            \"warmup_min_lr\": 0, \n",
      "            \"warmup_max_lr\": 5e-06, \n",
      "            \"warmup_num_steps\": 10\n",
      "        }\n",
      "    }, \n",
      "    \"zero_optimization\": {\n",
      "        \"stage\": 2, \n",
      "        \"allgather_partitions\": true, \n",
      "        \"allgather_bucket_size\": 2.000000e+08, \n",
      "        \"overlap_comm\": true, \n",
      "        \"reduce_scatter\": true, \n",
      "        \"reduce_bucket_size\": 2.000000e+08, \n",
      "        \"contiguous_gradients\": true, \n",
      "        \"cpu_offload\": true\n",
      "    }, \n",
      "    \"gradient_accumulation_steps\": 2, \n",
      "    \"gradient_clipping\": 1.0, \n",
      "    \"steps_per_print\": 2.000000e+03, \n",
      "    \"train_batch_size\": 8, \n",
      "    \"train_micro_batch_size_per_gpu\": 4, \n",
      "    \"wall_clock_breakdown\": false\n",
      "}\n",
      "[INFO|trainer.py:1156] 2021-06-14 15:34:15,982 >> ***** Running training *****\n",
      "[INFO|trainer.py:1157] 2021-06-14 15:34:15,982 >>   Num examples = 1428\n",
      "[INFO|trainer.py:1158] 2021-06-14 15:34:15,982 >>   Num Epochs = 1\n",
      "[INFO|trainer.py:1159] 2021-06-14 15:34:15,982 >>   Instantaneous batch size per device = 4\n",
      "[INFO|trainer.py:1160] 2021-06-14 15:34:15,982 >>   Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "[INFO|trainer.py:1161] 2021-06-14 15:34:15,982 >>   Gradient Accumulation steps = 2\n",
      "[INFO|trainer.py:1162] 2021-06-14 15:34:15,982 >>   Total optimization steps = 178\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "  8%|███▌                                      | 15/178 [01:45<19:19,  7.12s/it][INFO|trainer.py:2115] 2021-06-14 15:36:01,866 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:36:01,866 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:36:01,866 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.630859375, 'eval_runtime': 0.3171, 'eval_samples_per_second': 12.613, 'epoch': 0.08}\n",
      "  8%|███▌                                      | 15/178 [01:46<19:19,  7.12s/it]\n",
      "100%|████████████████████████████████████████████| 1/1 [00:00<00:00, 657.72it/s]\u001b[A\n",
      " 17%|███████                                   | 30/178 [03:31<17:28,  7.09s/it][INFO|trainer.py:2115] 2021-06-14 15:37:47,791 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:37:47,791 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:37:47,791 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.623046875, 'eval_runtime': 0.3149, 'eval_samples_per_second': 12.704, 'epoch': 0.17}\n",
      " 17%|███████                                   | 30/178 [03:32<17:28,  7.09s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.63it/s]\u001b[A\n",
      " 25%|██████████▌                               | 45/178 [05:15<15:34,  7.03s/it][INFO|trainer.py:2115] 2021-06-14 15:39:31,928 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:39:31,928 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:39:31,928 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.62109375, 'eval_runtime': 0.321, 'eval_samples_per_second': 12.461, 'epoch': 0.25}\n",
      " 25%|██████████▌                               | 45/178 [05:16<15:34,  7.03s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.51it/s]\u001b[A\n",
      " 34%|██████████████▏                           | 60/178 [07:00<13:31,  6.87s/it][INFO|trainer.py:2115] 2021-06-14 15:41:16,753 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:41:16,753 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:41:16,753 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.619140625, 'eval_runtime': 0.3053, 'eval_samples_per_second': 13.1, 'epoch': 0.34}\n",
      " 34%|██████████████▏                           | 60/178 [07:01<13:31,  6.87s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.75it/s]\u001b[A\n",
      " 42%|█████████████████▋                        | 75/178 [08:47<12:11,  7.10s/it][INFO|trainer.py:2115] 2021-06-14 15:43:03,467 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:43:03,467 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:43:03,467 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.6171875, 'eval_runtime': 0.3152, 'eval_samples_per_second': 12.692, 'epoch': 0.42}\n",
      " 42%|█████████████████▋                        | 75/178 [08:47<12:11,  7.10s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.58it/s]\u001b[A\n",
      " 51%|█████████████████████▏                    | 90/178 [10:33<10:20,  7.05s/it][INFO|trainer.py:2115] 2021-06-14 15:44:49,357 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:44:49,357 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:44:49,357 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.615234375, 'eval_runtime': 0.3079, 'eval_samples_per_second': 12.993, 'epoch': 0.5}\n",
      " 51%|█████████████████████▏                    | 90/178 [10:33<10:20,  7.05s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.83it/s]\u001b[A\n",
      " 59%|████████████████████████▏                | 105/178 [12:18<08:31,  7.00s/it][INFO|trainer.py:2115] 2021-06-14 15:46:34,327 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:46:34,327 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:46:34,327 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.611328125, 'eval_runtime': 0.3031, 'eval_samples_per_second': 13.197, 'epoch': 0.59}\n",
      " 59%|████████████████████████▏                | 105/178 [12:18<08:31,  7.00s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.83it/s]\u001b[A\n",
      " 67%|███████████████████████████▋             | 120/178 [14:01<06:34,  6.80s/it][INFO|trainer.py:2115] 2021-06-14 15:48:17,699 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:48:17,700 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:48:17,700 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.609375, 'eval_runtime': 0.301, 'eval_samples_per_second': 13.29, 'epoch': 0.67}\n",
      " 67%|███████████████████████████▋             | 120/178 [14:02<06:34,  6.80s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.72it/s]\u001b[A\n",
      " 76%|███████████████████████████████          | 135/178 [15:45<04:57,  6.92s/it][INFO|trainer.py:2115] 2021-06-14 15:50:01,847 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:50:01,847 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:50:01,847 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.607421875, 'eval_runtime': 0.3093, 'eval_samples_per_second': 12.934, 'epoch': 0.76}\n",
      " 76%|███████████████████████████████          | 135/178 [15:46<04:57,  6.92s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.62it/s]\u001b[A\n",
      " 84%|██████████████████████████████████▌      | 150/178 [17:29<03:11,  6.85s/it][INFO|trainer.py:2115] 2021-06-14 15:51:45,258 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:51:45,258 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:51:45,258 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.607421875, 'eval_runtime': 0.3047, 'eval_samples_per_second': 13.126, 'epoch': 0.84}\n",
      " 84%|██████████████████████████████████▌      | 150/178 [17:29<03:11,  6.85s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.77it/s]\u001b[A\n",
      " 93%|██████████████████████████████████████   | 165/178 [19:12<01:29,  6.85s/it]\u001b[A[INFO|trainer.py:2115] 2021-06-14 15:53:28,671 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:53:28,671 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:53:28,671 >>   Batch size = 8\n",
      "\n",
      "                                                                                \u001b[A\n",
      "\u001b[A{'eval_loss': 3.60546875, 'eval_runtime': 0.3025, 'eval_samples_per_second': 13.225, 'epoch': 0.92}\n",
      " 93%|██████████████████████████████████████   | 165/178 [19:12<01:29,  6.85s/it]\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.75it/s]\u001b[A\n",
      "100%|█████████████████████████████████████████| 178/178 [20:41<00:00,  6.85s/it][INFO|trainer.py:1352] 2021-06-14 15:54:57,926 >> \n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n",
      "{'train_runtime': 1241.9442, 'train_samples_per_second': 0.143, 'epoch': 1.0}   \n",
      "100%|█████████████████████████████████████████| 178/178 [20:41<00:00,  6.98s/it]\n",
      "[INFO|trainer.py:1885] 2021-06-14 15:54:58,061 >> Saving model checkpoint to finetuned\n",
      "[INFO|configuration_utils.py:351] 2021-06-14 15:54:58,062 >> Configuration saved in finetuned/config.json\n",
      "[INFO|modeling_utils.py:889] 2021-06-14 15:55:01,921 >> Model weights saved in finetuned/pytorch_model.bin\n",
      "[INFO|tokenization_utils_base.py:1924] 2021-06-14 15:55:01,922 >> tokenizer config file saved in finetuned/tokenizer_config.json\n",
      "[INFO|tokenization_utils_base.py:1930] 2021-06-14 15:55:01,922 >> Special tokens file saved in finetuned/special_tokens_map.json\n",
      "[INFO|trainer_pt_utils.py:907] 2021-06-14 15:55:02,011 >> ***** train metrics *****\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   epoch                      =        1.0\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   init_mem_cpu_alloc_delta   =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   init_mem_cpu_peaked_delta  =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   init_mem_gpu_alloc_delta   =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   init_mem_gpu_peaked_delta  =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_mem_cpu_alloc_delta  =    23539MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_mem_cpu_peaked_delta =     1854MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_mem_gpu_alloc_delta  =     2754MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_mem_gpu_peaked_delta =     4318MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_runtime              = 0:20:41.94\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_samples              =       1428\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,011 >>   train_samples_per_second   =      0.143\n",
      "INFO:__main__:*** Evaluate ***\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[INFO|trainer.py:2115] 2021-06-14 15:55:02,058 >> ***** Running Evaluation *****\n",
      "[INFO|trainer.py:2117] 2021-06-14 15:55:02,058 >>   Num examples = 4\n",
      "[INFO|trainer.py:2120] 2021-06-14 15:55:02,059 >>   Batch size = 8\n",
      "100%|█████████████████████████████████████████████| 1/1 [00:00<00:00,  3.66it/s]\n",
      "[INFO|trainer_pt_utils.py:907] 2021-06-14 15:55:02,432 >> ***** eval metrics *****\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   epoch                     =        1.0\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_loss                 =     3.6035\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_mem_cpu_alloc_delta  =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_mem_cpu_peaked_delta =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_mem_gpu_alloc_delta  =        0MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_mem_gpu_peaked_delta =     2371MB\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_runtime              = 0:00:00.31\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_samples              =          4\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   eval_samples_per_second   =     12.629\n",
      "[INFO|trainer_pt_utils.py:912] 2021-06-14 15:55:02,432 >>   perplexity                =    36.7271\n"
     ]
    }
   ],
   "source": [
    "!deepspeed --num_gpus=1 run_clm.py \\\n",
    "--deepspeed ds_config_gptneo.json \\\n",
    "--model_name_or_path EleutherAI/gpt-neo-1.3B \\\n",
    "--train_file train.csv \\\n",
    "--validation_file validation.csv \\\n",
    "--do_train \\\n",
    "--do_eval \\\n",
    "--fp16 \\\n",
    "--overwrite_cache \\\n",
    "--evaluation_strategy=\"steps\" \\\n",
    "--output_dir finetuned \\\n",
    "--num_train_epochs 1 \\\n",
    "--eval_steps 15 \\\n",
    "--gradient_accumulation_steps 2 \\\n",
    "--per_device_train_batch_size 4 \\\n",
    "--use_fast_tokenizer False \\\n",
    "--learning_rate 5e-06 \\\n",
    "--warmup_steps 10"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}