doc/BPE-Notebook.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "SubwordNMT",
      "version": "0.3.2",
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "metadata": {
        "id": "KyV4tue4Tr12",
        "colab_type": "code",
        "outputId": "991b70cf-60cb-497e-f0b2-64bf1415c40a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install subword-nmt"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting subword-nmt\n",
            "  Downloading https://files.pythonhosted.org/packages/26/08/58267cb3ac00f5f895457777ed9e0d106dbb5e6388fa7923d8663b04b849/subword_nmt-0.3.6-py2.py3-none-any.whl\n",
            "Installing collected packages: subword-nmt\n",
            "Successfully installed subword-nmt-0.3.6\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "waI8aoAZUGcX",
        "colab_type": "code",
        "outputId": "c945a8c3-f8b6-4597-f677-0bdccb9fff7c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 161
        }
      },
      "cell_type": "code",
      "source": [
        "!pip install https://github.com/rsennrich/subword-nmt/archive/master.zip"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting https://github.com/rsennrich/subword-nmt/archive/master.zip\n",
            "  Downloading https://github.com/rsennrich/subword-nmt/archive/master.zip\n",
            "\u001b[K     \\ 430kB 12.0MB/s\n",
            "Requirement already satisfied (use --upgrade to upgrade): subword-nmt==0.3.6 from https://github.com/rsennrich/subword-nmt/archive/master.zip in /usr/local/lib/python3.6/dist-packages\n",
            "Building wheels for collected packages: subword-nmt\n",
            "  Running setup.py bdist_wheel for subword-nmt ... \u001b[?25l-\b \bdone\n",
            "\u001b[?25h  Stored in directory: /tmp/pip-ephem-wheel-cache-fkhcbj17/wheels/f8/96/2f/67d2f7991a593e6f63468df1a71368ee07287c4006858a16e0\n",
            "Successfully built subword-nmt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "LfmnYw8JVue8",
        "colab_type": "code",
        "outputId": "3cfe30b7-71ff-4cd5-bc74-3ea5463e7903",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 143
        }
      },
      "cell_type": "code",
      "source": [
        "!git clone https://github.com/pytorch/fairseq.git"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'fairseq'...\n",
            "remote: Enumerating objects: 25, done.\u001b[K\n",
            "remote: Counting objects: 100% (25/25), done.\u001b[K\n",
            "remote: Compressing objects: 100% (18/18), done.\u001b[K\n",
            "remote: Total 3094 (delta 12), reused 7 (delta 7), pack-reused 3069\u001b[K\n",
            "Receiving objects: 100% (3094/3094), 3.13 MiB | 13.70 MiB/s, done.\n",
            "Resolving deltas: 100% (2250/2250), done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RSP033dLV3fj",
        "colab_type": "code",
        "outputId": "0dd03b75-6b72-49df-823b-c0d31a7c7367",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        }
      },
      "cell_type": "code",
      "source": [
        "!ls\n",
        "%cp fairseq/examples/translation/prepare-iwslt14.sh prepare-iwslt14.sh\n",
        "!rm -r fairseq"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "fairseq  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "RQnZ-Go8WVoT",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!chmod 777 prepare-iwslt14.sh"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "w8kKeLEqWkpz",
        "colab_type": "code",
        "outputId": "b697411a-9855-4aa4-f438-b4cdc6a2b4ce",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 2575
        }
      },
      "cell_type": "code",
      "source": [
        "!./prepare-iwslt14.sh"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning Moses github repository (for tokenization scripts)...\n",
            "Cloning into 'mosesdecoder'...\n",
            "remote: Enumerating objects: 56, done.\u001b[K\n",
            "remote: Counting objects: 100% (56/56), done.\u001b[K\n",
            "remote: Compressing objects: 100% (43/43), done.\u001b[K\n",
            "remote: Total 147254 (delta 28), reused 25 (delta 13), pack-reused 147198\u001b[K\n",
            "Receiving objects: 100% (147254/147254), 129.71 MiB | 20.21 MiB/s, done.\n",
            "Resolving deltas: 100% (113792/113792), done.\n",
            "Cloning Subword NMT repository (for BPE pre-processing)...\n",
            "Cloning into 'subword-nmt'...\n",
            "remote: Enumerating objects: 4, done.\u001b[K\n",
            "remote: Counting objects: 100% (4/4), done.\u001b[K\n",
            "remote: Compressing objects: 100% (4/4), done.\u001b[K\n",
            "remote: Total 485 (delta 0), reused 1 (delta 0), pack-reused 481\u001b[K\n",
            "Receiving objects: 100% (485/485), 205.64 KiB | 12.10 MiB/s, done.\n",
            "Resolving deltas: 100% (287/287), done.\n",
            "Downloading data from https://wit3.fbk.eu/archive/2014-01/texts/de/en/de-en.tgz...\n",
            "--2018-12-15 00:11:13--  https://wit3.fbk.eu/archive/2014-01/texts/de/en/de-en.tgz\n",
            "Resolving wit3.fbk.eu (wit3.fbk.eu)... 217.77.80.8\n",
            "Connecting to wit3.fbk.eu (wit3.fbk.eu)|217.77.80.8|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 19982877 (19M) [application/x-gzip]\n",
            "Saving to: ‘de-en.tgz’\n",
            "\n",
            "de-en.tgz           100%[===================>]  19.06M  4.70MB/s    in 8.1s    \n",
            "\n",
            "2018-12-15 00:11:22 (2.35 MB/s) - ‘de-en.tgz’ saved [19982877/19982877]\n",
            "\n",
            "Data successfully downloaded.\n",
            "de-en/\n",
            "de-en/IWSLT14.TED.dev2010.de-en.de.xml\n",
            "de-en/IWSLT14.TED.dev2010.de-en.en.xml\n",
            "de-en/IWSLT14.TED.tst2010.de-en.de.xml\n",
            "de-en/IWSLT14.TED.tst2010.de-en.en.xml\n",
            "de-en/IWSLT14.TED.tst2011.de-en.de.xml\n",
            "de-en/IWSLT14.TED.tst2011.de-en.en.xml\n",
            "de-en/IWSLT14.TED.tst2012.de-en.de.xml\n",
            "de-en/IWSLT14.TED.tst2012.de-en.en.xml\n",
            "de-en/IWSLT14.TEDX.dev2012.de-en.de.xml\n",
            "de-en/IWSLT14.TEDX.dev2012.de-en.en.xml\n",
            "de-en/README\n",
            "de-en/train.en\n",
            "de-en/train.tags.de-en.de\n",
            "de-en/train.tags.de-en.en\n",
            "pre-processing train data...\n",
            "Tokenizer Version 1.1\n",
            "Language: de\n",
            "Number of threads: 8\n",
            "\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 8\n",
            "\n",
            "clean-corpus.perl: processing iwslt14.tokenized.de-en/tmp/train.tags.de-en.tok.de & .en to iwslt14.tokenized.de-en/tmp/train.tags.de-en.clean, cutoff 1-175, ratio 1.5\n",
            "..........(100000).......\n",
            "Input sentences: 174443  Output sentences:  167522\n",
            "pre-processing valid/test data...\n",
            "orig/de-en/IWSLT14.TED.dev2010.de-en.de.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.dev2010.de-en.de\n",
            "Tokenizer Version 1.1\n",
            "Language: de\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.tst2010.de-en.de.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.tst2010.de-en.de\n",
            "Tokenizer Version 1.1\n",
            "Language: de\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.tst2011.de-en.de.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.tst2011.de-en.de\n",
            "Tokenizer Version 1.1\n",
            "Language: de\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.tst2012.de-en.de.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.tst2012.de-en.de\n",
            "Tokenizer Version 1.1\n",
            "Language: de\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TEDX.dev2012.de-en.de.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TEDX.dev2012.de-en.de\n",
            "Tokenizer Version 1.1\n",
            "Language: de\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.dev2010.de-en.en.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.dev2010.de-en.en\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.tst2010.de-en.en.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.tst2010.de-en.en\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.tst2011.de-en.en.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.tst2011.de-en.en\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TED.tst2012.de-en.en.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TED.tst2012.de-en.en\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 8\n",
            "\n",
            "orig/de-en/IWSLT14.TEDX.dev2012.de-en.en.xml iwslt14.tokenized.de-en/tmp/IWSLT14.TEDX.dev2012.de-en.en\n",
            "Tokenizer Version 1.1\n",
            "Language: en\n",
            "Number of threads: 8\n",
            "\n",
            "creating train, valid, test...\n",
            "learn_bpe.py on iwslt14.tokenized.de-en/tmp/train.en-de...\n",
            "subword-nmt/learn_bpe.py:267: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "apply_bpe.py to train.de...\n",
            "subword-nmt/apply_bpe.py:334: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "subword-nmt/apply_bpe.py:351: ResourceWarning: unclosed file <_io.TextIOWrapper name='iwslt14.tokenized.de-en/code' mode='r' encoding='UTF-8'>\n",
            "  args.codes = codecs.open(args.codes.name, encoding='utf-8')\n",
            "apply_bpe.py to valid.de...\n",
            "subword-nmt/apply_bpe.py:334: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "subword-nmt/apply_bpe.py:351: ResourceWarning: unclosed file <_io.TextIOWrapper name='iwslt14.tokenized.de-en/code' mode='r' encoding='UTF-8'>\n",
            "  args.codes = codecs.open(args.codes.name, encoding='utf-8')\n",
            "apply_bpe.py to test.de...\n",
            "subword-nmt/apply_bpe.py:334: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "subword-nmt/apply_bpe.py:351: ResourceWarning: unclosed file <_io.TextIOWrapper name='iwslt14.tokenized.de-en/code' mode='r' encoding='UTF-8'>\n",
            "  args.codes = codecs.open(args.codes.name, encoding='utf-8')\n",
            "apply_bpe.py to train.en...\n",
            "subword-nmt/apply_bpe.py:334: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "subword-nmt/apply_bpe.py:351: ResourceWarning: unclosed file <_io.TextIOWrapper name='iwslt14.tokenized.de-en/code' mode='r' encoding='UTF-8'>\n",
            "  args.codes = codecs.open(args.codes.name, encoding='utf-8')\n",
            "apply_bpe.py to valid.en...\n",
            "subword-nmt/apply_bpe.py:334: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "subword-nmt/apply_bpe.py:351: ResourceWarning: unclosed file <_io.TextIOWrapper name='iwslt14.tokenized.de-en/code' mode='r' encoding='UTF-8'>\n",
            "  args.codes = codecs.open(args.codes.name, encoding='utf-8')\n",
            "apply_bpe.py to test.en...\n",
            "subword-nmt/apply_bpe.py:334: DeprecationWarning: this script's location has moved to /content/subword-nmt/subword_nmt. This symbolic link will be removed in a future version. Please point to the new location, or install the package and use the command 'subword-nmt'\n",
            "  DeprecationWarning\n",
            "subword-nmt/apply_bpe.py:351: ResourceWarning: unclosed file <_io.TextIOWrapper name='iwslt14.tokenized.de-en/code' mode='r' encoding='UTF-8'>\n",
            "  args.codes = codecs.open(args.codes.name, encoding='utf-8')\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MZbBcmSJZsD1",
        "colab_type": "code",
        "outputId": "7b71a005-6dea-4a0e-8f04-d30f8792bedb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "cell_type": "code",
      "source": [
        "%cd content\n",
        "!ls"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[Errno 2] No such file or directory: 'content'\n",
            "/content\n",
            "iwslt14.tokenized.de-en  orig\t\t     sample_data\n",
            "mosesdecoder\t\t prepare-iwslt14.sh  subword-nmt\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Ae0dFuV6LD1B",
        "colab_type": "code",
        "outputId": "81ae5e0f-a675-45c7-f96d-885d68737796",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 127
        }
      },
      "cell_type": "code",
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&scope=email%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdocs.test%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdrive.photos.readonly%20https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fpeopleapi.readonly&response_type=code\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "1zIPDplJN8Pj",
        "colab_type": "code",
        "outputId": "95b96f19-7e10-483c-dd31-3a05579f2c0b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "cell_type": "code",
      "source": [
        "!ls\n",
        "%cd drive"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "drive\t\t\t mosesdecoder  prepare-iwslt14.sh  subword-nmt\n",
            "iwslt14.tokenized.de-en  orig\t       sample_data\n",
            "/content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "MWOgNEuQOExi",
        "colab_type": "code",
        "outputId": "d75ed39a-11b8-4885-85eb-1682b8e73cbe",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 251
        }
      },
      "cell_type": "code",
      "source": [
        "%cd 'My Drive'\n",
        "!ls"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive\n",
            " app\n",
            "'ASP.NE 4.0从入门到精通（张正礼）.pdf'\n",
            " ASP.NET开发实例大全（基础卷）.pdf\n",
            "'《ASP.NET开发实战范例宝典(使用C#)》.(章立民).[PDF].&ckook.pdf'\n",
            "'Colab Notebooks'\n",
            " lang8-test.en\n",
            " lang8-test.gec\n",
            " lang8-train.en\n",
            " lang8-train.gec\n",
            " lang8-valid.en\n",
            " lang8-valid.gec\n",
            " 无标题文档.gdoc\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "Yo5ABEu8OUcL",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "%cp lang8-test.en /content/\n",
        "%cp lang8-test.gec /content/\n",
        "%cp lang8-train.en /content/\n",
        "%cp lang8-train.gec /content/\n",
        "%cp lang8-valid.en /content/\n",
        "%cp lang8-valid.gec /content/"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "tcbDdkVMPVth",
        "colab_type": "code",
        "outputId": "8ca60b28-4897-4f06-9c79-4ac99136c88b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        }
      },
      "cell_type": "code",
      "source": [
        "%cd ..\n",
        "%cd .."
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive\n",
            "/content\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "HfNfgUE4PbO5",
        "colab_type": "code",
        "outputId": "2c30d070-8283-4777-96db-4c82815381ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "drive\t\t\t lang8-train.en   mosesdecoder\t      subword-nmt\n",
            "iwslt14.tokenized.de-en  lang8-train.gec  orig\n",
            "lang8-test.en\t\t lang8-valid.en   prepare-iwslt14.sh\n",
            "lang8-test.gec\t\t lang8-valid.gec  sample_data\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "nvtemd6vPjmQ",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!subword-nmt learn-bpe -s 10000 < lang8-train.en > cs410output.en"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2bpngkNVRT1m",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!subword-nmt learn-bpe -s 10000 < lang8-train.gec > cs410output.gec"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "-2SbiWjAUqr4",
        "colab_type": "code",
        "outputId": "5e0eeecb-8502-4d45-83f5-f911adb9ba98",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 89
        }
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 16,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cs410output.en\t\t lang8-test.en\t  lang8-valid.en   prepare-iwslt14.sh\n",
            "cs410output.gec\t\t lang8-test.gec   lang8-valid.gec  sample_data\n",
            "drive\t\t\t lang8-train.en   mosesdecoder\t   subword-nmt\n",
            "iwslt14.tokenized.de-en  lang8-train.gec  orig\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "FDY1nfM1SM1N",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!subword-nmt apply-bpe -c cs410output.en < lang8-test.en > new-lang8-test1.en\n",
        "!subword-nmt apply-bpe -c cs410output.en < lang8-valid.en > new-lang8-test.en\n",
        "!subword-nmt apply-bpe -c cs410output.gec < lang8-valid.gec > new-lang8-test.gec"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "2OKzkAOiU39Y",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!cat cs410output.en > head "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "aXZiXpSOPFc3",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "!cat lang8-train.en lang8-train.gec | subword-nmt learn-bpe -s 10000 -o cs410output.en\n",
        "!cat lang8-train.en lang8-train.gec | subword-nmt learn-bpe -s 10000 -o cs410output.gec\n",
        "!subword-nmt apply-bpe -c cs410output.en < lang8-train.en | subword-nmt get-vocab > vocab_file.en\n",
        "!subword-nmt apply-bpe -c cs410output.gec < lang8-train.gec | subword-nmt get-vocab > vocab_file.gec\n",
        "!subword-nmt apply-bpe -c cs410output.en < lang8-valid.en | subword-nmt get-vocab > vocab_filevalid.en\n",
        "!subword-nmt apply-bpe -c cs410output.gec < lang8-valid.gec | subword-nmt get-vocab > vocab_filevalid.gec\n",
        "!subword-nmt apply-bpe -c cs410output.en --vocabulary vocab_file.en --vocabulary-threshold 50 < lang8-train.en > lang8-trainBPE.en\n",
        "!subword-nmt apply-bpe -c cs410output.gec --vocabulary vocab_file.gec --vocabulary-threshold 50 < lang8-train.gec > lang8-trainBPE.gec\n",
        "!subword-nmt apply-bpe -c cs410output.en --vocabulary vocab_file.en --vocabulary-threshold 50 < lang8-valid.en > lang8-validBPE.en\n",
        "!subword-nmt apply-bpe -c cs410output.gec --vocabulary vocab_file.gec --vocabulary-threshold 50 < lang8-valid.gec > lang8-validBPE.gec"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "ZCHfjD58pCcC",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233
        },
        "outputId": "42b85d5f-b477-40fe-cd5a-60372c6b6ece"
      },
      "cell_type": "code",
      "source": [
        "!ls"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cs410output.en\t\t iwslt14.tokenized.de-en  new-lang8-test1.en\n",
            "cs410output.en.L1\t lang8-test.en\t\t  new-lang8-test.en\n",
            "cs410output.en.L2\t lang8-test.gec\t\t  new-lang8-test.gec\n",
            "cs410output.gec\t\t lang8-trainBPE.en\t  orig\n",
            "cs410outputVA.en\t lang8-trainBPE.gec\t  prepare-iwslt14.sh\n",
            "cs410outputVA.gec\t lang8-train.en\t\t  sample_data\n",
            "cs410outputvalid.en\t lang8-train.gec\t  subword-nmt\n",
            "cs410outputvalid.gec\t lang8-validBPE.en\t  vocab_file.en\n",
            "cs410output-vocab.en.L1  lang8-validBPE.gec\t  vocab_file.gec\n",
            "cs410output-vocab.en.L2  lang8-valid.en\t\t  vocab_filevalid.en\n",
            "drive\t\t\t lang8-valid.gec\t  vocab_filevalid.gec\n",
            "head\t\t\t mosesdecoder\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "metadata": {
        "id": "l6sjq5Cgp9gM",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "from google.colab import files\n",
        "files.download('lang8-trainBPE.en') "
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "metadata": {
        "id": "LvL7wnTHq5uf",
        "colab_type": "code",
        "colab": {}
      },
      "cell_type": "code",
      "source": [
        "files.download('lang8-trainBPE.gec') \n",
        "files.download('lang8-validBPE.en') \n",
        "files.download('lang8-validBPE.gec') \n"
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}