python-ucto.cast

{"version": 2, "width": 240, "height": 52, "timestamp": 1687956837, "idle_time_limit": 1.0, "env": {"SHELL": null, "TERM": "xterm"}, "title": "python-ucto.sh"}
[0.002164, "o", "\r\n"]
[0.002702, "o", "                      _   _                                  _        \r\n          _ __  _   _| |_| |__   ___  _ __        _   _  ___| |_ ___  \r\n         | '_ \\| | | | __| '_ \\ / _ \\| '_ \\ _____| | | |/ __| __/ _ \\ \r\n         | |_) | |_| | |_| | | | (_) | | | |_____| |_| | (__| || (_) |\r\n         | .__/ \\__, |\\__|_| |_|\\___/|_| |_|      \\__,_|\\___|\\__\\___/ \r\n         |_|    |___/                                                 \r\n"]
[0.002821, "o", "\r\n"]
[0.002854, "o", "  https://github.com/proycon/python-ucto\r\n\r\n"]
[0.015859, "o", "💬 \u001b[3;1m\u001b[38;5;231mUcto, the tokenizer, can be used directly from Python,\u001b[0m\u001b[0m\r\n"]
[0.016331, "o", "\r🕐  4s"]
[1.017188, "o", "\r🕐  3s"]
[2.017995, "o", "\r🕐  2s"]
[3.018704, "o", "\r🕐  1s"]
[4.019343, "o", "\r           \r"]
[4.032678, "o", "💬 \u001b[3;1m\u001b[38;5;231mvia its python binding \u001b[0m\u001b[38;5;203m`\u001b[0m\u001b[38;5;203mpython-ucto´\u001b[0m\u001b[0m\r\n"]
[4.033282, "o", "\r🕐  4s"]
[5.034161, "o", "\r🕐  3s"]
[6.034997, "o", "\r🕐  2s"]
[7.035774, "o", "\r🕐  1s"]
[8.036745, "o", "\r           \r"]
[8.051313, "o", "💬 \u001b[3;1m\u001b[38;5;231mThis exposes the functionality of Ucto via Python, rather than through a command-line interface.\u001b[0m\u001b[0m\r\n"]
[8.051792, "o", "\r🕐  4s"]
[9.052755, "o", "\r🕐  3s"]
[10.053638, "o", "\r🕐  2s"]
[11.054408, "o", "\r🕐  1s"]
[12.055337, "o", "\r           \r\r\n"]
[12.069369, "o", "💬 \u001b[3;1m\u001b[38;5;231mWe can install python-ucto as follows (you may want to ensure you are in a virtualenv first)\u001b[0m\u001b[0m\r\n"]
[12.069954, "o", "\r🕐  4s"]
[13.070699, "o", "\r🕐  3s"]
[14.071938, "o", "\r🕐  2s"]
[15.072901, "o", "\r🕐  1s"]
[16.073854, "o", "\r           \r\r\n"]
[16.073884, "o", "\u001b[33;1m$\u001b[0m "]
[17.08663, "o", "\u001b"]
[17.112745, "o", "["]
[17.138298, "o", "3"]
[17.163736, "o", "8"]
[17.189152, "o", ";"]
[17.214807, "o", "5"]
[17.2404, "o", ";"]
[17.26628, "o", "2"]
[17.292124, "o", "3"]
[17.317919, "o", "1"]
[17.343729, "o", "m"]
[17.369378, "o", "p"]
[17.395236, "o", "i"]
[17.421352, "o", "p"]
[17.44702, "o", "\u001b"]
[17.472469, "o", "["]
[17.498159, "o", "0"]
[17.523866, "o", "m"]
[17.549818, "o", "\u001b"]
[17.575561, "o", "["]
[17.601295, "o", "3"]
[17.627432, "o", "8"]
[17.653192, "o", ";"]
[17.679138, "o", "5"]
[17.704848, "o", ";"]
[17.730845, "o", "2"]
[17.756573, "o", "3"]
[17.782647, "o", "1"]
[17.808652, "o", "m"]
[17.834422, "o", " "]
[17.860021, "o", "i"]
[17.885824, "o", "n"]
[17.911593, "o", "s"]
[17.937339, "o", "t"]
[17.963147, "o", "a"]
[17.988863, "o", "l"]
[18.015215, "o", "l"]
[18.041002, "o", " "]
[18.066715, "o", "p"]
[18.092402, "o", "y"]
[18.118365, "o", "t"]
[18.144344, "o", "h"]
[18.170086, "o", "o"]
[18.195862, "o", "n"]
[18.221788, "o", "-"]
[18.247477, "o", "u"]
[18.273238, "o", "c"]
[18.298767, "o", "t"]
[18.32433, "o", "o"]
[18.349967, "o", "\u001b"]
[18.375536, "o", "["]
[18.401315, "o", "0"]
[18.427133, "o", "m"]
[19.453855, "o", "\u001b[0m\r\n"]
[20.068393, "o", "Collecting python-ucto\r\n"]
[20.124087, "o", "  Downloading python_ucto-0.6.5-cp311-cp311-musllinux_1_1_x86_64.whl (25.8 MB)\r\n"]
[20.124416, "o", "\u001b[?25l"]
[20.125222, "o", "     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/25.8 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m"]
[20.159827, "o", "\r\u001b[2K     \u001b[91m━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/25.8 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.194468, "o", "\r\u001b[2K     \u001b[91m━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/25.8 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.22914, "o", "\r\u001b[2K     \u001b[91m━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/25.8 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.263633, "o", "\r\u001b[2K     \u001b[91m━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.0/25.8 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.297971, "o", "\r\u001b[2K     \u001b[91m━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/25.8 MB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:04\u001b[0m"]
[20.332382, "o", "\r\u001b[2K     \u001b[91m━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.2/25.8 MB\u001b[0m \u001b[31m20.3 MB/s\u001b[0m eta \u001b[36m0:00:02\u001b[0m"]
[20.36682, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.3/25.8 MB\u001b[0m \u001b[31m31.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.401137, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.3/25.8 MB\u001b[0m \u001b[31m31.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.436578, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━\u001b[0m\u001b[90m╺\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.3/25.8 MB\u001b[0m \u001b[31m31.4 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.470948, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m8.3/25.8 MB\u001b[0m \u001b[31m24.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.505382, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m11.3/25.8 MB\u001b[0m \u001b[31m49.3 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.539799, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m14.8/25.8 MB\u001b[0m \u001b[31m52.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.574191, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━━━━\u001b[0m \u001b[32m17.8/25.8 MB\u001b[0m \u001b[31m99.2 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.608618, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━━━━━━\u001b[0m \u001b[32m20.0/25.8 MB\u001b[0m \u001b[31m87.6 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.643126, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m\u001b[90m━━━━\u001b[0m \u001b[32m23.2/25.8 MB\u001b[0m \u001b[31m82.9 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.677554, "o", "\r\u001b[2K     \u001b[91m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[91m╸\u001b[0m \u001b[32m25.8/25.8 MB\u001b[0m \u001b[31m86.0 MB/s\u001b[0m eta \u001b[36m0:00:01\u001b[0m"]
[20.685593, "o", "\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m25.8/25.8 MB\u001b[0m \u001b[31m73.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n\u001b[?25h"]
[20.81776, "o", "Collecting Cython (from python-ucto)\r\n"]
[20.843601, "o", "  Downloading Cython-0.29.35-cp311-cp311-musllinux_1_1_x86_64.whl (2.0 MB)\r\n"]
[20.843983, "o", "\u001b[?25l"]
[20.844348, "o", "     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/2.0 MB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m"]
[20.862701, "o", "\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m115.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\r\n\u001b[?25h"]
[20.893429, "o", "Installing collected packages: Cython, python-ucto\r\n"]
[21.516446, "o", "Successfully installed Cython-0.29.35 python-ucto-0.6.5\r\n"]
[21.516737, "o", "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\r\n\u001b[0m"]
[21.631402, "o", "\u001b[33;1m$\u001b[0m "]
[22.64499, "o", "\u001b"]
[22.671111, "o", "["]
[22.696894, "o", "3"]
[22.72282, "o", "8"]
[22.748591, "o", ";"]
[22.774082, "o", "5"]
[22.799939, "o", ";"]
[22.825904, "o", "2"]
[22.851684, "o", "3"]
[22.877736, "o", "1"]
[22.903519, "o", "m"]
[22.929446, "o", "p"]
[22.955225, "o", "y"]
[22.981263, "o", "t"]
[23.007118, "o", "h"]
[23.033396, "o", "o"]
[23.059234, "o", "n"]
[23.085122, "o", "\u001b"]
[23.111077, "o", "["]
[23.136855, "o", "0"]
[23.162768, "o", "m"]
[23.188576, "o", "\u001b"]
[23.214339, "o", "["]
[23.240242, "o", "3"]
[23.266096, "o", "8"]
[23.292207, "o", ";"]
[23.318153, "o", "5"]
[23.344225, "o", ";"]
[23.369985, "o", "2"]
[23.395976, "o", "0"]
[23.422036, "o", "8"]
[23.448215, "o", "m"]
[23.473995, "o", " "]
[23.500277, "o", "-"]
[23.526129, "o", "\u001b"]
[23.552082, "o", "["]
[23.577969, "o", "0"]
[23.603935, "o", "m"]
[23.63009, "o", "\u001b"]
[23.655817, "o", "["]
[23.681724, "o", "3"]
[23.707886, "o", "8"]
[23.733677, "o", ";"]
[23.759602, "o", "5"]
[23.785492, "o", ";"]
[23.811311, "o", "2"]
[23.837034, "o", "0"]
[23.862623, "o", "8"]
[23.888412, "o", "m"]
[23.914432, "o", "c"]
[23.940177, "o", "\u001b"]
[23.966466, "o", "["]
[23.992496, "o", "0"]
[24.018266, "o", "m"]
[24.044046, "o", "\u001b"]
[24.069984, "o", "["]
[24.09621, "o", "3"]
[24.122033, "o", "8"]
[24.147801, "o", ";"]
[24.173556, "o", "5"]
[24.199537, "o", ";"]
[24.225505, "o", "2"]
[24.251524, "o", "3"]
[24.277309, "o", "1"]
[24.303211, "o", "m"]
[24.329373, "o", " "]
[24.354978, "o", "\u001b"]
[24.381046, "o", "["]
[24.407317, "o", "0"]
[24.433092, "o", "m"]
[24.459219, "o", "\u001b"]
[24.484982, "o", "["]
[24.511, "o", "3"]
[24.53681, "o", "8"]
[24.562713, "o", ";"]
[24.588595, "o", "5"]
[24.614518, "o", ";"]
[24.640764, "o", "2"]
[24.666573, "o", "3"]
[24.692535, "o", "1"]
[24.718357, "o", "m"]
[24.744183, "o", "\""]
[24.770314, "o", "\u001b"]
[24.796298, "o", "["]
[24.822269, "o", "0"]
[24.848166, "o", "m"]
[24.873943, "o", "\u001b"]
[24.899869, "o", "["]
[24.926019, "o", "3"]
[24.952106, "o", "8"]
[24.978381, "o", ";"]
[25.00417, "o", "5"]
[25.030415, "o", ";"]
[25.056228, "o", "1"]
[25.082141, "o", "8"]
[25.108134, "o", "6"]
[25.133851, "o", "m"]
[25.159675, "o", "i"]
[25.185562, "o", "m"]
[25.211399, "o", "p"]
[25.237285, "o", "o"]
[25.263255, "o", "r"]
[25.289174, "o", "t"]
[25.315098, "o", " "]
[25.340857, "o", "u"]
[25.367033, "o", "c"]
[25.393087, "o", "t"]
[25.418889, "o", "o"]
[25.445026, "o", ";"]
[25.47084, "o", " "]
[25.496918, "o", "u"]
[25.522925, "o", "c"]
[25.548929, "o", "t"]
[25.574898, "o", "o"]
[25.600543, "o", "."]
[25.626369, "o", "i"]
[25.65231, "o", "n"]
[25.678179, "o", "s"]
[25.70402, "o", "t"]
[25.730018, "o", "a"]
[25.756014, "o", "l"]
[25.781952, "o", "l"]
[25.807873, "o", "d"]
[25.833456, "o", "a"]
[25.859625, "o", "t"]
[25.885523, "o", "a"]
[25.911518, "o", "("]
[25.937287, "o", ")"]
[25.963357, "o", "\u001b"]
[25.98893, "o", "["]
[26.014589, "o", "0"]
[26.040304, "o", "m"]
[26.066202, "o", "\u001b"]
[26.092017, "o", "["]
[26.118007, "o", "3"]
[26.143784, "o", "8"]
[26.16978, "o", ";"]
[26.196026, "o", "5"]
[26.22191, "o", ";"]
[26.24779, "o", "2"]
[26.273626, "o", "3"]
[26.299437, "o", "1"]
[26.325255, "o", "m"]
[26.3513, "o", "\""]
[26.377047, "o", "\u001b"]
[26.402876, "o", "["]
[26.429089, "o", "0"]
[26.454884, "o", "m"]
[27.481928, "o", "\u001b[0m\r\n"]
[27.512165, "o", "Connecting to github.com (140.82.121.4:443)\r\n"]
[27.765361, "o", "Connecting to objects.githubusercontent.com (185.199.109.133:443)\r\n"]
[28.002374, "o", "saving to 'uctodata.tar.gz'\r\n"]
[28.014118, "o", "\ructodata.tar.gz      100% |************************************************************************************************************************************************************************************************| 99786  0:00:00 ETA\r\n'uctodata.tar.gz' saved\r\n"]
[28.018985, "o", "Installation of uctodata 0.9.1 complete\r\n"]
[28.032557, "o", "Connecting to raw.githubusercontent.com (185.199.110.133:443)\r\n"]
[28.232471, "o", "saving to 'textcat.cfg'\r\n\rtextcat.cfg          100% |************************************************************************************************************************************************************************************************|   738  0:00:00 ETA\r\n'textcat.cfg' saved\r\n"]
[28.235468, "o", "\r\n"]
[28.248824, "o", "💬 \u001b[3;1m\u001b[38;5;231mHere is a simple python script with ucto:\u001b[0m\u001b[0m\r\n"]
[28.249468, "o", "\r🕐  4s"]
[29.25036, "o", "\r🕐  3s"]
[30.251122, "o", "\r🕐  2s"]
[31.25247, "o", "\r🕐  1s"]
[32.253242, "o", "\r           \r\r\n"]
[32.25327, "o", "\u001b[33;1m$\u001b[0m "]
[33.265988, "o", "\u001b"]
[33.29187, "o", "["]
[33.317752, "o", "3"]
[33.343477, "o", "8"]
[33.369154, "o", ";"]
[33.394924, "o", "5"]
[33.420693, "o", ";"]
[33.446522, "o", "2"]
[33.47244, "o", "3"]
[33.498246, "o", "1"]
[33.524087, "o", "m"]
[33.549985, "o", "b"]
[33.575848, "o", "a"]
[33.601535, "o", "t"]
[33.627196, "o", "\u001b"]
[33.652909, "o", "["]
[33.679089, "o", "0"]
[33.704867, "o", "m"]
[33.731152, "o", "\u001b"]
[33.757248, "o", "["]
[33.783025, "o", "3"]
[33.80874, "o", "8"]
[33.8342, "o", ";"]
[33.860297, "o", "5"]
[33.88611, "o", ";"]
[33.911987, "o", "2"]
[33.937848, "o", "3"]
[33.963748, "o", "1"]
[33.989613, "o", "m"]
[34.015423, "o", " "]
[34.041177, "o", "u"]
[34.067059, "o", "c"]
[34.092809, "o", "t"]
[34.118581, "o", "o"]
[34.14468, "o", "e"]
[34.170512, "o", "x"]
[34.196817, "o", "a"]
[34.222618, "o", "m"]
[34.248399, "o", "p"]
[34.273895, "o", "l"]
[34.299626, "o", "e"]
[34.325473, "o", "."]
[34.351542, "o", "p"]
[34.37732, "o", "y"]
[34.403225, "o", "\u001b"]
[34.429046, "o", "["]
[34.45482, "o", "0"]
[34.480902, "o", "m"]
[35.507531, "o", "\u001b[0m\r\n"]
[35.526175, "o", "\u001b[38;5;238m───────┬────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\r\n       \u001b[38;5;238m│ \u001b[0mFile: \u001b[1muctoexample.py\u001b[0m\r\n\u001b[38;5;238m───────┼─────────────────────────────────────────────────────────────────"]
[35.526192, "o", "───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\r\n"]
[35.528013, "o", "\u001b[38;5;238m   1\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;242m#\u001b[0m\u001b[38;5;242m!/usr/bin/env python3\u001b[0m\r\n\u001b[38;5;238m   2\u001b[0m   \u001b[38;5;238m│\u001b[0m \r\n"]
[35.528801, "o", "\u001b[38;5;238m   3\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;203mimport\u001b[0m\u001b[38;5;231m \u001b[0m\u001b[38;5;231mucto\u001b[0m\r\n\u001b[38;5;238m   4\u001b[0m   \u001b[38;5;238m│\u001b[0m \r\n"]
[35.528855, "o", "\u001b[38;5;238m   5\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;242m#\u001b[0m\u001b[38;5;242m Instantiate the tokenizer \u001b[0m\r\n"]
[35.529615, "o", "\u001b[38;5;238m   6\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;231mtokenizer\u001b[0m\u001b[38;5;231m \u001b[0m\u001b[38;5;203m=\u001b[0m\u001b[38;5;231m \u001b[0m\u001b[38;5;231mucto\u001b[0m\u001b[38;5;231m.\u001b[0m\u001b[38;5;231mTokenizer\u001b[0m\u001b[38;5;231m(\u001b[0m\u001b[38;5;186m\"\u001b[0m\u001b[38;5;186mtokconfig-eng\u001b[0m"]
[35.529644, "o", "\u001b[38;5;186m\"\u001b[0m\u001b[38;5;231m)\u001b[0m"]
[35.529673, "o", "\r\n\u001b[38;5;238m   7\u001b[0m   \u001b[38;5;238m│\u001b[0m \r\n"]
[35.529698, "o", "\u001b[38;5;238m   8\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;242m#\u001b[0m\u001b[38;5;242mpass the text\u001b[0m\r\n"]
[35.529884, "o", "\u001b[38;5;238m   9\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;231mtokenizer\u001b[0m\u001b[38;5;231m.\u001b[0m\u001b[38;5;231mprocess\u001b[0m\u001b[38;5;231m(\u001b[0m\u001b[38;5;186m\"\u001b[0m"]
[35.529907, "o", "\u001b[38;5;186mTo be, or not to be, that's the question.\u001b[0m\u001b[38;5;186m\"\u001b[0m\u001b[38;5;231m)\u001b[0m"]
[35.529933, "o", "\r\n\u001b[38;5;238m  10\u001b[0m   \u001b[38;5;238m│\u001b[0m \r\n"]
[35.529955, "o", "\u001b[38;5;238m  11\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;242m#\u001b[0m\u001b[38;5;242mread the results from the tokenizer\u001b[0m\r\n"]
[35.53002, "o", "\u001b[38;5;238m  12\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;203mfor\u001b[0m\u001b[38;5;231m \u001b[0m\u001b[38;5;231mtoken\u001b[0m\u001b[38;5;231m \u001b[0m\u001b[38;5;203min\u001b[0m\u001b[38;5;231m \u001b[0m\u001b[38;5;231mtokenizer\u001b[0m"]
[35.530044, "o", "\u001b[38;5;231m:\u001b[0m\r\n"]
[35.530161, "o", "\u001b[38;5;238m  13\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;231m    \u001b[0m\u001b[38;5;81mprint\u001b[0m\u001b[38;5;231m(\u001b[0m\u001b[38;5;231mtoken\u001b[0m\u001b[38;5;231m)\u001b[0m\r\n"]
[35.530185, "o", "\u001b[38;5;238m  14\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;231m    \u001b[0m\r\n"]
[35.530214, "o", "\u001b[38;5;238m  15\u001b[0m   \u001b[38;5;238m│\u001b[0m \u001b[38;5;242m#\u001b[0m\u001b[38;5;242mthe API also exposes methods like token.isendofsentence(), token.nospace(), and attributes like token.type\u001b[0m\r\n\u001b[38;5;238m───────┴────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────\u001b[0m\r\n"]
[35.531801, "o", "\r🕐  10s"]
[36.532509, "o", "\r🕐  9s"]
[37.533365, "o", "\r🕐  8s"]
[38.534544, "o", "\r🕐  7s"]
[39.535427, "o", "\r🕐  6s"]
[40.536397, "o", "\r🕐  5s"]
[41.537201, "o", "\r🕐  4s"]
[42.538033, "o", "\r🕐  3s"]
[43.539068, "o", "\r🕐  2s"]
[44.539903, "o", "\r🕐  1s"]
[45.541089, "o", "\r           \r\r\n"]
[45.555135, "o", "💬 \u001b[3;1m\u001b[38;5;231mLet's run it and look at the output:\u001b[0m\u001b[0m\r\n"]
[45.555719, "o", "\r🕐  4s"]
[46.556323, "o", "\r🕐  3s"]
[47.557352, "o", "\r🕐  2s"]
[48.558439, "o", "\r🕐  1s"]
[49.559547, "o", "\r           \r\r\n"]
[49.559575, "o", "\u001b[33;1m$\u001b[0m "]
[50.572209, "o", "\u001b"]
[50.597944, "o", "["]
[50.623744, "o", "3"]
[50.649712, "o", "8"]
[50.675586, "o", ";"]
[50.701532, "o", "5"]
[50.727761, "o", ";"]
[50.753547, "o", "2"]
[50.779304, "o", "3"]
[50.805134, "o", "1"]
[50.830997, "o", "m"]
[50.85677, "o", "p"]
[50.882594, "o", "y"]
[50.908367, "o", "t"]
[50.934443, "o", "h"]
[50.960253, "o", "o"]
[50.985824, "o", "n"]
[51.011562, "o", "\u001b"]
[51.037294, "o", "["]
[51.063451, "o", "0"]
[51.089058, "o", "m"]
[51.114903, "o", "\u001b"]
[51.140674, "o", "["]
[51.166239, "o", "3"]
[51.192342, "o", "8"]
[51.218218, "o", ";"]
[51.243981, "o", "5"]
[51.27009, "o", ";"]
[51.29593, "o", "2"]
[51.321757, "o", "3"]
[51.347973, "o", "1"]
[51.373686, "o", "m"]
[51.399484, "o", " "]
[51.425371, "o", "u"]
[51.451358, "o", "c"]
[51.477456, "o", "t"]
[51.50314, "o", "o"]
[51.529041, "o", "e"]
[51.554872, "o", "x"]
[51.580758, "o", "a"]
[51.606058, "o", "m"]
[51.631963, "o", "p"]
[51.657723, "o", "l"]
[51.68347, "o", "e"]
[51.709609, "o", "."]
[51.735428, "o", "p"]
[51.760993, "o", "y"]
[51.78648, "o", "\u001b"]
[51.812249, "o", "["]
[51.838153, "o", "0"]
[51.86421, "o", "m"]
[52.890767, "o", "\u001b[0m\r\n"]
[52.920025, "o", "To\r\nbe\r\n,\r\nor\r\nnot\r\nto\r\nbe\r\n,\r\nthat\r\n's\r\nthe\r\nquestion\r\n.\r\n"]
[52.922956, "o", "\r🕐  10s"]
[53.923832, "o", "\r🕐  9s"]
[54.924801, "o", "\r🕐  8s"]
[55.925793, "o", "\r🕐  7s"]
[56.926901, "o", "\r🕐  6s"]
[57.927982, "o", "\r🕐  5s"]
[58.92909, "o", "\r🕐  4s"]
[59.930184, "o", "\r🕐  3s"]
[60.931387, "o", "\r🕐  2s"]
[61.932462, "o", "\r🕐  1s"]
[62.933523, "o", "\r           \r\r\n"]
[62.947017, "o", "💬 \u001b[3;1m\u001b[38;5;231mVisit \u001b[0m\u001b[38;5;231m<\u001b[0m\u001b[4;38;5;149mhttps://github.com/proycon/python-ucto\u001b[0m\u001b[38;5;231m>\u001b[0m\u001b[38;5;231m for more information.\u001b[0m\u001b[0m\r\n"]
[62.947649, "o", "\r🕐  4s"]
[63.948226, "o", "\r🕐  3s"]
[64.949077, "o", "\r🕐  2s"]
[65.949929, "o", "\r🕐  1s"]
[66.950823, "o", "\r           \r"]