6.1 文本数据处理.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1 单词字符的 one-hot 编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  1.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  1.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  1.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  1.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  1.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]\n",
      "\n",
      " [[ 0.  1.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  1.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  1.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  1.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  1.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]\n",
      "  [ 0.  0.  0.  0.  0.  0.  0.  0.  0.  0.  0.]]]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "samples = ['The cat sat on the mat.', 'The dog ate my homework.'] # 两个样本\n",
    "token_index = {} # 字典，构建数据中所有标记的索引\n",
    "\n",
    "for sample in samples:\n",
    "    for word in sample.split():\n",
    "        if word not in token_index:\n",
    "            token_index[word] = len(token_index) + 1\n",
    "# token_index = {'The': 1, 'cat': 2, 'sat': 3, 'on': 4, 'the': 5, 'mat.': 6, 'dog': 7, 'ate': 8, 'my': 9, 'homework.': 10}\n",
    "\n",
    "max_length = 10 # 每个样本只考虑前10个单词\n",
    "#(样本数，每个样本的最大长度，索引字典的长度+1)\n",
    "results = np.zeros(shape=(len(samples),\n",
    "                          max_length,\n",
    "                          max(token_index.values()) + 1)) # 注意：编号为0，并没有制定单词，所以要+1\n",
    "for i, sample in enumerate(samples):\n",
    "    for j, word in list(enumerate(sample.split()))[:max_length]:\n",
    "        index = token_index.get(word)\n",
    "        results[i, j, index] = 1\n",
    "\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2 字符级的 one-hot 编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[ 1.  1.  1. ...,  1.  1.  1.]\n",
      "  [ 1.  1.  1. ...,  1.  1.  1.]\n",
      "  [ 1.  1.  1. ...,  1.  1.  1.]\n",
      "  ..., \n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
      "\n",
      " [[ 1.  1.  1. ...,  1.  1.  1.]\n",
      "  [ 1.  1.  1. ...,  1.  1.  1.]\n",
      "  [ 1.  1.  1. ...,  1.  1.  1.]\n",
      "  ..., \n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]]]\n"
     ]
    }
   ],
   "source": [
    "import string\n",
    "\n",
    "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n",
    "characters = string.printable # 所有的ASCII码字符\n",
    "token_index = dict(zip(range(1, len(characters) + 1), characters))\n",
    "max_length = 50\n",
    "results = np.zeros((len(samples), max_length, max(token_index.keys()) + 1))\n",
    "for i, sample in enumerate(samples):\n",
    "    for j, character in enumerate(sample):\n",
    "        index = token_index.get(character)\n",
    "        results[i, j, index] = 1.\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3 用 Keras 实现单词级的 one-hot 编码"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Found 9 unique tokens.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.text import Tokenizer\n",
    "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n",
    "\n",
    "tokenizer = Tokenizer(num_words=1000)# 创建分词器，只考虑前1000个常用的单词\n",
    "tokenizer.fit_on_texts(samples) # 构建单词索引\n",
    "\n",
    "sequences = tokenizer.texts_to_sequences(samples) # 将字符串转换为整数索引组成的列表\n",
    "\n",
    "one_hot_results = tokenizer.texts_to_matrix(samples, mode='binary') # 得到one-hot二进制编码\n",
    "\n",
    "word_index = tokenizer.word_index\n",
    "print('Found %s unique tokens.' % len(word_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'ate': 7,\n",
       " 'cat': 2,\n",
       " 'dog': 6,\n",
       " 'homework': 9,\n",
       " 'mat': 5,\n",
       " 'my': 8,\n",
       " 'on': 4,\n",
       " 'sat': 3,\n",
       " 'the': 1}"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###  4 使用散列技巧的单词级的 one-hot 编码（简单示例）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[[ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  ..., \n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]]\n",
      "\n",
      " [[ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  ..., \n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]\n",
      "  [ 0.  0.  0. ...,  0.  0.  0.]]]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "samples = ['The cat sat on the mat.', 'The dog ate my homework.']\n",
    "dimensionality = 1000\n",
    "max_length = 10\n",
    "results = np.zeros((len(samples), max_length, dimensionality))\n",
    "for i, sample in enumerate(samples):\n",
    "    for j, word in list(enumerate(sample.split()))[:max_length]:\n",
    "        index = abs(hash(word)) % dimensionality\n",
    "        results[i, j, index] = 1.\n",
    "print(results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}