"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "from IPython.display import HTML, display\n",
- "conf_matrix = [[tn, fp],\n",
- " [fn, tp]]\n",
- "display(HTML(''.format(\n",
- " ''.join('| {} | '.format(\n",
- " ''.join(str(_) for _ in row)) \n",
- " for row in conf_matrix))))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "count = tn + tp + fn + fp\n",
- "percent_matrix = [[\"{:.1%}\".format(tn/count), \"{:.1%}\".format(fp/count)],\n",
- " [\"{:.1%}\".format(fn/count), \"{:.1%}\".format(tp/count)]]\n",
- "display(HTML(''.format(\n",
- " ' |
'.join('| {} | '.format(\n",
- " ''.join(str(_) for _ in row)) \n",
- " for row in percent_matrix))))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Classification accuracy: 68.6%\n"
- ]
- }
- ],
- "source": [
- "print(\"Classification accuracy: {}\".format(\"{:.1%}\".format((tp+tn)/count)))"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
From be3aa08cf549a4f26dcc10b688ee6c532537eccb Mon Sep 17 00:00:00 2001
From: Nurgalym2001 <89890787+Nurgalym2001@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:04:48 +0600
Subject: [PATCH 02/11] Add files via upload
---
chapter1/spam-fighting-blacklist.ipynb | 1350 ++++++++++++++++++++++++
1 file changed, 1350 insertions(+)
create mode 100644 chapter1/spam-fighting-blacklist.ipynb
diff --git a/chapter1/spam-fighting-blacklist.ipynb b/chapter1/spam-fighting-blacklist.ipynb
new file mode 100644
index 0000000..7959fa3
--- /dev/null
+++ b/chapter1/spam-fighting-blacklist.ipynb
@@ -0,0 +1,1350 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## В этом коде используется черный список (Blacklist) для борьбы со спамом в электронной почте"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Импорт необходимых модулей\n",
+ "import os # Модуль для работы с операционной системой\n",
+ "import pickle # Модуль для сериализации и десериализации объектов Python\n",
+ "import email_read_util # Модуль для чтения и обработки электронных писем"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Скачайте общедоступный корпус спама TREC 2007 года выпуска\n",
+ "1. Прочитайте \"Соглашение об использовании\"\n",
+ " https://plg.uwaterloo.ca/~gvcormac/treccorpus07/\n",
+ "\n",
+ "2. Загрузите корпус объемом 255 МБ (trec07p.tgz) и распакуйте в каталог \"глава 1/наборы данных\"\n",
+ "\n",
+ "3. Убедитесь, что указанные ниже пути для \"DATA_DIR\" и \"LABELS_FILE\" существуют"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Загрузка данных:\n",
+ "### Данные загружаются из каталога datasets/trec07p/data/. Также загружаются метки классов (спам/не спам) из файла datasets/trec07p/full/index. Черный список содержит известные серверы, распространяющие спам, и блокирует письма с таких серверов."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Определение директории с данными и файла с метками\n",
+ "DATA_DIR = 'datasets/trec07p/data/' # Путь к директории с данными\n",
+ "LABELS_FILE = 'datasets/trec07p/full/index' # Путь к файлу с метками\n",
+ "TRAINING_SET_RATIO = 0.7 # Пропорция разделения набора данных на тренировочный и тестовый"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Инициализация пустых структур данных\n",
+ "labels = {} # Словарь для хранения меток (имя файла: метка)\n",
+ "spam_words = set() # Множество слов, характерных для спама\n",
+ "ham_words = set() # Множество слов, характерных для нормальной корреспонденции"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Чтение и обработка файла меток\n",
+ "with open(LABELS_FILE) as f:\n",
+ " for line in f:\n",
+ " line = line.strip()\n",
+ " label, key = line.split()\n",
+ " labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0# Чтение и обработка файла меток\n",
+ "with open(LABELS_FILE) as f:\n",
+ " for line in f:\n",
+ " line = line.strip()\n",
+ " label, key = line.split()\n",
+ " labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Разделение корпуса на тренировочный и тестовый наборы\n",
+ "filelist = os.listdir(DATA_DIR)\n",
+ "X_train = filelist[:int(len(filelist)*TRAINING_SET_RATIO)]\n",
+ "X_test = filelist[int(len(filelist)*TRAINING_SET_RATIO):]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Код загружает данные о транзакциях с кредитными картами и обучает модель на основе черного списка"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Построение/загрузка \"черного списка\":\n",
+ "### Черный список содержит слова, которые характерны для спам-сообщений и не характерны для нормальных писем.\n",
+ "### Если файл blacklist.pkl уже существует, черный список загружается из него. В противном случае он строится на основе данных обучающего набора и сохраняется для последующего использования."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Blacklist of 97939 tokens successfully built/loaded\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Построение или загрузка списка слов-маркеров для спама (blacklist)\n",
+ "if not os.path.exists('blacklist.pkl'):\n",
+ " for filename in X_train:\n",
+ " path = os.path.join(DATA_DIR, filename)\n",
+ " if filename in labels:\n",
+ " label = labels[filename]\n",
+ " stems = email_read_util.load(path)\n",
+ " if not stems:\n",
+ " continue\n",
+ " if label == 1:\n",
+ " ham_words.update(stems)\n",
+ " elif label == 0:\n",
+ " spam_words.update(stems)\n",
+ " else:\n",
+ " continue\n",
+ " blacklist = spam_words - ham_words\n",
+ " pickle.dump(blacklist, open('blacklist.pkl', 'wb'))\n",
+ "else:\n",
+ " blacklist = pickle.load(open('blacklist.pkl', 'rb') )\n",
+ "\n",
+ " # Печать сообщения о построении или загрузке blacklist\n",
+ "print('Blacklist of {} tokens successfully built/loaded'.format(len(blacklist)))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "scrolled": false
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "{'pectora',\n",
+ " 'sleet',\n",
+ " 'soma',\n",
+ " 'sorb',\n",
+ " 'raglan',\n",
+ " 'pluma',\n",
+ " 'thrower',\n",
+ " 'ducal',\n",
+ " 'vatman',\n",
+ " 'biaxial',\n",
+ " 'choral',\n",
+ " 'muzz',\n",
+ " 'merk',\n",
+ " 'degum',\n",
+ " 'lino',\n",
+ " 'punctual',\n",
+ " 'zig',\n",
+ " 'whopper',\n",
+ " 'saunter',\n",
+ " 'commot',\n",
+ " 'pian',\n",
+ " 'enchant',\n",
+ " 'starlit',\n",
+ " 'handmaid',\n",
+ " 'matchbook',\n",
+ " 'chil',\n",
+ " 'prote',\n",
+ " 'cush',\n",
+ " 'feme',\n",
+ " 'cerulean',\n",
+ " 'flamenco',\n",
+ " 'fie',\n",
+ " 'adroit',\n",
+ " 'calor',\n",
+ " 'electrician',\n",
+ " 'batik',\n",
+ " 'depositor',\n",
+ " 'ammonia',\n",
+ " 'dor',\n",
+ " 'humph',\n",
+ " 'throb',\n",
+ " 'osteosarcoma',\n",
+ " 'phylum',\n",
+ " 'staphylococci',\n",
+ " 'redub',\n",
+ " 'romaunt',\n",
+ " 'castrum',\n",
+ " 'billyboy',\n",
+ " 'snug',\n",
+ " 'phosphor',\n",
+ " 'caroli',\n",
+ " 'sweatband',\n",
+ " 'sequent',\n",
+ " 'flutter',\n",
+ " 'abac',\n",
+ " 'subjunct',\n",
+ " 'matriarch',\n",
+ " 'trisect',\n",
+ " 'hazelwood',\n",
+ " 'crept',\n",
+ " 'sienna',\n",
+ " 'camelopard',\n",
+ " 'austral',\n",
+ " 'amaranth',\n",
+ " 'embank',\n",
+ " 'briar',\n",
+ " 'fishtail',\n",
+ " 'shou',\n",
+ " 'chessboard',\n",
+ " 'reman',\n",
+ " 'ponent',\n",
+ " 'coxa',\n",
+ " 'ornithologist',\n",
+ " 'colter',\n",
+ " 'ammono',\n",
+ " 'bajada',\n",
+ " 'heterotopia',\n",
+ " 'cand',\n",
+ " 'rainless',\n",
+ " 'nar',\n",
+ " 'polka',\n",
+ " 'burrow',\n",
+ " 'augend',\n",
+ " 'irrupt',\n",
+ " 'quarrel',\n",
+ " 'nothing',\n",
+ " 'goldfish',\n",
+ " 'fibrin',\n",
+ " 'anvil',\n",
+ " 'bree',\n",
+ " 'malm',\n",
+ " 'coloss',\n",
+ " 'tenderloin',\n",
+ " 'siskin',\n",
+ " 'precinct',\n",
+ " 'sider',\n",
+ " 'centrum',\n",
+ " 'rhymer',\n",
+ " 'artificial',\n",
+ " 'brackish',\n",
+ " 'schooner',\n",
+ " 'quickset',\n",
+ " 'repugn',\n",
+ " 'ironwork',\n",
+ " 'chare',\n",
+ " 'sparerib',\n",
+ " 'thee',\n",
+ " 'citril',\n",
+ " 'stade',\n",
+ " 'facer',\n",
+ " 'escolar',\n",
+ " 'cyp',\n",
+ " 'orograph',\n",
+ " 'fennel',\n",
+ " 'saltbush',\n",
+ " 'cantar',\n",
+ " 'bedpost',\n",
+ " 'conic',\n",
+ " 'unal',\n",
+ " 'retrogress',\n",
+ " 'baw',\n",
+ " 'yuca',\n",
+ " 'humanist',\n",
+ " 'lour',\n",
+ " 'jacinth',\n",
+ " 'jud',\n",
+ " 'pergola',\n",
+ " 'simular',\n",
+ " 'passeriform',\n",
+ " 'protean',\n",
+ " 'cashier',\n",
+ " 'mejorana',\n",
+ " 'affront',\n",
+ " 'dragoon',\n",
+ " 'expansion',\n",
+ " 'moksha',\n",
+ " 'smolt',\n",
+ " 'poter',\n",
+ " 'baboon',\n",
+ " 'pater',\n",
+ " 'elegiac',\n",
+ " 'mellow',\n",
+ " 'gopher',\n",
+ " 'hiation',\n",
+ " 'boomerang',\n",
+ " 'birk',\n",
+ " 'conclusion',\n",
+ " 'credenza',\n",
+ " 'gat',\n",
+ " 'maud',\n",
+ " 'fana',\n",
+ " 'nativist',\n",
+ " 'astern',\n",
+ " 'karma',\n",
+ " 'garlic',\n",
+ " 'cinerea',\n",
+ " 'individualist',\n",
+ " 'saddler',\n",
+ " 'nonet',\n",
+ " 'tomahawk',\n",
+ " 'petulant',\n",
+ " 'slut',\n",
+ " 'bronchi',\n",
+ " 'precess',\n",
+ " 'lank',\n",
+ " 'raffia',\n",
+ " 'anonymous',\n",
+ " 'toother',\n",
+ " 'condominium',\n",
+ " 'beechwood',\n",
+ " 'mermaid',\n",
+ " 'macer',\n",
+ " 'abut',\n",
+ " 'hereabout',\n",
+ " 'autobahn',\n",
+ " 'wort',\n",
+ " 'bolling',\n",
+ " 'kelt',\n",
+ " 'eland',\n",
+ " 'excreta',\n",
+ " 'appet',\n",
+ " 'cowbird',\n",
+ " 'shama',\n",
+ " 'spart',\n",
+ " 'ordinar',\n",
+ " 'envier',\n",
+ " 'neurotoxin',\n",
+ " 'terzo',\n",
+ " 'Og',\n",
+ " 'dawnlight',\n",
+ " 'proo',\n",
+ " 'yowl',\n",
+ " 'perciform',\n",
+ " 'glee',\n",
+ " 'avera',\n",
+ " 'concoct',\n",
+ " 'amomum',\n",
+ " 'dimorph',\n",
+ " 'barium',\n",
+ " 'shrew',\n",
+ " 'propagandist',\n",
+ " 'nog',\n",
+ " 'fot',\n",
+ " 'empt',\n",
+ " 'pome',\n",
+ " 'wack',\n",
+ " 'companionship',\n",
+ " 'anadrom',\n",
+ " 'sportswear',\n",
+ " 'whir',\n",
+ " 'cava',\n",
+ " 'untrain',\n",
+ " 'leaflet',\n",
+ " 'chapeau',\n",
+ " 'ashen',\n",
+ " 'quod',\n",
+ " 'intercolumn',\n",
+ " 'mokum',\n",
+ " 'lungi',\n",
+ " 'bacteriostat',\n",
+ " 'mestizo',\n",
+ " 'ostrich',\n",
+ " 'munj',\n",
+ " 'scupper',\n",
+ " 'liman',\n",
+ " 'awlwort',\n",
+ " 'kersey',\n",
+ " 'pussycat',\n",
+ " 'horticulturist',\n",
+ " 'stallion',\n",
+ " 'brakeman',\n",
+ " 'lectual',\n",
+ " 'sarsaparilla',\n",
+ " 'sitch',\n",
+ " 'rind',\n",
+ " 'precook',\n",
+ " 'positron',\n",
+ " 'cantor',\n",
+ " 'stanch',\n",
+ " 'dhai',\n",
+ " 'creed',\n",
+ " 'wight',\n",
+ " 'zeal',\n",
+ " 'circumpolar',\n",
+ " 'carbonyl',\n",
+ " 'alfresco',\n",
+ " 'chaffinch',\n",
+ " 'sawdust',\n",
+ " 'reobtain',\n",
+ " 'looker',\n",
+ " 'lifeguard',\n",
+ " 'unimodular',\n",
+ " 'raceway',\n",
+ " 'keratin',\n",
+ " 'drool',\n",
+ " 'guan',\n",
+ " 'becoward',\n",
+ " 'beheld',\n",
+ " 'abbey',\n",
+ " 'cordillera',\n",
+ " 'threefold',\n",
+ " 'eyelet',\n",
+ " 'sordid',\n",
+ " 'toeboard',\n",
+ " 'smite',\n",
+ " 'colder',\n",
+ " 'unreal',\n",
+ " 'knockdown',\n",
+ " 'pagoda',\n",
+ " 'flane',\n",
+ " 'consonant',\n",
+ " 'alchemist',\n",
+ " 'rutherford',\n",
+ " 'harn',\n",
+ " 'furioso',\n",
+ " 'dha',\n",
+ " 'upo',\n",
+ " 'nard',\n",
+ " 'nightlong',\n",
+ " 'concerto',\n",
+ " 'bebar',\n",
+ " 'apraxia',\n",
+ " 'naiad',\n",
+ " 'yachtsman',\n",
+ " 'estrogen',\n",
+ " 'bastion',\n",
+ " 'pouch',\n",
+ " 'unio',\n",
+ " 'speck',\n",
+ " 'saya',\n",
+ " 'disinfect',\n",
+ " 'teapot',\n",
+ " 'creak',\n",
+ " 'titi',\n",
+ " 'godhead',\n",
+ " 'discolor',\n",
+ " 'lepidopterist',\n",
+ " 'scalpel',\n",
+ " 'herewith',\n",
+ " 'topper',\n",
+ " 'royalist',\n",
+ " 'codicil',\n",
+ " 'comedown',\n",
+ " 'raindrop',\n",
+ " 'waar',\n",
+ " 'shrivel',\n",
+ " 'pock',\n",
+ " 'kicker',\n",
+ " 'nasi',\n",
+ " 'scram',\n",
+ " 'tacker',\n",
+ " 'apaid',\n",
+ " 'cenotaph',\n",
+ " 'utch',\n",
+ " 'ersatz',\n",
+ " 'capuchin',\n",
+ " 'quinto',\n",
+ " 'barracuda',\n",
+ " 'fairyland',\n",
+ " 'samson',\n",
+ " 'leafless',\n",
+ " 'humic',\n",
+ " 'marmot',\n",
+ " 'brander',\n",
+ " 'hatchet',\n",
+ " 'tabla',\n",
+ " 'toat',\n",
+ " 'saccharum',\n",
+ " 'yeoman',\n",
+ " 'henceforth',\n",
+ " 'rondeau',\n",
+ " 'mink',\n",
+ " 'fingertip',\n",
+ " 'tracer',\n",
+ " 'quadrivium',\n",
+ " 'capybara',\n",
+ " 'corbeau',\n",
+ " 'mildew',\n",
+ " 'fowl',\n",
+ " 'soothsay',\n",
+ " 'talker',\n",
+ " 'tung',\n",
+ " 'whirlwind',\n",
+ " 'individual',\n",
+ " 'maxilla',\n",
+ " 'proximo',\n",
+ " 'pert',\n",
+ " 'brough',\n",
+ " 'taxonomist',\n",
+ " 'husbandman',\n",
+ " 'tempora',\n",
+ " 'patrician',\n",
+ " 'doff',\n",
+ " 'coot',\n",
+ " 'dactyl',\n",
+ " 'relevant',\n",
+ " 'nese',\n",
+ " 'megohm',\n",
+ " 'sentient',\n",
+ " 'limequat',\n",
+ " 'branchlet',\n",
+ " 'electro',\n",
+ " 'damiana',\n",
+ " 'sequin',\n",
+ " 'bluer',\n",
+ " 'malar',\n",
+ " 'anta',\n",
+ " 'axon',\n",
+ " 'dodo',\n",
+ " 'shopworn',\n",
+ " 'lintel',\n",
+ " 'redevelop',\n",
+ " 'hoi',\n",
+ " 'parson',\n",
+ " 'tyke',\n",
+ " 'psalm',\n",
+ " 'skittish',\n",
+ " 'satanist',\n",
+ " 'paler',\n",
+ " 'desi',\n",
+ " 'mola',\n",
+ " 'vicar',\n",
+ " 'chital',\n",
+ " 'kachin',\n",
+ " 'hoatzin',\n",
+ " 'serum',\n",
+ " 'ilex',\n",
+ " 'mallard',\n",
+ " 'vinifera',\n",
+ " 'holler',\n",
+ " 'swordfish',\n",
+ " 'dimmer',\n",
+ " 'adulter',\n",
+ " 'tari',\n",
+ " 'battel',\n",
+ " 'aquavit',\n",
+ " 'housecoat',\n",
+ " 'decagon',\n",
+ " 'vallum',\n",
+ " 'armadillo',\n",
+ " 'putter',\n",
+ " 'tode',\n",
+ " 'ephyra',\n",
+ " 'spinneret',\n",
+ " 'minnow',\n",
+ " 'longhair',\n",
+ " 'glover',\n",
+ " 'sensational',\n",
+ " 'natal',\n",
+ " 'carbo',\n",
+ " 'florid',\n",
+ " 'skiff',\n",
+ " 'crust',\n",
+ " 'ballast',\n",
+ " 'rebook',\n",
+ " 'bonair',\n",
+ " 'chorda',\n",
+ " 'comfrey',\n",
+ " 'crouton',\n",
+ " 'bulkhead',\n",
+ " 'moorland',\n",
+ " 'stam',\n",
+ " 'goi',\n",
+ " 'muskeg',\n",
+ " 'stench',\n",
+ " 'manipular',\n",
+ " 'loca',\n",
+ " 'corymb',\n",
+ " 'downbear',\n",
+ " 'bisson',\n",
+ " 'chorist',\n",
+ " 'octillion',\n",
+ " 'smacker',\n",
+ " 'egghead',\n",
+ " 'vermilion',\n",
+ " 'strew',\n",
+ " 'persimmon',\n",
+ " 'teamwork',\n",
+ " 'ascan',\n",
+ " 'darter',\n",
+ " 'pugilist',\n",
+ " 'sweatshop',\n",
+ " 'deign',\n",
+ " 'stardom',\n",
+ " 'coelacanth',\n",
+ " 'motorway',\n",
+ " 'rata',\n",
+ " 'fatherless',\n",
+ " 'woodpeck',\n",
+ " 'triton',\n",
+ " 'repin',\n",
+ " 'ament',\n",
+ " 'melon',\n",
+ " 'zeppelin',\n",
+ " 'chard',\n",
+ " 'courtier',\n",
+ " 'bayamo',\n",
+ " 'hangman',\n",
+ " 'zag',\n",
+ " 'gamecock',\n",
+ " 'incognito',\n",
+ " 'geneticist',\n",
+ " 'oblong',\n",
+ " 'thill',\n",
+ " 'cyme',\n",
+ " 'tua',\n",
+ " 'astral',\n",
+ " 'prater',\n",
+ " 'coauthor',\n",
+ " 'yeat',\n",
+ " 'raptor',\n",
+ " 'dvaita',\n",
+ " 'monotheist',\n",
+ " 'twite',\n",
+ " 'boarder',\n",
+ " 'spittoon',\n",
+ " 'tweet',\n",
+ " 'snifter',\n",
+ " 'saccharin',\n",
+ " 'trichi',\n",
+ " 'testudo',\n",
+ " 'daredevil',\n",
+ " 'prig',\n",
+ " 'headstrong',\n",
+ " 'waer',\n",
+ " 'rhine',\n",
+ " 'kapp',\n",
+ " 'grosbeak',\n",
+ " 'lum',\n",
+ " 'chromo',\n",
+ " 'yarrow',\n",
+ " 'melodia',\n",
+ " 'plainer',\n",
+ " 'maggot',\n",
+ " 'whereupon',\n",
+ " 'kora',\n",
+ " 'voivodeship',\n",
+ " 'twilight',\n",
+ " 'cornea',\n",
+ " 'halbert',\n",
+ " 'cassinoid',\n",
+ " 'manship',\n",
+ " 'redcoat',\n",
+ " 'hither',\n",
+ " 'axial',\n",
+ " 'apartheid',\n",
+ " 'satchel',\n",
+ " 'shim',\n",
+ " 'affection',\n",
+ " 'wetter',\n",
+ " 'pelt',\n",
+ " 'tarsi',\n",
+ " 'ringlead',\n",
+ " 'hypanthium',\n",
+ " 'arpent',\n",
+ " 'pizzeria',\n",
+ " 'gulp',\n",
+ " 'empower',\n",
+ " 'dool',\n",
+ " 'abrupt',\n",
+ " 'fiberboard',\n",
+ " 'metaphysician',\n",
+ " 'nobleman',\n",
+ " 'copperhead',\n",
+ " 'fontanel',\n",
+ " 'nonsectarian',\n",
+ " 'mew',\n",
+ " 'turbid',\n",
+ " 'repen',\n",
+ " 'windshield',\n",
+ " 'prehuman',\n",
+ " 'washbowl',\n",
+ " 'auk',\n",
+ " 'phono',\n",
+ " 'brewster',\n",
+ " 'sura',\n",
+ " 'survivalist',\n",
+ " 'downturn',\n",
+ " 'slag',\n",
+ " 'nana',\n",
+ " 'lora',\n",
+ " 'cupola',\n",
+ " 'libido',\n",
+ " 'henna',\n",
+ " 'shredder',\n",
+ " 'escheat',\n",
+ " 'renal',\n",
+ " 'ethel',\n",
+ " 'corrigendum',\n",
+ " 'muslin',\n",
+ " 'reservoir',\n",
+ " 'calid',\n",
+ " 'tute',\n",
+ " 'turndown',\n",
+ " 'valerian',\n",
+ " 'bigmouth',\n",
+ " 'buxom',\n",
+ " 'pilar',\n",
+ " 'illegal',\n",
+ " 'taha',\n",
+ " 'terron',\n",
+ " 'adolescent',\n",
+ " 'pacifist',\n",
+ " 'footpad',\n",
+ " 'bandstand',\n",
+ " 'tired',\n",
+ " 'matsu',\n",
+ " 'rooker',\n",
+ " 'daphnoid',\n",
+ " 'acta',\n",
+ " 'plover',\n",
+ " 'gerfalcon',\n",
+ " 'arca',\n",
+ " 'stratagem',\n",
+ " 'dentata',\n",
+ " 'rougher',\n",
+ " 'sixer',\n",
+ " 'droplet',\n",
+ " 'dauber',\n",
+ " 'archway',\n",
+ " 'workbook',\n",
+ " 'cider',\n",
+ " 'folia',\n",
+ " 'cerium',\n",
+ " 'quintic',\n",
+ " 'mandala',\n",
+ " 'kola',\n",
+ " 'missel',\n",
+ " 'daresay',\n",
+ " 'clitter',\n",
+ " 'yokel',\n",
+ " 'inheritor',\n",
+ " 'sticktight',\n",
+ " 'bassist',\n",
+ " 'otolith',\n",
+ " 'navel',\n",
+ " 'bailiwick',\n",
+ " 'bight',\n",
+ " 'simper',\n",
+ " 'chevron',\n",
+ " 'mosey',\n",
+ " 'uncontrol',\n",
+ " 'bairn',\n",
+ " 'adieu',\n",
+ " 'expressway',\n",
+ " 'deme',\n",
+ " 'principiant',\n",
+ " 'cay',\n",
+ " 'multicolor',\n",
+ " 'larviform',\n",
+ " 'multiplet',\n",
+ " 'latchkey',\n",
+ " 'fob',\n",
+ " 'flaccid',\n",
+ " 'marler',\n",
+ " 'whitehead',\n",
+ " 'scrubland',\n",
+ " 'endear',\n",
+ " 'waterman',\n",
+ " 'hansel',\n",
+ " 'fireman',\n",
+ " 'planter',\n",
+ " 'clove',\n",
+ " 'howe',\n",
+ " 'charac',\n",
+ " 'skulk',\n",
+ " 'thacker',\n",
+ " 'jann',\n",
+ " 'flattish',\n",
+ " 'proconsul',\n",
+ " 'groper',\n",
+ " 'huff',\n",
+ " 'cataract',\n",
+ " 'whitebark',\n",
+ " 'scat',\n",
+ " 'treeless',\n",
+ " 'proa',\n",
+ " 'debar',\n",
+ " 'boun',\n",
+ " 'bowsprit',\n",
+ " 'stagecoach',\n",
+ " 'croucher',\n",
+ " 'revivalist',\n",
+ " 'unsought',\n",
+ " 'churr',\n",
+ " 'seamanship',\n",
+ " 'spay',\n",
+ " 'piciform',\n",
+ " 'bogard',\n",
+ " 'oriel',\n",
+ " 'telford',\n",
+ " 'microfilm',\n",
+ " 'ruru',\n",
+ " 'druggist',\n",
+ " 'nomad',\n",
+ " 'thyroid',\n",
+ " 'assistor',\n",
+ " 'uitspan',\n",
+ " 'wren',\n",
+ " 'equestrian',\n",
+ " 'dildo',\n",
+ " 'overstock',\n",
+ " 'hush',\n",
+ " 'spooler',\n",
+ " 'rehash',\n",
+ " 'crayfish',\n",
+ " 'scriber',\n",
+ " 'mapach',\n",
+ " 'ptarmigan',\n",
+ " 'sudd',\n",
+ " 'daffodil',\n",
+ " 'stude',\n",
+ " 'tice',\n",
+ " 'glycerin',\n",
+ " 'fragrant',\n",
+ " 'occultist',\n",
+ " 'musicianship',\n",
+ " 'brill',\n",
+ " 'laird',\n",
+ " 'kickshaw',\n",
+ " 'mukti',\n",
+ " 'rancher',\n",
+ " 'blanch',\n",
+ " 'snowbound',\n",
+ " 'carnal',\n",
+ " 'covey',\n",
+ " 'brushwood',\n",
+ " 'bamboo',\n",
+ " 'fizz',\n",
+ " 'pecker',\n",
+ " 'spellbind',\n",
+ " 'brickel',\n",
+ " 'capulet',\n",
+ " 'viga',\n",
+ " 'paprika',\n",
+ " 'hemlock',\n",
+ " 'bettor',\n",
+ " 'dere',\n",
+ " 'ataman',\n",
+ " 'dult',\n",
+ " 'flannel',\n",
+ " 'rowan',\n",
+ " 'brownish',\n",
+ " 'beldam',\n",
+ " 'entropion',\n",
+ " 'beld',\n",
+ " 'houseboat',\n",
+ " 'draper',\n",
+ " 'eyelash',\n",
+ " 'norther',\n",
+ " 'knob',\n",
+ " 'knop',\n",
+ " 'savoy',\n",
+ " 'betoken',\n",
+ " 'attern',\n",
+ " 'crowder',\n",
+ " 'cuarenta',\n",
+ " 'crucifix',\n",
+ " 'alderman',\n",
+ " 'woodbin',\n",
+ " 'labrusca',\n",
+ " 'butyl',\n",
+ " 'stion',\n",
+ " 'extravascular',\n",
+ " 'binder',\n",
+ " 'exoskeleton',\n",
+ " 'declaim',\n",
+ " 'tweed',\n",
+ " 'rusk',\n",
+ " 'peregrin',\n",
+ " 'exigent',\n",
+ " 'biochemist',\n",
+ " 'taum',\n",
+ " 'plateau',\n",
+ " 'grassland',\n",
+ " 'oaken',\n",
+ " 'dockyard',\n",
+ " 'rattail',\n",
+ " 'splendor',\n",
+ " 'grout',\n",
+ " 'stree',\n",
+ " 'causey',\n",
+ " 'register',\n",
+ " 'bott',\n",
+ " 'glottal',\n",
+ " 'cockerel',\n",
+ " 'japonica',\n",
+ " 'cravat',\n",
+ " 'chider',\n",
+ " 'uncut',\n",
+ " 'shyness',\n",
+ " 'drier',\n",
+ " 'lemur',\n",
+ " 'fandom',\n",
+ " 'reefer',\n",
+ " 'souther',\n",
+ " 'microlepidoptera',\n",
+ " 'melodrama',\n",
+ " 'avast',\n",
+ " 'sailboat',\n",
+ " 'memento',\n",
+ " 'educationalist',\n",
+ " 'helminth',\n",
+ " 'dorsal',\n",
+ " 'gane',\n",
+ " 'raja',\n",
+ " 'coper',\n",
+ " 'dodecahedron',\n",
+ " 'heptagon',\n",
+ " 'linwood',\n",
+ " 'akimbo',\n",
+ " 'anteroom',\n",
+ " 'sutra',\n",
+ " 'talisman',\n",
+ " 'nihilist',\n",
+ " 'eclat',\n",
+ " 'skyway',\n",
+ " 'reservist',\n",
+ " 'hawthorn',\n",
+ " 'simpleton',\n",
+ " 'agog',\n",
+ " 'orgasm',\n",
+ " 'transaction',\n",
+ " 'tern',\n",
+ " 'caudal',\n",
+ " 'grazer',\n",
+ " 'ako',\n",
+ " 'hereof',\n",
+ " 'impost',\n",
+ " 'grossen',\n",
+ " 'cratch',\n",
+ " 'verbena',\n",
+ " 'encamp',\n",
+ " 'beggar',\n",
+ " 'burgh',\n",
+ " 'ascidian',\n",
+ " 'vagrant',\n",
+ " 'milord',\n",
+ " 'overzeal',\n",
+ " 'barit',\n",
+ " 'grue',\n",
+ " 'snorkel',\n",
+ " 'carillon',\n",
+ " 'beseech',\n",
+ " 'burgher',\n",
+ " 'peal',\n",
+ " 'struthioniform',\n",
+ " 'kitcat',\n",
+ " 'xylitol',\n",
+ " 'jarl',\n",
+ " 'runt',\n",
+ " 'agal',\n",
+ " 'teet',\n",
+ " 'moveless',\n",
+ " 'vermeil',\n",
+ " 'lifeless',\n",
+ " 'sacrament',\n",
+ " 'legate',\n",
+ " 'trillium',\n",
+ " 'tramway',\n",
+ " 'starlight',\n",
+ " 'trinitarian',\n",
+ " 'peignoir',\n",
+ " 'diamagnet',\n",
+ " 'plump',\n",
+ " 'pupal',\n",
+ " 'salicyl',\n",
+ " 'hander',\n",
+ " 'wadi',\n",
+ " 'sweatproof',\n",
+ " 'tamarind',\n",
+ " 'multiplex',\n",
+ " 'pand',\n",
+ " 'parakeet',\n",
+ " 'hawfinch',\n",
+ " 'hydro',\n",
+ " 'levant',\n",
+ " 'knave',\n",
+ " 'tinct',\n",
+ " 'matricula',\n",
+ " 'wrongdoer',\n",
+ " 'mister',\n",
+ " 'lifehold',\n",
+ " 'nook',\n",
+ " 'topmost',\n",
+ " 'adjunct',\n",
+ " 'injector',\n",
+ " 'trill',\n",
+ " 'rond',\n",
+ " 'thorax',\n",
+ " 'bota',\n",
+ " 'kinnikinnick',\n",
+ " 'northeastward',\n",
+ " 'macaroni',\n",
+ " 'blackguard',\n",
+ " 'brougham',\n",
+ " 'tripper',\n",
+ " 'supraocular',\n",
+ " 'shag',\n",
+ " 'earthwork',\n",
+ " 'pall',\n",
+ " 'feu',\n",
+ " 'overfish',\n",
+ " 'pinkish',\n",
+ " 'sextet',\n",
+ " 'paroxysm',\n",
+ " 'balderdash',\n",
+ " 'heyday',\n",
+ " 'broomcorn',\n",
+ " 'vanadium',\n",
+ " 'mikado',\n",
+ " 'lien',\n",
+ " 'nightdress',\n",
+ " 'kali',\n",
+ " 'anuran',\n",
+ " 'posey',\n",
+ " 'byway',\n",
+ " 'carrion',\n",
+ " 'daybreak',\n",
+ " 'derat',\n",
+ " 'arcadian',\n",
+ " 'crowbait',\n",
+ " 'breadroot',\n",
+ " 'nebular',\n",
+ " 'blain',\n",
+ " 'aneurin',\n",
+ " 'depthless',\n",
+ " 'Ko',\n",
+ " 'fourfold',\n",
+ " 'bodkin',\n",
+ " 'wallboard',\n",
+ " 'basilica',\n",
+ " 'steamer',\n",
+ " 'airman',\n",
+ " 'arhat',\n",
+ " 'pernyi',\n",
+ " 'scend',\n",
+ " 'macrolepidoptera',\n",
+ " 'subphylum',\n",
+ " 'sixteenth',\n",
+ " 'ungloss',\n",
+ " 'wem',\n",
+ " 'greyhound',\n",
+ " 'mysid',\n",
+ " 'federalist',\n",
+ " 'atlas',\n",
+ " 'whither',\n",
+ " 'hilt',\n",
+ " 'vestal',\n",
+ " 'dern',\n",
+ " 'pollard',\n",
+ " 'westernmost',\n",
+ " 'themsel',\n",
+ " 'almagest',\n",
+ " 'tical',\n",
+ " 'camber',\n",
+ " 'tripod',\n",
+ " 'woodruff',\n",
+ " 'formic',\n",
+ " 'striction',\n",
+ " 'diarrhea',\n",
+ " 'retrench',\n",
+ " 'yar',\n",
+ " 'ponto',\n",
+ " 'tup',\n",
+ " 'caracol',\n",
+ " 'smew',\n",
+ " 'casein',\n",
+ " 'scallop',\n",
+ " 'atman',\n",
+ " 'reek',\n",
+ " 'bade',\n",
+ " 'recti',\n",
+ " 'subshrub',\n",
+ " 'pur',\n",
+ " 'burro',\n",
+ " 'freehold',\n",
+ " 'carpel',\n",
+ " 'statant',\n",
+ " 'caesura',\n",
+ " 'chanson',\n",
+ " 'docent',\n",
+ " 'succumb',\n",
+ " 'sika',\n",
+ " 'motherless',\n",
+ " 'southernmost',\n",
+ " 'mout',\n",
+ " 'ingredient',\n",
+ " 'custard',\n",
+ " 'tubman',\n",
+ " 'crossbreed',\n",
+ " 'urea',\n",
+ " 'airstrip',\n",
+ " 'campground',\n",
+ " 'productid',\n",
+ " 'olden',\n",
+ " 'oclock',\n",
+ " 'aluminium',\n",
+ " 'augen',\n",
+ " 'medievalist',\n",
+ " 'neolith',\n",
+ " 'waif',\n",
+ " 'chartist',\n",
+ " 'debutant',\n",
+ " 'zapatero',\n",
+ " 'majo',\n",
+ " 'clincher',\n",
+ " 'amour',\n",
+ " 'shack',\n",
+ " 'retrain',\n",
+ " 'caraway',\n",
+ " 'thine',\n",
+ " 'regrow',\n",
+ " 'pumpkin',\n",
+ " 'redd',\n",
+ " 'charger',\n",
+ " 'abigail',\n",
+ " 'deerskin',\n",
+ " 'nychthemeron',\n",
+ " 'holl',\n",
+ " 'aught',\n",
+ " 'scrambler',\n",
+ " 'cebid',\n",
+ " 'shanna',\n",
+ " 'kiosk',\n",
+ " 'unidirect',\n",
+ " 'tantra',\n",
+ " 'ental',\n",
+ " 'assumer',\n",
+ " 'deforest',\n",
+ " 'ecru',\n",
+ " 'rame',\n",
+ " 'pagan',\n",
+ " 'tablespoon',\n",
+ " 'tickler',\n",
+ " 'gambol',\n",
+ " 'moist',\n",
+ " 'limitless',\n",
+ " 'hexaploid',\n",
+ " 'cero',\n",
+ " ...}"
+ ]
+ },
+ "execution_count": 7,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "# Импорт модуля NLTK для работы с естественным языком\n",
+ "from nltk.corpus import words\n",
+ "word_set = set(words.words())\n",
+ "# Поиск пересечения blacklist и множества английских слов из NLTK\n",
+ "word_set.intersection(blacklist)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Анализ текста:\n",
+ "### Для каждого электронного письма из тестового набора:\n",
+ "### Текст сообщения анализируется на наличие слов из черного списка. Подсчитывается количество ложноположительных (FP), ложноотрицательных (FN), истинноположительных (TP) и истинноотрицательных (TN) результатов."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Инициализация переменных для подсчета статистики классификации\n",
+ "fp = 0 # Ложноположительные (ошибочно определенные как спам)\n",
+ "tp = 0 # Истинноположительные (правильно определенные как спам)\n",
+ "fn = 0 # Ложноотрицательные (ошибочно определенные как не спам)\n",
+ "tn = 0 # Истинноотрицательные (правильно определенные как не спам)\n",
+ "\n",
+ "\n",
+ "# Проверка каждого письма из тестового набора и классификация по наличию слов из blacklist\n",
+ "for filename in X_test:\n",
+ " path = os.path.join(DATA_DIR, filename)\n",
+ " if filename in labels:\n",
+ " label = labels[filename]\n",
+ " stems = email_read_util.load(path)\n",
+ " if not stems:\n",
+ " continue\n",
+ " stems_set = set(stems)\n",
+ " if stems_set & blacklist:\n",
+ " if label == 1:\n",
+ " fp = fp + 1\n",
+ " else:\n",
+ " tp = tp + 1\n",
+ " else:\n",
+ " if label == 1:\n",
+ " tn = tn + 1\n",
+ " else:\n",
+ " fn = fn + 1"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Визуализация:\n",
+ "### Строится матрица ошибок для оценки результатов классификации. Выводится процентное соотношение каждой ячейки матрицы ошибок относительно общего количества наблюдений."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Вывод матрицы ошибок в HTML-таблице\n",
+ "from IPython.display import HTML, display\n",
+ "conf_matrix = [[tn, fp],\n",
+ " [fn, tp]]\n",
+ "display(HTML(''.format(\n",
+ " ' |
'.join('| {} | '.format(\n",
+ " ''.join(str(_) for _ in row)) \n",
+ " for row in conf_matrix))))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Вывод процентной матрицы ошибок в HTML-таблице\n",
+ "count = tn + tp + fn + fp\n",
+ "percent_matrix = [[\"{:.1%}\".format(tn/count), \"{:.1%}\".format(fp/count)],\n",
+ " [\"{:.1%}\".format(fn/count), \"{:.1%}\".format(tp/count)]]\n",
+ "display(HTML(''.format(\n",
+ " ' |
'.join('| {} | '.format(\n",
+ " ''.join(str(_) for _ in row)) \n",
+ " for row in percent_matrix))))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Оценка точности классификации:\n",
+ "### Вычисляется и выводится процент точности классификации, основанный на количестве правильно классифицированных объектов.\n",
+ "### Результаты показывают, что классификация имеет точность около 68.6%."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Classification accuracy: 68.6%\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Вывод процента правильно классифицированных писем\n",
+ "print(\"Classification accuracy: {}\".format(\"{:.1%}\".format((tp+tn)/count)))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Этот метод - один из старейших способов фильтрации спама."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
From 9355be29f3b5fcaf4ff8757cd31a23f05c4b9611 Mon Sep 17 00:00:00 2001
From: Nurgalym2001 <89890787+Nurgalym2001@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:31:28 +0600
Subject: [PATCH 03/11] Delete chapter1/spam-fighting-lsh.ipynb
---
chapter1/spam-fighting-lsh.ipynb | 264 -------------------------------
1 file changed, 264 deletions(-)
delete mode 100644 chapter1/spam-fighting-lsh.ipynb
diff --git a/chapter1/spam-fighting-lsh.ipynb b/chapter1/spam-fighting-lsh.ipynb
deleted file mode 100644
index 0f7c8be..0000000
--- a/chapter1/spam-fighting-lsh.ipynb
+++ /dev/null
@@ -1,264 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import pickle\n",
- "import email_read_util\n",
- "from datasketch import MinHash, MinHashLSH"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Download 2007 TREC Public Spam Corpus\n",
- "1. Read the \"Agreement for use\"\n",
- " https://plg.uwaterloo.ca/~gvcormac/treccorpus07/\n",
- "\n",
- "2. Download 255 MB Corpus (trec07p.tgz) and untar into the 'chapter1/datasets' directory\n",
- "\n",
- "3. Check that the below paths for 'DATA_DIR' and 'LABELS_FILE' exist"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "DATA_DIR = 'datasets/trec07p/data/'\n",
- "LABELS_FILE = 'datasets/trec07p/full/index'\n",
- "TRAINING_SET_RATIO = 0.7"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "labels = {}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read the labels\n",
- "with open(LABELS_FILE) as f:\n",
- " for line in f:\n",
- " line = line.strip()\n",
- " label, key = line.split()\n",
- " labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Split corpus into train and test sets\n",
- "filelist = os.listdir(DATA_DIR)\n",
- "X_train = filelist[:int(len(filelist)*TRAINING_SET_RATIO)]\n",
- "X_test = filelist[int(len(filelist)*TRAINING_SET_RATIO):]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Extract only spam files for inserting into the LSH matcher\n",
- "spam_files = [x for x in X_train if labels[x] == 0]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Initialize MinHashLSH matcher with a Jaccard \n",
- "# threshold of 0.5 and 128 MinHash permutation functions\n",
- "lsh = MinHashLSH(threshold=0.5, num_perm=128)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Populate the LSH matcher with training spam MinHashes\n",
- "for idx, f in enumerate(spam_files):\n",
- " minhash = MinHash(num_perm=128)\n",
- " stems = email_read_util.load(os.path.join(DATA_DIR, f))\n",
- " if len(stems) < 2: continue\n",
- " for s in stems:\n",
- " minhash.update(s.encode('utf-8'))\n",
- " lsh.insert(f, minhash)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {},
- "outputs": [],
- "source": [
- "def lsh_predict_label(stems):\n",
- " '''\n",
- " Queries the LSH matcher and returns:\n",
- " 0 if predicted spam\n",
- " 1 if predicted ham\n",
- " -1 if parsing error\n",
- " '''\n",
- " minhash = MinHash(num_perm=128)\n",
- " if len(stems) < 2:\n",
- " return -1\n",
- " for s in stems:\n",
- " minhash.update(s.encode('utf-8'))\n",
- " matches = lsh.query(minhash)\n",
- " if matches:\n",
- " return 0\n",
- " else:\n",
- " return 1"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {},
- "outputs": [],
- "source": [
- "fp = 0\n",
- "tp = 0\n",
- "fn = 0\n",
- "tn = 0\n",
- "\n",
- "for filename in X_test:\n",
- " path = os.path.join(DATA_DIR, filename)\n",
- " if filename in labels:\n",
- " label = labels[filename]\n",
- " stems = email_read_util.load(path)\n",
- " if not stems:\n",
- " continue\n",
- " pred = lsh_predict_label(stems)\n",
- " if pred == -1:\n",
- " continue\n",
- " elif pred == 0:\n",
- " if label == 1:\n",
- " fp = fp + 1\n",
- " else:\n",
- " tp = tp + 1\n",
- " elif pred == 1:\n",
- " if label == 1:\n",
- " tn = tn + 1\n",
- " else:\n",
- " fn = fn + 1"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 11,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "from IPython.display import HTML, display\n",
- "conf_matrix = [[tn, fp],\n",
- " [fn, tp]]\n",
- "display(HTML(''.format(\n",
- " ' |
'.join('| {} | '.format(\n",
- " ''.join(str(_) for _ in row)) \n",
- " for row in conf_matrix))))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 12,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- ""
- ],
- "text/plain": [
- ""
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "count = tn + tp + fn + fp\n",
- "percent_matrix = [[\"{:.1%}\".format(tn/count), \"{:.1%}\".format(fp/count)],\n",
- " [\"{:.1%}\".format(fn/count), \"{:.1%}\".format(tp/count)]]\n",
- "display(HTML(''.format(\n",
- " ' |
'.join('| {} | '.format(\n",
- " ''.join(str(_) for _ in row)) \n",
- " for row in percent_matrix))))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Classification accuracy: 88.6%\n"
- ]
- }
- ],
- "source": [
- "print(\"Classification accuracy: {}\".format(\"{:.1%}\".format((tp+tn)/count)))"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
From 487806e13ff8e3c5e1f40a2a7703bdb53880da7a Mon Sep 17 00:00:00 2001
From: Nurgalym2001 <89890787+Nurgalym2001@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:31:50 +0600
Subject: [PATCH 04/11] Add files via upload
---
chapter1/spam-fighting-lsh.ipynb | 256 +++++++++++++++++++++++++++++++
1 file changed, 256 insertions(+)
create mode 100644 chapter1/spam-fighting-lsh.ipynb
diff --git a/chapter1/spam-fighting-lsh.ipynb b/chapter1/spam-fighting-lsh.ipynb
new file mode 100644
index 0000000..113bcec
--- /dev/null
+++ b/chapter1/spam-fighting-lsh.ipynb
@@ -0,0 +1,256 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Этот код выполняет классификацию электронных писем на спам и не спам, используя алгоритм Locality Sensitive Hashing (LSH) с помощью MinHash и MinHashLSH из библиотеки datasketch. Каждое письмо представлено как множество уникальных слов (stems), и на основе этого множества строится MinHash. Далее производится поиск совпадений MinHash с помощью LSH для определения, является ли письмо спамом или не спамом. После классификации вычисляются метрики качества, такие как матрица ошибок, процент ошибок и точность классификации."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Импорт необходимых модулей\n",
+ "import os # Модуль для работы с операционной системой\n",
+ "import pickle # Модуль для сериализации и десериализации объектов Python\n",
+ "import email_read_util # Модуль для чтения и обработки электронных писем\n",
+ "from datasketch import MinHash, MinHashLSH # Модули для работы с MinHash и MinHashLSH"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Определение директории с данными и файла с метками\n",
+ "DATA_DIR = 'datasets/trec07p/data/' # Путь к директории с данными\n",
+ "LABELS_FILE = 'datasets/trec07p/full/index' # Путь к файлу с метками\n",
+ "TRAINING_SET_RATIO = 0.7 # Пропорция разделения набора данных на тренировочный и тестовый\n",
+ "labels = {} # Словарь для хранения меток (имя файла: метка)\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Чтение и обработка файла меток\n",
+ "with open(LABELS_FILE) as f:\n",
+ " for line in f:\n",
+ " line = line.strip()\n",
+ " label, key = line.split()\n",
+ " labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Разделение корпуса на тренировочный и тестовый наборы\n",
+ "filelist = os.listdir(DATA_DIR)\n",
+ "X_train = filelist[:int(len(filelist)*TRAINING_SET_RATIO)]\n",
+ "X_test = filelist[int(len(filelist)*TRAINING_SET_RATIO):]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Извлечение только файлов со спамом для вставки в сопоставитель LSH\n",
+ "spam_files = [x for x in X_train if labels[x] == 0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Инициализация сопоставителя MinHashLSH с порогом Жаккара 0.5 и 128 функциями перестановки MinHash\n",
+ "lsh = MinHashLSH(threshold=0.5, num_perm=128)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Заполнение сопоставителя LSH тренировочными MinHashes для спама\n",
+ "for idx, f in enumerate(spam_files):\n",
+ " minhash = MinHash(num_perm=128)\n",
+ " stems = email_read_util.load(os.path.join(DATA_DIR, f))\n",
+ " if len(stems) < 2: continue\n",
+ " for s in stems:\n",
+ " minhash.update(s.encode('utf-8'))\n",
+ " lsh.insert(f, minhash)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def lsh_predict_label(stems):\n",
+ " '''\n",
+ " Queries the LSH matcher and returns:\n",
+ " 0 if predicted spam\n",
+ " 1 if predicted ham\n",
+ " -1 if parsing error\n",
+ " '''\n",
+ " minhash = MinHash(num_perm=128)\n",
+ " if len(stems) < 2:\n",
+ " return -1\n",
+ " for s in stems:\n",
+ " minhash.update(s.encode('utf-8'))\n",
+ " matches = lsh.query(minhash)\n",
+ " if matches:\n",
+ " return 0\n",
+ " else:\n",
+ " return 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Инициализация переменных для подсчета статистики классификации\n",
+ "fp = 0 # Ложноположительные (ошибочно определенные как спам)\n",
+ "tp = 0 # Истинноположительные (правильно определенные как спам)\n",
+ "fn = 0 # Ложноотрицательные (ошибочно определенные как не спам)\n",
+ "tn = 0 # Истинноотрицательные (правильно определенные как не спам)\n",
+ "\n",
+ "# Проверка каждого письма из тестового набора и классификация с помощью сопоставителя LSH\n",
+ "for filename in X_test:\n",
+ " path = os.path.join(DATA_DIR, filename)\n",
+ " if filename in labels:\n",
+ " label = labels[filename]\n",
+ " stems = email_read_util.load(path)\n",
+ " if not stems:\n",
+ " continue\n",
+ " pred = lsh_predict_label(stems)\n",
+ " if pred == -1:\n",
+ " continue\n",
+ " elif pred == 0:\n",
+ " if label == 1:\n",
+ " fp = fp + 1\n",
+ " else:\n",
+ " tp = tp + 1\n",
+ " elif pred == 1:\n",
+ " if label == 1:\n",
+ " tn = tn + 1\n",
+ " else:\n",
+ " fn = fn + 1"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 11,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Вывод матрицы ошибок в HTML-таблице\n",
+ "from IPython.display import HTML, display\n",
+ "conf_matrix = [[tn, fp],\n",
+ " [fn, tp]]\n",
+ "display(HTML(''.format(\n",
+ " ' |
'.join('| {} | '.format(\n",
+ " ''.join(str(_) for _ in row)) \n",
+ " for row in conf_matrix))))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ ""
+ ],
+ "text/plain": [
+ ""
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Вывод процентной матрицы ошибок в HTML-таблице\n",
+ "count = tn + tp + fn + fp\n",
+ "percent_matrix = [[\"{:.1%}\".format(tn/count), \"{:.1%}\".format(fp/count)],\n",
+ " [\"{:.1%}\".format(fn/count), \"{:.1%}\".format(tp/count)]]\n",
+ "display(HTML(''.format(\n",
+ " ' |
'.join('| {} | '.format(\n",
+ " ''.join(str(_) for _ in row)) \n",
+ " for row in percent_matrix))))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Classification accuracy: 88.6%\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Вывод процента правильно классифицированных писем\n",
+ "print(\"Classification accuracy: {}\".format(\"{:.1%}\".format((tp+tn)/count)))"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
From eaa101ebcc451828ae105f661e55b427bad6cd8e Mon Sep 17 00:00:00 2001
From: Nurgalym2001 <89890787+Nurgalym2001@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:35:36 +0600
Subject: [PATCH 05/11] Delete chapter1/spam-fighting-naivebayes.ipynb
---
chapter1/spam-fighting-naivebayes.ipynb | 171 ------------------------
1 file changed, 171 deletions(-)
delete mode 100644 chapter1/spam-fighting-naivebayes.ipynb
diff --git a/chapter1/spam-fighting-naivebayes.ipynb b/chapter1/spam-fighting-naivebayes.ipynb
deleted file mode 100644
index 5170e27..0000000
--- a/chapter1/spam-fighting-naivebayes.ipynb
+++ /dev/null
@@ -1,171 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import email_read_util"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Download 2007 TREC Public Spam Corpus\n",
- "1. Read the \"Agreement for use\"\n",
- " https://plg.uwaterloo.ca/~gvcormac/treccorpus07/\n",
- "\n",
- "2. Download 255 MB Corpus (trec07p.tgz) and untar into the 'chapter1/datasets' directory\n",
- "\n",
- "3. Check that the below paths for 'DATA_DIR' and 'LABELS_FILE' exist"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "DATA_DIR = 'datasets/trec07p/data/'\n",
- "LABELS_FILE = 'datasets/trec07p/full/index'\n",
- "TRAINING_SET_RATIO = 0.7"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [],
- "source": [
- "labels = {}\n",
- "# Read the labels\n",
- "with open(LABELS_FILE) as f:\n",
- " for line in f:\n",
- " line = line.strip()\n",
- " label, key = line.split()\n",
- " labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [],
- "source": [
- "def read_email_files():\n",
- " X = []\n",
- " y = [] \n",
- " for i in range(len(labels)):\n",
- " filename = 'inmail.' + str(i+1)\n",
- " email_str = email_read_util.extract_email_text(\n",
- " os.path.join(DATA_DIR, filename))\n",
- " X.append(email_str)\n",
- " y.append(labels[filename])\n",
- " return X, y"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [],
- "source": [
- "X, y = read_email_files()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "/usr/local/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
- " FutureWarning)\n"
- ]
- }
- ],
- "source": [
- "from sklearn.model_selection import train_test_split \n",
- "\n",
- "X_train, X_test, y_train, y_test, idx_train, idx_test = \\\n",
- " train_test_split(X, y, range(len(y)), \n",
- " train_size=TRAINING_SET_RATIO, random_state=2)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {},
- "outputs": [],
- "source": [
- "from sklearn.feature_extraction.text import CountVectorizer\n",
- "\n",
- "vectorizer = CountVectorizer()\n",
- "X_train_vector = vectorizer.fit_transform(X_train)\n",
- "X_test_vector = vectorizer.transform(X_test)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- " precision recall f1-score support\n",
- "\n",
- " Spam 0.99 0.94 0.97 15035\n",
- " Ham 0.90 0.98 0.94 7591\n",
- "\n",
- "avg / total 0.96 0.96 0.96 22626\n",
- "\n",
- "Classification accuracy 95.6%\n"
- ]
- }
- ],
- "source": [
- "from sklearn.naive_bayes import MultinomialNB\n",
- "from sklearn.metrics import accuracy_score\n",
- "from sklearn.metrics import classification_report\n",
- "\n",
- "# Initialize the classifier and make label predictions\n",
- "mnb = MultinomialNB()\n",
- "mnb.fit(X_train_vector, y_train)\n",
- "y_pred = mnb.predict(X_test_vector)\n",
- "\n",
- "# Print results\n",
- "print(classification_report(y_test, y_pred, target_names=['Spam', 'Ham']))\n",
- "print('Classification accuracy {:.1%}'.format(accuracy_score(y_test, y_pred)))"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.6.3"
- }
- },
- "nbformat": 4,
- "nbformat_minor": 2
-}
From 990012ae0bf99bf04542f8eb92daae6695d80246 Mon Sep 17 00:00:00 2001
From: Nurgalym2001 <89890787+Nurgalym2001@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:36:24 +0600
Subject: [PATCH 06/11] Add files via upload
---
chapter1/spam-fighting-naivebayes.ipynb | 249 ++++++++++++++++++++++++
1 file changed, 249 insertions(+)
create mode 100644 chapter1/spam-fighting-naivebayes.ipynb
diff --git a/chapter1/spam-fighting-naivebayes.ipynb b/chapter1/spam-fighting-naivebayes.ipynb
new file mode 100644
index 0000000..f7ecac9
--- /dev/null
+++ b/chapter1/spam-fighting-naivebayes.ipynb
@@ -0,0 +1,249 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Этот код выполняет классификацию электронных писем на спам и не спам с использованием метода Multinomial Naive Bayes. Он используется для оценки производительности классификатора на тестовом наборе данных, вычисляя такие метрики, как точность (precision), полнота (recall), F1-мера (F1-score) и общая точность классификации (classification accuracy). В результате вы получаете оценку эффективности классификации модели на задаче определения спама в электронной почте."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os # Модуль для работы с операционной системой\n",
+ "import email_read_util # Модуль для чтения и обработки электронных писем\n",
+ "from sklearn.model_selection import train_test_split # Модуль для разделения набора данных на обучающий и тестовый\n",
+ "from sklearn.feature_extraction.text import CountVectorizer # Модуль для извлечения признаков из текста\n",
+ "from sklearn.naive_bayes import MultinomialNB # Модуль для реализации наивного байесовского классификатора\n",
+ "from sklearn.metrics import accuracy_score, classification_report # Модуль для оценки качества классификации"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Download 2007 TREC Public Spam Corpus\n",
+ "1. Read the \"Agreement for use\"\n",
+ " https://plg.uwaterloo.ca/~gvcormac/treccorpus07/\n",
+ "\n",
+ "2. Download 255 MB Corpus (trec07p.tgz) and untar into the 'chapter1/datasets' directory\n",
+ "\n",
+ "3. Check that the below paths for 'DATA_DIR' and 'LABELS_FILE' exist"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Определение директории данных и файлов меток:\n",
+ "### В этом блоке определяются пути к директории с данными и файлу с метками. Переменные DATA_DIR и LABELS_FILE содержат пути к директории и файлу, соответственно. Также задается параметр TRAINING_SET_RATIO, который определяет пропорцию разделения данных на обучающий и тестовый наборы."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Определение директории с данными и файла с метками\n",
+ "DATA_DIR = 'datasets/trec07p/data/' # Путь к директории с данными\n",
+ "LABELS_FILE = 'datasets/trec07p/full/index' # Путь к файлу с метками\n",
+ "TRAINING_SET_RATIO = 0.7 # Пропорция разделения набора данных на тренировочный и тестовый\n",
+ "labels = {} # Словарь для хранения меток (имя файла: метка)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Чтение и обработка меток:\n",
+ "### Далее происходит чтение и обработка файла с метками. С помощью цикла for считываются метки из файла LABELS_FILE и сохраняются в словаре labels, где ключами являются имена файлов, а значениями - метки"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Чтение и обработка файла меток\n",
+ "with open(LABELS_FILE) as f:\n",
+ " for line in f:\n",
+ " line = line.strip()\n",
+ " label, key = line.split()\n",
+ " labels[key.split('/')[-1]] = 1 if label.lower() == 'ham' else 0"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Функция для чтения электронных писем:\n",
+ "### Здесь определяется функция read_email_files(), которая читает и обрабатывает электронные письма. В цикле for происходит итерация по количеству меток, и для каждой метки считывается соответствующее письмо из директории DATA_DIR с помощью функции extract_email_text из модуля email_read_util. Текст письма добавляется в список X, а его метка - в список y."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def read_email_files():\n",
+ " X = []\n",
+ " y = [] \n",
+ " for i in range(len(labels)):\n",
+ " filename = 'inmail.' + str(i+1)\n",
+ " email_str = email_read_util.extract_email_text(\n",
+ " os.path.join(DATA_DIR, filename))\n",
+ " X.append(email_str)\n",
+ " y.append(labels[filename])\n",
+ " return X, y"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Чтение и обработка электронных писем\n",
+ "X, y = read_email_files()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Чтение и обработка данных:\n",
+ "### В этой части кода вызывается функция read_email_files(), результат ее выполнения сохраняется в переменные X и y. После этого данные разделяются на обучающий и тестовый наборы с помощью функции train_test_split, где X и y - это тексты писем и их метки соответственно."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/usr/local/lib/python3.6/site-packages/sklearn/model_selection/_split.py:2026: FutureWarning: From version 0.21, test_size will always complement train_size unless both are specified.\n",
+ " FutureWarning)\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Разделение набора данных на обучающий и тестовый\n",
+ "from sklearn.model_selection import train_test_split \n",
+ "\n",
+ "X_train, X_test, y_train, y_test, idx_train, idx_test = \\\n",
+ " train_test_split(X, y, range(len(y)), \n",
+ " train_size=TRAINING_SET_RATIO, random_state=2)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Преобразование текста в признаки:\n",
+ "### Для преобразования текста писем в признаки используется класс CountVectorizer. Данные обучающего и тестового наборов преобразуются в матрицы признаков с помощью методов fit_transform и transform соответственно."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Преобразование текста писем в векторы признаков\n",
+ "from sklearn.feature_extraction.text import CountVectorizer\n",
+ "\n",
+ "vectorizer = CountVectorizer()\n",
+ "X_train_vector = vectorizer.fit_transform(X_train)\n",
+ "X_test_vector = vectorizer.transform(X_test)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Обучение классификатора и оценка его производительности:\n",
+ "### Далее создается объект классификатора MultinomialNB, который обучается на обучающем наборе с помощью метода fit. Затем выполняется предсказание меток для тестового набора с помощью метода predict. После этого выводится отчет о классификации с помощью функции classification_report и выводится точность классификации с помощью функции accuracy_score."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ " precision recall f1-score support\n",
+ "\n",
+ " Spam 0.99 0.94 0.97 15035\n",
+ " Ham 0.90 0.98 0.94 7591\n",
+ "\n",
+ "avg / total 0.96 0.96 0.96 22626\n",
+ "\n",
+ "Classification accuracy 95.6%\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Обучение наивного байесовского классификатора\n",
+ "from sklearn.naive_bayes import MultinomialNB\n",
+ "from sklearn.metrics import accuracy_score\n",
+ "from sklearn.metrics import classification_report\n",
+ "\n",
+ "# Инициализируйте классификатор и делайте предсказания меток\n",
+ "mnb = MultinomialNB()\n",
+ "mnb.fit(X_train_vector, y_train)\n",
+ "y_pred = mnb.predict(X_test_vector)\n",
+ "\n",
+ "# Вывод результатов классификации\n",
+ "print(classification_report(y_test, y_pred, target_names=['Spam', 'Ham']))\n",
+ "print('Classification accuracy {:.1%}'.format(accuracy_score(y_test, y_pred)))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Классификационная точность (Classification accuracy) составляет 95.6%, что является общей точностью классификации модели на тестовом наборе данных. Это процент правильно классифицированных объектов относительно общего числа объектов в тестовом наборе."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.9"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
From b2da7ebd61ddfc9336e4a450523a1b854825380f Mon Sep 17 00:00:00 2001
From: Nurgalym2001 <89890787+Nurgalym2001@users.noreply.github.com>
Date: Fri, 8 Mar 2024 18:37:01 +0600
Subject: [PATCH 07/11] Delete
chapter2/logistic-regression-fraud-detection.ipynb
---
.../logistic-regression-fraud-detection.ipynb | 579 ------------------
1 file changed, 579 deletions(-)
delete mode 100644 chapter2/logistic-regression-fraud-detection.ipynb
diff --git a/chapter2/logistic-regression-fraud-detection.ipynb b/chapter2/logistic-regression-fraud-detection.ipynb
deleted file mode 100644
index 2305cb7..0000000
--- a/chapter2/logistic-regression-fraud-detection.ipynb
+++ /dev/null
@@ -1,579 +0,0 @@
-{
- "cells": [
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {},
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "from sklearn.model_selection import train_test_split\n",
- "from sklearn.linear_model import LogisticRegression\n",
- "from sklearn.metrics import accuracy_score, confusion_matrix"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read in the data from the CSV file\n",
- "df = pd.read_csv('datasets/payment_fraud.csv')"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "\n",
- " \n",
- " \n",
- " \n",
- " | \n",
- " accountAgeDays | \n",
- " numItems | \n",
- " localTime | \n",
- " paymentMethod | \n",
- " paymentMethodAgeDays | \n",
- " label | \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " | 12221 | \n",
- " 1045 | \n",
- " 1 | \n",
- " 4.505662 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 36961 | \n",
- " 1 | \n",
- " 2 | \n",
- " 3.575983 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 1 | \n",
- " \n",
- " \n",
- " | 12589 | \n",
- " 1961 | \n",
- " 1 | \n",
- " 4.921349 | \n",
- " creditcard | \n",
- " 186.035417 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 2526 | \n",
- " 579 | \n",
- " 1 | \n",
- " 4.895263 | \n",
- " paypal | \n",
- " 34.721528 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 20964 | \n",
- " 2000 | \n",
- " 1 | \n",
- " 4.962055 | \n",
- " creditcard | \n",
- " 128.497222 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 11717 | \n",
- " 1976 | \n",
- " 1 | \n",
- " 4.745402 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 13600 | \n",
- " 1644 | \n",
- " 1 | \n",
- " 4.962055 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 8375 | \n",
- " 2000 | \n",
- " 1 | \n",
- " 4.876771 | \n",
- " paypal | \n",
- " 375.588889 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 29591 | \n",
- " 45 | \n",
- " 1 | \n",
- " 4.965339 | \n",
- " creditcard | \n",
- " 44.139583 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 34313 | \n",
- " 1209 | \n",
- " 2 | \n",
- " 4.524580 | \n",
- " creditcard | \n",
- " 0.020833 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 25712 | \n",
- " 2 | \n",
- " 1 | \n",
- " 4.962055 | \n",
- " paypal | \n",
- " 0.001389 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 37196 | \n",
- " 31 | \n",
- " 2 | \n",
- " 4.886641 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 32667 | \n",
- " 512 | \n",
- " 1 | \n",
- " 4.524580 | \n",
- " paypal | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 36154 | \n",
- " 1587 | \n",
- " 1 | \n",
- " 5.034622 | \n",
- " creditcard | \n",
- " 0.002083 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 30017 | \n",
- " 2 | \n",
- " 1 | \n",
- " 4.836982 | \n",
- " creditcard | \n",
- " 0.008333 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 1105 | \n",
- " 26 | \n",
- " 1 | \n",
- " 4.524580 | \n",
- " creditcard | \n",
- " 25.702083 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 4459 | \n",
- " 514 | \n",
- " 1 | \n",
- " 4.057414 | \n",
- " paypal | \n",
- " 1.040278 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 29969 | \n",
- " 1097 | \n",
- " 1 | \n",
- " 4.965339 | \n",
- " paypal | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 25584 | \n",
- " 1526 | \n",
- " 1 | \n",
- " 4.057414 | \n",
- " paypal | \n",
- " 174.113889 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 16879 | \n",
- " 2000 | \n",
- " 1 | \n",
- " 5.017904 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 29092 | \n",
- " 178 | \n",
- " 1 | \n",
- " 3.483951 | \n",
- " creditcard | \n",
- " 97.879167 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 4479 | \n",
- " 926 | \n",
- " 1 | \n",
- " 4.505662 | \n",
- " paypal | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 38638 | \n",
- " 1268 | \n",
- " 1 | \n",
- " 4.876771 | \n",
- " paypal | \n",
- " 0.340972 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 22678 | \n",
- " 25 | \n",
- " 1 | \n",
- " 4.921349 | \n",
- " paypal | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 18864 | \n",
- " 2000 | \n",
- " 1 | \n",
- " 4.876771 | \n",
- " creditcard | \n",
- " 1163.151389 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 9340 | \n",
- " 5 | \n",
- " 1 | \n",
- " 4.505662 | \n",
- " creditcard | \n",
- " 0.002778 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 27379 | \n",
- " 2000 | \n",
- " 1 | \n",
- " 4.965339 | \n",
- " creditcard | \n",
- " 260.971528 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 14383 | \n",
- " 2000 | \n",
- " 2 | \n",
- " 4.524580 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 22274 | \n",
- " 4 | \n",
- " 1 | \n",
- " 4.921318 | \n",
- " creditcard | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 4071 | \n",
- " 2 | \n",
- " 1 | \n",
- " 4.836982 | \n",
- " creditcard | \n",
- " 1.967361 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " \n",
- " \n",
- "\n",
- " \n",
- " \n",
- " \n",
- " | \n",
- " accountAgeDays | \n",
- " numItems | \n",
- " localTime | \n",
- " paymentMethodAgeDays | \n",
- " label | \n",
- " paymentMethod_creditcard | \n",
- " paymentMethod_paypal | \n",
- " paymentMethod_storecredit | \n",
- " \n",
- " \n",
- " \n",
- " \n",
- " | 24205 | \n",
- " 2000 | \n",
- " 1 | \n",
- " 4.895263 | \n",
- " 753.620139 | \n",
- " 0 | \n",
- " 1 | \n",
- " 0 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 21090 | \n",
- " 3 | \n",
- " 1 | \n",
- " 4.962055 | \n",
- " 0.000000 | \n",
- " 0 | \n",
- " 1 | \n",
- " 0 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " | 36676 | \n",
- " 120 | \n",
- " 1 | \n",
- " 4.524580 | \n",
- " 119.510417 | \n",
- " 0 | \n",
- " 1 | \n",
- " 0 | \n",
- " 0 | \n",
- " \n",
- " \n",
- " \n",
- " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " accountAgeDays | \n",
+ " numItems | \n",
+ " localTime | \n",
+ " paymentMethod | \n",
+ " paymentMethodAgeDays | \n",
+ " label | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | 33182 | \n",
+ " 153 | \n",
+ " 1 | \n",
+ " 4.962055 | \n",
+ " paypal | \n",
+ " 152.063889 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 21764 | \n",
+ " 112 | \n",
+ " 1 | \n",
+ " 4.505662 | \n",
+ " creditcard | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 4871 | \n",
+ " 4 | \n",
+ " 1 | \n",
+ " 4.895263 | \n",
+ " creditcard | \n",
+ " 3.559722 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 14890 | \n",
+ " 836 | \n",
+ " 1 | \n",
+ " 5.040929 | \n",
+ " creditcard | \n",
+ " 721.078472 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 17382 | \n",
+ " 2000 | \n",
+ " 1 | \n",
+ " 4.886641 | \n",
+ " creditcard | \n",
+ " 1728.866667 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 16121 | \n",
+ " 2000 | \n",
+ " 1 | \n",
+ " 4.057414 | \n",
+ " creditcard | \n",
+ " 0.081250 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 33405 | \n",
+ " 154 | \n",
+ " 1 | \n",
+ " 4.921318 | \n",
+ " creditcard | \n",
+ " 152.000000 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 12112 | \n",
+ " 1859 | \n",
+ " 1 | \n",
+ " 4.921318 | \n",
+ " creditcard | \n",
+ " 0.011806 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 32347 | \n",
+ " 8 | \n",
+ " 1 | \n",
+ " 4.748314 | \n",
+ " paypal | \n",
+ " 7.636806 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 14898 | \n",
+ " 2000 | \n",
+ " 1 | \n",
+ " 4.836982 | \n",
+ " creditcard | \n",
+ " 0.000694 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 22285 | \n",
+ " 22 | \n",
+ " 1 | \n",
+ " 4.921318 | \n",
+ " paypal | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 22754 | \n",
+ " 193 | \n",
+ " 1 | \n",
+ " 4.057414 | \n",
+ " creditcard | \n",
+ " 0.002778 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 3500 | \n",
+ " 2 | \n",
+ " 1 | \n",
+ " 4.895263 | \n",
+ " creditcard | \n",
+ " 0.000694 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 21487 | \n",
+ " 231 | \n",
+ " 1 | \n",
+ " 5.040929 | \n",
+ " creditcard | \n",
+ " 0.001389 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 14531 | \n",
+ " 2000 | \n",
+ " 1 | \n",
+ " 4.921349 | \n",
+ " creditcard | \n",
+ " 0.050694 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 37042 | \n",
+ " 114 | \n",
+ " 1 | \n",
+ " 4.962055 | \n",
+ " creditcard | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 2682 | \n",
+ " 22 | \n",
+ " 1 | \n",
+ " 4.745402 | \n",
+ " creditcard | \n",
+ " 21.052083 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 29150 | \n",
+ " 1010 | \n",
+ " 1 | \n",
+ " 4.921318 | \n",
+ " creditcard | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 7574 | \n",
+ " 46 | \n",
+ " 1 | \n",
+ " 4.524580 | \n",
+ " creditcard | \n",
+ " 45.248611 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 14732 | \n",
+ " 8 | \n",
+ " 1 | \n",
+ " 4.745402 | \n",
+ " creditcard | \n",
+ " 0.650000 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " | \n",
+ " accountAgeDays | \n",
+ " numItems | \n",
+ " localTime | \n",
+ " paymentMethodAgeDays | \n",
+ " label | \n",
+ " paymentMethod_creditcard | \n",
+ " paymentMethod_paypal | \n",
+ " paymentMethod_storecredit | \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " | 31752 | \n",
+ " 1170 | \n",
+ " 1 | \n",
+ " 4.836982 | \n",
+ " 0.002083 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 361 | \n",
+ " 2000 | \n",
+ " 1 | \n",
+ " 5.034622 | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 28381 | \n",
+ " 1303 | \n",
+ " 1 | \n",
+ " 4.742303 | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 25580 | \n",
+ " 216 | \n",
+ " 1 | \n",
+ " 4.748314 | \n",
+ " 0.000000 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " | 16312 | \n",
+ " 21 | \n",
+ " 1 | \n",
+ " 4.745402 | \n",
+ " 20.944444 | \n",
+ " 0 | \n",
+ " 0 | \n",
+ " 1 | \n",
+ " 0 | \n",
+ " \n",
+ " \n",
+ " \n",
+ " |