from_tira.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import requests\n",
    "from string import punctuation \n",
    "import json\n",
    "import pickle\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.stem import WordNetLemmatizer \n",
    "from nltk.corpus import stopwords\n",
    "import nltk\n",
    "#nltk.download('averaged_perceptron_tagger')\n",
    "from nltk import pos_tag\n",
    "from nltk.corpus import wordnet\n",
    "import re\n",
    "import pandas as pd\n",
    "import sys\n",
    "import os\n",
    "from zipfile import ZipFile\n",
    "from boilerpy3 import extractors\n",
    "from urllib.error import HTTPError\n",
    "from urllib.error import URLError\n",
    "from urllib3.exceptions import MaxRetryError\n",
    "from urllib3.exceptions import NewConnectionError\n",
    "from requests.exceptions import ConnectionError\n",
    "from elasticsearch import Elasticsearch\n",
    "from elasticsearch import NotFoundError\n",
    "import time\n",
    "\n",
    "#input_dir = 'data_final'\n",
    "#output_dir = 'results_final'\n",
    "\n",
    "input_dir = sys.argv[1]\n",
    "output_dir = sys.argv[2]\n",
    "\n",
    "\n",
    "# In[2]:\n",
    "\n",
    "\n",
    "#parse the XML-file with queries\n",
    "\n",
    "mytree = ET.parse(input_dir+'/topics.xml')\n",
    "myroot = mytree.getroot()\n",
    "\n",
    "#preprocess the queries\n",
    "q = []\n",
    "topics = []\n",
    "for item in myroot:\n",
    "    d = {}\n",
    "    for x in item:\n",
    "        d[x.tag] = x.text.strip('\\n')\n",
    "        print(d)\n",
    "\n",
    "        if x.tag == \"title\":\n",
    "            #append the query to an array\n",
    "            q.append(x.text)\n",
    "    topics.append(d)\n",
    "print(\"q[0]:\")\n",
    "print(q[0])    \n",
    "#save topics as json\n",
    "#with open(input_dir+'/topics_final.json', 'w') as file:\n",
    "#    json.dump(topics, file)\n",
    "results = []\n",
    "chatnoir = \"https://www.chatnoir.eu/api/v1/_search\"\n",
    "attr = {\"apikey\": \"7dd15626-53aa-46c6-bd34-b2feaa2d9d81\",\n",
    "        \"query\": \"hello world\",\n",
    "        \"index\": \"cw12\",\n",
    "        \"pretty\": True\n",
    "}\n",
    "\n",
    "for x in q:\n",
    "    attr[\"query\"] = x\n",
    "    #somehow index doesn't work correctly when passed as an array (it only searches the 1st index of the array), so search\n",
    "    #in each index separately and sum up the results\n",
    "    while True:\n",
    "        try:\n",
    "             response = requests.post(chatnoir, data = attr)\n",
    "             print(response)\n",
    "             res = response.json()[\"meta\"][\"total_results\"]\n",
    "        except KeyError:\n",
    "            print(\"trying collecting results number for each query: KeyError\")\n",
    "            continue\n",
    "        break\n",
    "    results.append(res)\n",
    "    \n",
    "#save results as txt\n",
    "\n",
    "#with open(input_dir+\"/results_final.txt\", \"wb\") as fp:   #Pickling\n",
    "#    pickle.dump(results, fp)\n",
    "#RETRIEVAL\n",
    "#open the file with topics\n",
    "#f = open(input_dir+\"/topics_final.json\", encoding='utf8')\n",
    "#topics = json.load(f)\n",
    "\n",
    "#open the file with amount of results for each topic\n",
    "#with open(input_dir+\"/results_final.txt\", \"rb\") as fp:   # Unpickling\n",
    "#    results = pickle.load(fp)\n",
    "\n",
    "\n",
    " \n",
    "#preprocess the queries\n",
    "for i in range(len(topics)):\n",
    "    topics[i]['title'] = topics[i]['title'].replace(' ', ' AND ')\n",
    "    topics[i]['title'] = re.sub('[?:,]', '', topics[i]['title'])\n",
    "    topics[i]['results'] = results[i]\n",
    "    \n",
    "attr = {\"apikey\": \"7dd15626-53aa-46c6-bd34-b2feaa2d9d81\",\n",
    "        \"query\": \"\",\n",
    "        \"index\": \"cw12\",\n",
    "        \"size\": 10\n",
    "       }\n",
    "#save first 110 docs for each topic\n",
    "extractor = extractors.ArticleExtractor()\n",
    "def topics_iter(q):\n",
    "    docs = []\n",
    "    attr[\"query\"] = q['title']\n",
    "    #attr['size'] = q['results']\n",
    "    if q['results']<=110:\n",
    "        num_of_res = q['results']\n",
    "    else:\n",
    "        num_of_res = 110\n",
    "    count = 0\n",
    "    print(attr)\n",
    "    print(num_of_res)\n",
    "    url = \"https://www.chatnoir.eu/api/v1/_search?\"\n",
    "    while count < num_of_res:\n",
    "        attr[\"from\"] = count\n",
    "        err = 0\n",
    "        while True:\n",
    "            try:\n",
    "                r = requests.post(url, json = attr)\n",
    "                res = r.json()\n",
    "                print(count)\n",
    "                #print(res)\n",
    "                res_len = len(res['results'])\n",
    "                print(res_len)\n",
    "            except KeyError:\n",
    "                print(\"trying load sepr of ChatNoir: KeyError\")\n",
    "                continue\n",
    "            except NewConnectionError:\n",
    "                    print(\"NewConnectionError\")\n",
    "                    if err<10:\n",
    "                        err+=1\n",
    "                        time.sleep(60)\n",
    "                        continue\n",
    "                    else:\n",
    "                        break\n",
    "            break\n",
    "\n",
    "        for i in range(res_len):\n",
    "\n",
    "            doc_url = \"https://www.chatnoir.eu/cache?uuid=\"+res['results'][i]['uuid']+\"&index=cw12&raw\"\n",
    "            #print(doc_url)\n",
    "            err = 0\n",
    "            while True:\n",
    "                try:\n",
    "                    doc = extractor.get_doc_from_url(doc_url)\n",
    "                    content = doc.content\n",
    "                    title = doc.title\n",
    "                    res['results'][i]['document'] = content\n",
    "                    err=0\n",
    "                except URLError:\n",
    "                    print(\"trying to read the doc: URLError\")\n",
    "                    print(doc_url)\n",
    "                    if err<10:\n",
    "                        err+=1\n",
    "                        time.sleep(10)\n",
    "                        continue\n",
    "                    else:\n",
    "                        break\n",
    "                except HTTPError:\n",
    "                    print(\"HTTPError\")\n",
    "                    break\n",
    "\n",
    "                break\n",
    "        docs.append(res['results'])\n",
    "        count+=10\n",
    "\n",
    "    return docs\n",
    "# In[8]:\n",
    "input_dir = 'data_final'\n",
    "if not os.path.exists(input_dir+'/docs_final'):\n",
    "        os.mkdir(input_dir+'/docs_final')\n",
    "\n",
    "for q in topics:\n",
    "    q['documents'] = topics_iter(q)\n",
    "    with open(input_dir+\"/docs_final/docs_for_topic_{}.txt\".format(q['number']), \"w\") as f:\n",
    "        json.dump(q, f)\n",
    "\n",
    "    f.close()\n",
    "    for filename in os.listdir(input_dir+'/docs_final'):\n",
    "        print(\"Saved:\")\n",
    "        print(filename)\n",
    "\n",
    "\n",
    "\n",
    "# In[10]:\n",
    "\n",
    "\n",
    "#lemmatize topics\n",
    "#f = open(input_dir+'/topics_final.json')\n",
    "#topics = json.load(f)\n",
    "#f.close()\n",
    "lis =[]\n",
    "for i in range(len(topics)):\n",
    "    x=(topics[i]['title'])\n",
    "    lis.append(re.sub('[?:,]', '', x))\n",
    "converted_list = [x.lower() for x in lis]\n",
    "#print (\"Topics: \", converted_list)\n",
    "#print(\"\\n\")\n",
    "tokenized_sents = [word_tokenize(i) for i in converted_list]\n",
    "#for i in tokenized_sents:\n",
    "    #print (i)\n",
    "lis3 =[]\n",
    "\n",
    "for i in tokenized_sents:\n",
    "    tokens_without_sw = [word for word in i if not word in stopwords.words()]\n",
    "    lemmatizer = WordNetLemmatizer()\n",
    "    lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "    lemmatized_output_1 = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "    lis3.append(lemmatized_output_1)   \n",
    "#print(\"Lemmatized verbs and nouns: \\n\", lis3)\n",
    "#with open(input_dir+'/topics_final_lemmatized.txt', 'wb') as fp:\n",
    "#    pickle.dump(lis3, fp)\n",
    "#fp.close()\n",
    "\n",
    "\n",
    "# In[ ]:\n",
    "\n",
    "\n",
    "#create and save bulk data for index\n",
    "#input_dir = 'data/docs_final'\n",
    "#output_dir = 'res'\n",
    "def strip_punct(s):\n",
    "    s = re.sub('[^A-Za-z0-9]', ' ', s)\n",
    "    s = s.lower()\n",
    "    return \" \".join(s.split())\n",
    "count = 0\n",
    "c=0\n",
    "t = 1\n",
    "headers = {'accept': 'application/json', 'Content-Type': 'text/plain'}\n",
    "bulk_data = []\n",
    "#extract docs from zip-files\n",
    "for filename in os.listdir(input_dir+'/docs_final'):\n",
    "    if filename.endswith(\".zip\"):\n",
    "        with ZipFile((input_dir+\"/{}\").format(filename), 'r') as zip:\n",
    "            zip.extractall(input_dir)\n",
    "\n",
    "for filename in os.listdir(input_dir+'/docs_final'):\n",
    "    if filename.endswith(\".txt\"):\n",
    "        name = re.split('_|\\.', filename)\n",
    "        num = next(obj for obj in name if obj.isdigit())\n",
    "        with open((input_dir+\"/docs_final\"+\"/{}\").format(filename), \"r\") as f:\n",
    "            topic = json.load(f)\n",
    "            topic['title'] = strip_punct(topic['title']).lower()\n",
    "            print(filename)\n",
    "            print(topic['title'])\n",
    "            print(t)\n",
    "            t+=1\n",
    "            for n in topic[\"documents\"]:\n",
    "                print(len(n))\n",
    "                if count<10:\n",
    "                    count+=1\n",
    "                    for doc in n:\n",
    "                        try:\n",
    "                            doc_raw = doc['document'].rstrip('\\n')\n",
    "                            doc_raw = doc_raw.rstrip('\\\\n')\n",
    "\n",
    "                            doc['lem'] = word_tokenize(strip_punct(doc_raw).lower())\n",
    "                            tokens_without_sw = [word for word in doc['lem'] if not word in stopwords.words()]\n",
    "                            lemmatizer = WordNetLemmatizer()\n",
    "                            lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "                            doc['lem'] = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "                            doc['lem'] = doc['lem'].rstrip()\n",
    "\n",
    "                            title_lem = strip_punct(doc['title'].lower().rstrip())\n",
    "                            title_lem = word_tokenize(title_lem)\n",
    "                            tokens_without_sw = [word for word in title_lem if not word in stopwords.words()]\n",
    "                            lemmatizer = WordNetLemmatizer()\n",
    "                            lemmatized_output_0 = ([lemmatizer.lemmatize(w,pos=\"n\") for w in tokens_without_sw])\n",
    "                            title_lem = ' '.join(([lemmatizer.lemmatize(w,pos=\"v\") for w in lemmatized_output_0]))\n",
    "\n",
    "\n",
    "                            b = {\n",
    "                                    'query': topic['title'],\n",
    "                                    'title': doc['title'],\n",
    "                                    'title_lem': title_lem,\n",
    "                                    'num': num,\n",
    "                                    'uuid': doc['uuid'],\n",
    "                                    'score': doc['score'],\n",
    "                                    'document': doc['document'],\n",
    "                                    'lem': doc['lem']\n",
    "                                }\n",
    "\n",
    "                            templ = {'index': {'_index': 'test_index', \n",
    "                                           '_type': 'doc', \n",
    "                                           '_id': doc['trec_id']}}\n",
    "                            bulk_data.append(templ)\n",
    "                            bulk_data.append(b)\n",
    "\n",
    "                            c+=1\n",
    "                        except KeyError:\n",
    "                            pass\n",
    "                else:\n",
    "                    break\n",
    "            count=0\n",
    "\n",
    "#with open(input_dir+\"/bulk_data_final.json\", \"w\") as f:\n",
    "#    json.dump(bulk_data,f)\n",
    "\n",
    "# In[7]:\n",
    "\n",
    "\n",
    "#create synonyms\n",
    "#with open(input_dir+'/topics_final_lemmatized.txt', 'rb') as f:\n",
    "#    queries_lem = pickle.load(f)\n",
    "#f.close()\n",
    "df = pd.DataFrame(columns = ['query', 'syn'])\n",
    "df['query'] = lis3\n",
    "#print(df)\n",
    "#df.to_csv(input_dir+'/q_for_syn_final.tsv', sep = '\\t')\n",
    "\n",
    "#define words for synonyms in the .tsv file manually!\n",
    "\n",
    "\n",
    "# In[8]:\n",
    "\n",
    "\n",
    "#df = pd.read_csv(input_dir+'/q_for_syn_final.tsv', sep = '\\t')\n",
    "#df = df.fillna(0)\n",
    "\n",
    "for idx, row in df.iterrows():\n",
    "    if row['query']!=0:\n",
    "        tokenized = word_tokenize(row['query'])\n",
    "        pos_tagged = nltk.pos_tag(tokenized)\n",
    "        satz_synonyms = []\n",
    "        for wort in pos_tagged:\n",
    "            if  wort[1] != 'RB' and wort[1] != 'JJ' and wort[1] != 'JJS' and wort[1]!='RBR'and wort[1]!='RBS':\n",
    "                #print(wort[0])\n",
    "                for syn in wordnet.synsets(wort[0]):\n",
    "                    for l in syn.lemmas()[:1]:\n",
    "                        for n in l.name().split():\n",
    "                            if n not in satz_synonyms:\n",
    "                                satz_synonyms.append(n)\n",
    "        else:\n",
    "            satz_synonyms.append(wort[0])\n",
    "        s = ' '.join(str(v) for v in satz_synonyms)\n",
    "        s = ''.join(s)\n",
    "        s = s.split('_')\n",
    "        s = ' '.join(s)\n",
    "        #print(s)\n",
    "        df.at[idx,['syn']] = s\n",
    "#df.to_csv(input_dir+'/q_for_syn_final.tsv', sep = '\\t', index = False)\n",
    "\n",
    "\n",
    "# In[17]:\n",
    "#create index\n",
    "\n",
    "es = Elasticsearch()\n",
    "\n",
    "'''\n",
    "for filename in os.listdir(input_dir):\n",
    "    if filename.endswith(\".zip\"):\n",
    "        with ZipFile(filename, 'r') as zip:\n",
    "            zip.extractall()\n",
    "'''            \n",
    "def chunks(lst, n):\n",
    "    \"\"\"Yield successive n-sized chunks from lst.\"\"\"\n",
    "    for i in range(0, len(lst), n):\n",
    "        yield lst[i:i + n]\n",
    "\n",
    "def search_extended(search_param, filename, q, ind):\n",
    "    qid = []\n",
    "    Q0 = []\n",
    "    doc = []\n",
    "    rank = []\n",
    "    score = []\n",
    "    tag = []\n",
    "\n",
    "    num=51\n",
    "#['bool']['should']\n",
    "    for idx, row in q.iterrows():\n",
    "\n",
    "            #for key in x['match']:\n",
    "                #x['match'][key] = row['query']\n",
    "        for x in search_param['query']['bool']['should']:\n",
    "            for key in x[\"match\"]:\n",
    "                #print(key)\n",
    "                if \"boost\" in x['match'][key]:\n",
    "                    x['match'][key][\"query\"] = row['query']\n",
    "                else:\n",
    "                    x['match'][key] = row['syn']\n",
    "        #print(search_param)\n",
    "        response = es.search(index=ind, body=search_param)\n",
    "        r = 1\n",
    "        for x in response['hits']['hits']:\n",
    "            qid.append(num)\n",
    "            Q0.append('Q0')\n",
    "            doc.append(x['_id'])\n",
    "            rank.append(r)\n",
    "            score.append(x['_score'])\n",
    "            tag.append('uh-t2-thor')\n",
    "            r+=1\n",
    "        num+=1\n",
    "    qrels = {'qid': qid, 'Q0': Q0, 'doc': doc, 'rank':rank, 'score':score, 'tag':tag}\n",
    "\n",
    "    df = pd.DataFrame(qrels)\n",
    "    df.to_csv((output_dir+'/'+'run.txt'), sep = ' ', index = False, header = False)\n",
    "    \n",
    "    \n",
    "def create_index(b, k1, index):\n",
    "    \n",
    "    #create template for index\n",
    "    request_body = {\n",
    "    \"settings\" : {\n",
    "                \"number_of_shards\": 1,\n",
    "                \"number_of_replicas\": 0,\n",
    "        \"similarity\": {\n",
    "      \"default\": { \n",
    "        \"type\": \"BM25\",\n",
    "        \"b\":b,\n",
    "        \"k1\":k1\n",
    "      }\n",
    "    }\n",
    "            }\n",
    "        }\n",
    "\n",
    "    es.indices.create(index = index, body = request_body)\n",
    "    \n",
    "    \n",
    "    #load data to index\n",
    "    #with open(input_dir+'/bulk_data_final.json') as f:\n",
    "    #    bulk_data = json.load(f)\n",
    "        \n",
    "    for x in bulk_data[::2]:\n",
    "        x['index']['_index']=index\n",
    "        \n",
    "    bulks = chunks(bulk_data, 100)\n",
    "    for x in bulks:\n",
    "        res = es.bulk(index = index, body = x)\n",
    "    \n",
    "def create_run(d, run_name, index, mode):\n",
    "    #d, run_name, index\n",
    "    \n",
    "    if mode=='simple':\n",
    "        #f = open(input_dir+\"/topics_final_lemmatized.txt\", 'rb')\n",
    "        #queries_lem = pickle.load(f)\n",
    "        #f.close()\n",
    "        queries_lem = lis3\n",
    "        search_param = {\n",
    "    \n",
    "        'size': 50,\n",
    "        \"query\": {\n",
    "        \"bool\": {\n",
    "        \"should\": d\n",
    "        }\n",
    "        }\n",
    "        }\n",
    "        #print(search_param)\n",
    "\n",
    "        search(search_param, run_name, queries_lem, index)\n",
    "    elif mode=='syn':\n",
    "        #df_syn = pd.read_csv(input_dir+'/q_for_syn.tsv', sep = '\\t')\n",
    "        search_param = {\n",
    "    \n",
    "        'size': 50,\n",
    "        \"query\": {\n",
    "        \"bool\": {\n",
    "        \"should\": d\n",
    "        }\n",
    "        }\n",
    "        }\n",
    "        #print(search_param)\n",
    "\n",
    "        search_extended(search_param, run_name, df, index)\n",
    "    else:\n",
    "        print('Wrong arguments!')\n",
    "        \n",
    "index = 'final_0.68'\n",
    "create_index(0.68, 1.2, index)\n",
    "\n",
    "#create run for doc+title with synonyms\n",
    "d=[]\n",
    "d.append({\"match\": {\"title_lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"lem\": {\n",
    "                        \"query\":\"\",\n",
    "                    \"boost\":5}}})\n",
    "d.append({\"match\": {\"title_lem\": {\"\"}}})\n",
    "d.append({\"match\": {\"lem\": {\"\"}}})\n",
    "create_run(d, 'run_title_doc_syn', index, 'syn')\n",
    "\n",
    "\n",
    "# In[16]:\n",
    "\n",
    "\n",
    "es.indices.delete(index=index)\n",
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}