04_feature.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "da944f99-542c-4862-88e8-f4806bc7d7c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import warnings\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import StratifiedKFold,GroupKFold\n",
    "from sklearn.decomposition import TruncatedSVD\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "from tqdm import tqdm\n",
    "import gc\n",
    "import re\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import os\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer\n",
    "from gensim.models import Word2Vec\n",
    "import polars as pl\n",
    "from pathlib import Path\n",
    "from glob import glob\n",
    "import json\n",
    "import joblib\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8eb07835-818c-4228-9e4e-76bd4adb3ab8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_word2vec_feature(seq,emb,feat,ikx,ext='',prex = '',feature=[]):\n",
    "    sentence = [[str(x) for x in x] for x in seq]\n",
    "    if os.path.exists('w2v/w2v_model_{}_{}_{}.model'.format(prex,'_'.join(feat),ext)):\n",
    "        model = Word2Vec.load('w2v/w2v_model_{}_{}_{}.model'.format(prex,'_'.join(feat),ext))\n",
    "    else:\n",
    "        model = Word2Vec(sentence, vector_size=emb, window=5, min_count=1, workers=8, epochs=10, sg=1, seed=42)\n",
    "        model.save('w2v/w2v_model_{}_{}_{}.model'.format(prex,'_'.join(feat),ext))\n",
    "    return model\n",
    "\n",
    "def generate_w2v_feat(df,prex,col_name,dim):\n",
    "\n",
    "    feature = []\n",
    "    f = col_name\n",
    "    dim = dim\n",
    "\n",
    "    \n",
    "    res = df.groupby(prex)[f].apply(lambda x:list(x)).reset_index()\n",
    "    model = get_word2vec_feature(res[f].values,dim,[prex,f],f,ext='{}'.format(dim),prex = prex,feature=[])\n",
    "\n",
    "    i = 0\n",
    "    emb_matrix = []\n",
    "    for col in tqdm(res[f].values):\n",
    "        tmp = [model.wv[str(seq)] for seq in col]\n",
    "        tmp = np.mean(tmp,axis = 0)\n",
    "        emb_matrix.append(tmp)\n",
    "    emb_matrix = np.array(emb_matrix)\n",
    "\n",
    "    for i in range(dim):\n",
    "        res['{}_{}_{}'.format(prex,f + '_emb_mean',i)] = emb_matrix[:,i]\n",
    "        feature.append('{}_{}_{}'.format(prex,f + '_emb_mean',i))\n",
    "\n",
    "\n",
    "\n",
    "    joblib.dump(res[[prex,col_name] +feature ],'feats/w2v_{}_{}_emb.pkl'.format(prex,f))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2621243f-ef78-4be2-9936-1caf267155db",
   "metadata": {},
   "outputs": [],
   "source": [
    "def tfidf_char(input_values, output_num, output_prefix, seed=1024):\n",
    "    tfidf_enc = TfidfVectorizer(ngram_range=(1, 4), analyzer=\"char_wb\")\n",
    "    #tfidf_enc = TfidfVectorizer(ngram_range=(1, 2))\n",
    "    #tfidf_enc = TfidfVectorizer()\n",
    "\n",
    "    tfidf_vec = tfidf_enc.fit_transform(input_values)\n",
    "    svd_tmp = TruncatedSVD(n_components=output_num, n_iter=20, random_state=seed)\n",
    "    svd_tmp = svd_tmp.fit_transform(tfidf_vec)\n",
    "    svd_tmp = pd.DataFrame(svd_tmp)\n",
    "    svd_tmp.columns = ['{}_tfidf_char_{}'.format(output_prefix, i) for i in range(output_num)]\n",
    "    return svd_tmp\n",
    "\n",
    "def tfidf_word(input_values, output_num, output_prefix, seed=1024):\n",
    "    tfidf_enc = TfidfVectorizer(ngram_range=(1, 4),sublinear_tf = True)\n",
    "    #tfidf_enc = TfidfVectorizer(ngram_range=(1, 2))\n",
    "    #tfidf_enc = TfidfVectorizer()\n",
    "\n",
    "    tfidf_vec = tfidf_enc.fit_transform(input_values)\n",
    "    svd_tmp = TruncatedSVD(n_components=output_num, n_iter=20, random_state=seed)\n",
    "    svd_tmp = svd_tmp.fit_transform(tfidf_vec)\n",
    "    svd_tmp = pd.DataFrame(svd_tmp)\n",
    "    svd_tmp.columns = ['{}_tfidf_word_{}'.format(output_prefix, i) for i in range(output_num)]\n",
    "    return svd_tmp\n",
    "\n",
    "def count2vec(input_values, output_num, output_prefix, seed=1024):\n",
    "    #count_enc = CountVectorizer(ngram_range=(1, 3), analyzer=\"char_wb\")\n",
    "    count_enc = CountVectorizer(ngram_range=(1, 4))\n",
    "\n",
    "    count_vec = count_enc.fit_transform(input_values)\n",
    "    svd_tmp = TruncatedSVD(n_components=output_num, n_iter=20, random_state=seed)\n",
    "    svd_tmp = svd_tmp.fit_transform(count_vec)\n",
    "    svd_tmp = pd.DataFrame(svd_tmp)\n",
    "    svd_tmp.columns = ['{}_countvec_{}'.format(output_prefix, i) for i in range(output_num)]\n",
    "    return svd_tmp\n",
    "\n",
    "\n",
    "def  get_tfidf(tmp,group_id, group_target, num):\n",
    "    #tmp[group_target] = tmp[group_target].apply(lambda x: ' '.join(x))\n",
    "    tfidf_tmp1 = tfidf_word(tmp[group_target], num, group_target)\n",
    "    #tfidf_tmp2 = tfidf_char(tmp[group_target], num, group_target)\n",
    "\n",
    "    count_tmp = count2vec(tmp[group_target], num, group_target)\n",
    "    return pd.concat([tmp[group_id], tfidf_tmp1,count_tmp], axis=1)\n",
    "    #return pd.concat([tmp[group_id], tfidf_tmp], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "66509d81-fd35-40e2-9b36-549117eaf84e",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../IND-WhoIsWho/pid_to_info_all.json', 'r') as file:\n",
    "    pid = json.load(file)\n",
    "train = pd.read_feather('data/train.feather')\n",
    "valid = pd.read_feather('data/valid.feather')\n",
    "test = pd.read_feather('data/test.feather')\n",
    "\n",
    "piddf = joblib.load('data/pid_df.pkl')\n",
    "\n",
    "data = pd.concat([train,valid,test]).reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b6c06770-30d5-47a4-a9f6-1f26202434c2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def text_clean(x,mode = 1):\n",
    "    x = x.lower()\n",
    "    for f in list('-?=～—/_？）￥:#\\\\\\'.》”^>$]}|+)、（&{`《,(%!“<’\"】；【‘~*@…：，。[;') :\n",
    "        x = x.replace(f,'')\n",
    "        \n",
    "    if mode == 'venue':\n",
    "        number_pattern = r'\\d+'\n",
    "        x = re.sub(number_pattern, '', x)\n",
    "    \n",
    "    \n",
    "    for i in range(3):\n",
    "        x = x.replace('  ',' ')\n",
    "    return x\n",
    "piddf['title'] = piddf['title'].apply(text_clean)\n",
    "piddf['abstract'] = piddf['abstract'].apply(text_clean)\n",
    "\n",
    "piddf['venue'] = piddf['venue'].fillna('')\n",
    "piddf['venue'] = piddf['venue'].apply(lambda x:text_clean(x,mode = 'venue'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "aa91d164-7639-4d97-baca-db3fb57c91ec",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "title\n"
     ]
    }
   ],
   "source": [
    "piddf = piddf.reset_index(drop = True)\n",
    "piddf['title'] = piddf['title'].apply(lambda x:x.lower())\n",
    "piddf['index'] = piddf.index + 1\n",
    "\n",
    "for f in [\n",
    "       'title']:\n",
    "    print(f)\n",
    "    piddf[f] = piddf[f].fillna('')\n",
    "    #data[feat + '_v_len'] = data[feat].apply(lambda x:x.count(' '))\n",
    "    tmp = piddf[['index',f]]\n",
    "\n",
    "    tfidf_df = get_tfidf(tmp, ['index'], f, 32)\n",
    "    for f in tfidf_df.columns[1:]:\n",
    "        piddf[f] = tfidf_df[f]\n",
    "\n",
    "    del tmp,tfidf_df\n",
    "temp = piddf[['id']  +list(piddf.columns[8:])]\n",
    "temp.columns = ['PID'] +  list(temp.columns[1:])\n",
    "temp.to_feather('feats/title_tfidf2vec_feat.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "d7635fb1-cb83-4b45-be78-7fe35312118f",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:01<00:00,  9.20it/s]\n"
     ]
    }
   ],
   "source": [
    "pid_title_dict = dict(zip(temp['PID'],temp[temp.columns[1:]].values))\n",
    "mean_vec = temp[temp.columns[1:]].values.mean(axis = 0)\n",
    "\n",
    "def cos_similarity(target, embedding):\n",
    "    numerator = np.sum(target * embedding, axis=1)\n",
    "    denominator = np.sqrt(np.sum(np.square(target)) * np.sum(np.square(embedding),axis=1))\n",
    "    return numerator / denominator\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    for f1 in pid_list:\n",
    "        x1 = pid_title_dict.get(f1,mean_vec)\n",
    "        x2 = [f for f in pid_list if f != f1]\n",
    "        x2 = [pid_title_dict.get(f,mean_vec) for f in x2]\n",
    "        ans.append(list(cos_similarity(x1, x2)))\n",
    "data['pid_title_sim'] = ans\n",
    "\n",
    "\n",
    "for f in ['pid_title_sim']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x))\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x))\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x))\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x))\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x))\n",
    "data = data.drop(['autherName'],axis = 1)\n",
    "\n",
    "data.to_feather('feats/title_tfidf_sim.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "07707a0d-437c-4a5d-a7fc-ccc3f91bda01",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "b1ee08cd-5e91-48eb-803a-63ca22f2af6c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "abstract\n"
     ]
    }
   ],
   "source": [
    "piddf = piddf.reset_index(drop = True)\n",
    "piddf['abstract'] = piddf['abstract'].apply(lambda x:x.lower())\n",
    "piddf['index'] = piddf.index + 1\n",
    "\n",
    "for f in [\n",
    "       'abstract']:\n",
    "    print(f)\n",
    "    piddf[f] = piddf[f].fillna('')\n",
    "    #data[feat + '_v_len'] = data[feat].apply(lambda x:x.count(' '))\n",
    "    tmp = piddf[['index',f]]\n",
    "\n",
    "    tfidf_df = get_tfidf(tmp, ['index'], f, 32)\n",
    "    for f in tfidf_df.columns[1:]:\n",
    "        piddf[f] = tfidf_df[f]\n",
    "\n",
    "    del tmp,tfidf_df\n",
    "    \n",
    "\n",
    "\n",
    "temp = piddf[['id']  +[f for f in piddf.columns if 'abstract_' in f]]\n",
    "temp.columns = ['PID'] +  list(temp.columns[1:])\n",
    "temp.to_feather('feats/abstract_abstract2vec_feat.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "097ce9ae-1ba7-45e5-86f5-48e368256cc7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:01<00:00,  7.93it/s]\n"
     ]
    }
   ],
   "source": [
    "pid_abstract_dict = dict(zip(temp['PID'],temp[temp.columns[1:]].values))\n",
    "mean_vec = temp[temp.columns[1:]].values.mean(axis = 0)\n",
    "\n",
    "def cos_similarity(target, embedding):\n",
    "    numerator = np.sum(target * embedding, axis=1)\n",
    "    denominator = np.sqrt(np.sum(np.square(target)) * np.sum(np.square(embedding),axis=1))\n",
    "    return numerator / denominator\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    \n",
    "    for f1 in pid_list:\n",
    "        x1 = pid_abstract_dict.get(f1,mean_vec)\n",
    "        x2 = [f for f in pid_list if f != f1]\n",
    "        x2 = [pid_abstract_dict.get(f,mean_vec) for f in x2]\n",
    "        ans.append([f for f in list(cos_similarity(x1, x2)) if f >-10])\n",
    "\n",
    "data['pid_abstract_sim'] = ans\n",
    "\n",
    "#data = data.drop([ 'pid_title_sim', 'pid_title_sim_mean',\n",
    "#       'pid_title_sim_max', 'pid_title_sim_min', 'pid_title_sim_std',\n",
    "#       'pid_title_sim_median',],axis = 1)\n",
    "\n",
    "\n",
    "for f in ['pid_abstract_sim']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x) if len(x) > 0 else np.nan)\n",
    "\n",
    "\n",
    "data.to_feather('feats/abstract_tfidf_sim.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "08510f9d-f34f-46a5-a3e3-16df85731c46",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "123"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "76c5925e-e6c1-431f-9da4-d6b9a243d1b5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "keyword_text\n"
     ]
    }
   ],
   "source": [
    "piddf['keyword_text'] = piddf['keywords'].apply(lambda x:(' '.join(x)).lower())\n",
    "piddf['keyword_text'] = piddf['keyword_text'].fillna('')\n",
    "piddf = piddf.reset_index(drop = True)\n",
    "piddf['index'] = piddf.index + 1\n",
    "\n",
    "for f in [\n",
    "       'keyword_text']:\n",
    "    print(f)\n",
    "    piddf[f] = piddf[f].fillna('')\n",
    "    #data[feat + '_v_len'] = data[feat].apply(lambda x:x.count(' '))\n",
    "    tmp = piddf[['index',f]]\n",
    "\n",
    "    tfidf_df = get_tfidf(tmp, ['index'], f, 16)\n",
    "    for f in tfidf_df.columns[1:]:\n",
    "        piddf[f] = tfidf_df[f]\n",
    "\n",
    "    del tmp,tfidf_df\n",
    "    \n",
    "\n",
    "\n",
    "temp = piddf[['id']  +[f for f in piddf.columns if 'keyword_text_' in f]]\n",
    "temp.columns = ['PID'] +  list(temp.columns[1:])\n",
    "temp.to_feather('feats/keyword_text_keyword2vec_feat.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d4712592-6f74-422a-a9e8-31e0b3050056",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "11a9d44a-bf76-4468-a756-1caa0cab2d5c",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:01<00:00,  8.99it/s]\n"
     ]
    }
   ],
   "source": [
    "pid_keyword_text_dict = dict(zip(temp['PID'],temp[temp.columns[1:]].values))\n",
    "mean_vec = temp[temp.columns[1:]].values.mean(axis = 0)\n",
    "\n",
    "def cos_similarity(target, embedding):\n",
    "    numerator = np.sum(target * embedding, axis=1)\n",
    "    denominator = np.sqrt(np.sum(np.square(target)) * np.sum(np.square(embedding),axis=1))\n",
    "    return numerator / denominator\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    \n",
    "    for f1 in pid_list:\n",
    "        x1 = pid_keyword_text_dict.get(f1,mean_vec)\n",
    "        x2 = [f for f in pid_list if f != f1]\n",
    "        x2 = [pid_keyword_text_dict.get(f,mean_vec) for f in x2]\n",
    "        ans.append([f for f in list(cos_similarity(x1, x2)) if f >-10])\n",
    "\n",
    "data['pid_keyword_text_sim'] = ans\n",
    "\n",
    "#data = data.drop([ 'pid_title_sim', 'pid_title_sim_mean',\n",
    "#       'pid_title_sim_max', 'pid_title_sim_min', 'pid_title_sim_std',\n",
    "#       'pid_title_sim_median',],axis = 1)\n",
    "\n",
    "\n",
    "for f in ['pid_keyword_text_sim']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x) if len(x) > 0 else np.nan)\n",
    "\n",
    "\n",
    "data.to_feather('feats/keyword_text_tfidf_sim.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "17cf6287-37c5-4822-bd0c-1098141c3104",
   "metadata": {},
   "outputs": [],
   "source": [
    "#123456789"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e3e7d68f-a71f-45ce-a7c4-bf0192e0d84b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "venue\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:01<00:00,  8.47it/s]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "piddf = piddf.reset_index(drop = True)\n",
    "piddf['venue'] = piddf['venue'].fillna('')\n",
    "piddf['venue'] = piddf['venue'].apply(lambda x:x.lower())\n",
    "piddf['index'] = piddf.index + 1\n",
    "\n",
    "for f in [\n",
    "       'venue']:\n",
    "    print(f)\n",
    "    piddf[f] = piddf[f].fillna('')\n",
    "    #data[feat + '_v_len'] = data[feat].apply(lambda x:x.count(' '))\n",
    "    tmp = piddf[['index',f]]\n",
    "\n",
    "    tfidf_df = get_tfidf(tmp, ['index'], f, 16)\n",
    "    for f in tfidf_df.columns[1:]:\n",
    "        piddf[f] = tfidf_df[f]\n",
    "\n",
    "    del tmp,tfidf_df\n",
    "temp = piddf[['id']  +[f for f in piddf.columns if 'venue_' in f]]\n",
    "temp.columns = ['PID'] +  list(temp.columns[1:])\n",
    "temp.to_feather('feats/venue_venue2vec_feat.feather')\n",
    "\n",
    "pid_venue_dict = dict(zip(temp['PID'],temp[temp.columns[1:]].values))\n",
    "mean_vec = temp[temp.columns[1:]].values.mean(axis = 0)\n",
    "\n",
    "def cos_similarity(target, embedding):\n",
    "    numerator = np.sum(target * embedding, axis=1)\n",
    "    denominator = np.sqrt(np.sum(np.square(target)) * np.sum(np.square(embedding),axis=1))\n",
    "    return numerator / denominator\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    \n",
    "    for f1 in pid_list:\n",
    "        x1 = pid_venue_dict.get(f1,mean_vec)\n",
    "        x2 = [f for f in pid_list if f != f1]\n",
    "        x2 = [pid_venue_dict.get(f,mean_vec) for f in x2]\n",
    "        ans.append([f for f in list(cos_similarity(x1, x2)) if f >-10])\n",
    "\n",
    "data['pid_venue_sim'] = ans\n",
    "\n",
    "#data = data.drop([ 'autherName'],axis = 1)\n",
    "\n",
    "\n",
    "for f in ['pid_venue_sim']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x) if len(x) > 0 else np.nan)\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x) if len(x) > 0 else np.nan)\n",
    "\n",
    "\n",
    "data.to_feather('feats/venue_tfidf_sim.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49b568ec-9230-44e3-8fe9-7d6ef2217e56",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "5cab26b1-a568-4c55-8b48-ae3be7096bc9",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "piddf = joblib.load('data/pid_df.pkl')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "e5c13472-f8a0-4820-ab71-ea7e01d3f5c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2957/2957 [00:00<00:00, 3130.40it/s]\n",
      "100%|██████████| 15/15 [00:02<00:00,  5.23it/s]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "ans = []\n",
    "for pid,keyword in tqdm(piddf[['id','keywords']].values):\n",
    "    if len(keyword)>0 and type(keyword)== list:\n",
    "        for key in keyword:\n",
    "            ans.append([pid,key])\n",
    "df = pd.DataFrame(ans,columns = ['PID','keyword'])\n",
    "df['keyword'] = df['keyword'].apply(lambda x:x.lower())\n",
    "df = df.groupby('PID')['keyword'].agg(list).reset_index()\n",
    "pid_keyword_dict = dict(df.values)\n",
    "\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    for f1 in pid_list:\n",
    "        x1 = set(pid_keyword_dict.get(f1,[]))\n",
    "        temp = []\n",
    "        for f2 in pid_list:\n",
    "            if f1!= f2:\n",
    "                x2 = set(pid_keyword_dict.get(f2,[]))\n",
    "                temp.append([len(x1 & x2),len(x1| x2)])\n",
    "        ans.append(temp.copy())\n",
    "        \n",
    "data['pid_sim'] = ans\n",
    "data['pid_keyword_cnt_1'] = data['pid_sim'].apply(lambda x:[f[0] for f in x])\n",
    "data['pid_keyword_cnt_2'] = data['pid_sim'].apply(lambda x:[f[1] for f in x])\n",
    "data['pid_keyword_cnt_3'] = data['pid_sim'].apply(lambda x:[(f[0]+1) / (f[1] + 1) for f in x])\n",
    "\n",
    "\n",
    "for f in ['pid_keyword_cnt_1','pid_keyword_cnt_2','pid_keyword_cnt_3']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x))\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x))\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x))\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x))\n",
    "    data[f + '_median'] = data[f].apply(lambda x:np.median(x))\n",
    "\n",
    "data = data.drop(['autherName','pid_keyword_cnt_1','pid_keyword_cnt_2','pid_keyword_cnt_3'],axis = 1)\n",
    "data.to_feather('feats/keywords_jaccard_sim.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3799d429-1612-410f-a000-67963d93a33f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0019ec41-3cb9-478c-bee4-5fd779952554",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "7cb1184d-3e35-494c-9432-6008bb7d7a98",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2957/2957 [00:00<00:00, 224379.14it/s]\n"
     ]
    }
   ],
   "source": [
    "piddf = joblib.load('data/pid_df.pkl')\n",
    "\n",
    "ans = []\n",
    "for pid,auther in tqdm(piddf[['id','authors']].values):\n",
    "    if len(auther)>0 and type(auther)== list:\n",
    "        for key in auther:\n",
    "            ans.append([pid,key['name'],key['org']])\n",
    "df = pd.DataFrame(ans,columns = ['PID','autherName','author_org'])\n",
    "df['autherName'] = df['autherName'].apply(lambda x:x.lower())\n",
    "df['author_org'] = df['author_org'].apply(lambda x:x.lower())\n",
    "\n",
    "from pypinyin import pinyin, Style\n",
    "def chinese2ping_name(x):\n",
    "    x = pinyin(x,Style.NORMAL)\n",
    "    #print(x)\n",
    "    x1 = x[0][0]\n",
    "    x2 = ''.join([f[0] for f in x[1:]])\n",
    "    #print(x1,x2)\n",
    "    return x2 + ' ' + x1\n",
    "\n",
    "\n",
    "def is_contain_chinese(text):\n",
    "    # 定义匹配中文字符的正则表达式模式\n",
    "    chinese_pattern = re.compile(r'[\\u4e00-\\u9fa5]')\n",
    "    # 使用正则表达式模式搜索文本\n",
    "    match = chinese_pattern.search(text)\n",
    "    # 如果找到匹配的中文字符，则返回True，否则返回False\n",
    "    return match is not None\n",
    "\n",
    "\n",
    "df.loc[df['autherName'].apply(lambda x:'高阳' in x),'autherName'] = 'yang gao'\n",
    "df.loc[(df['autherName'].apply(lambda x:is_contain_chinese(x))),'autherName'] = df.loc[  (df['autherName'].apply(lambda x:is_contain_chinese(x))) ,'autherName'].map(chinese2ping_name)\n",
    "df['autherName'] = df['autherName'] = df['autherName'].apply(lambda x:x.replace('.',' ').replace('-',' '))\n",
    "df['revision'] = df['autherName'].apply(lambda x:' '.join(sorted([f for f in x.split(' ') if len(f)>0])))\n",
    "df = df.groupby('PID')['revision'].agg(list).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d13dd619-db2e-40fd-bb42-ae7ac82245ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15/15 [00:03<00:00,  4.51it/s]\n"
     ]
    }
   ],
   "source": [
    "pid_auther_dict = dict(df.values)\n",
    "\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    for f1 in pid_list:\n",
    "        x1 = set(pid_auther_dict.get(f1,[]))\n",
    "        temp = []\n",
    "        for f2 in pid_list:\n",
    "            if f1!= f2:\n",
    "                x2 = set(pid_auther_dict.get(f2,[]))\n",
    "                temp.append([len(x1 & x2),len(x1| x2)])\n",
    "        ans.append(temp.copy())\n",
    "        \n",
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "data['pid_sim'] = ans"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "fc35a1d6-d02b-472b-a7fe-2f690f416179",
   "metadata": {},
   "outputs": [],
   "source": [
    "data['pid_auther_cnt_1'] = data['pid_sim'].apply(lambda x:[f[0] for f in x])\n",
    "data['pid_auther_cnt_2'] = data['pid_sim'].apply(lambda x:[f[1] for f in x])\n",
    "data['pid_auther_cnt_3'] = data['pid_sim'].apply(lambda x:[(f[0]+1) / (f[1] + 1) for f in x])\n",
    "\n",
    "for f in ['pid_auther_cnt_1','pid_auther_cnt_2','pid_auther_cnt_3']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x))\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x))\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x))\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x))\n",
    "data = data.drop(['autherName','pid_auther_cnt_1','pid_auther_cnt_2','pid_auther_cnt_3'],axis = 1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "7a07c559-4394-40f6-b42d-5989c319bdf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data.to_feather('feats/autherName_jaccard_sim.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9de8cc25-9da5-4555-becf-cc66b1bf1171",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4667e9f3-04f4-42c2-ae91-20dafd12258b",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "2816fa65-359d-492b-a997-7c76aeef4b0d",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2957/2957 [00:00<00:00, 247368.40it/s]\n",
      "100%|██████████| 15/15 [00:03<00:00,  4.21it/s]\n"
     ]
    }
   ],
   "source": [
    "\n",
    "piddf = joblib.load('data/pid_df.pkl')\n",
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "\n",
    "ans = []\n",
    "for pid,auther in tqdm(piddf[['id','authors']].values):\n",
    "    if len(auther)>0 and type(auther)== list:\n",
    "        for key in auther:\n",
    "            ans.append([pid,key['name']])\n",
    "df = pd.DataFrame(ans,columns = ['PID','autherName'])\n",
    "df['autherName'] = df['autherName'].apply(lambda x:x.lower())\n",
    "df = df.groupby('PID')['autherName'].agg(list).reset_index()\n",
    "pid_auther_dict = dict(df.values)\n",
    "\n",
    "ans = []\n",
    "for autherID in tqdm(data['autherID'].unique()):\n",
    "    pid_list = data[data['autherID'] == autherID]['PID'].to_list()\n",
    "    for f1 in pid_list:\n",
    "        x1 = set(pid_auther_dict.get(f1,[]))\n",
    "        temp = []\n",
    "        for f2 in pid_list:\n",
    "            if f1!= f2:\n",
    "                x2 = set(pid_auther_dict.get(f2,[]))\n",
    "                temp.append([len(x1 & x2),len(x1| x2)])\n",
    "        ans.append(temp.copy())\n",
    "        \n",
    "data['pid_sim'] = ans\n",
    "data['pid_autherraw_cnt_1'] = data['pid_sim'].apply(lambda x:[f[0] for f in x])\n",
    "data['pid_autherraw_cnt_2'] = data['pid_sim'].apply(lambda x:[f[1] for f in x])\n",
    "data['pid_autherraw_cnt_3'] = data['pid_sim'].apply(lambda x:[(f[0]+1) / (f[1] + 1) for f in x])\n",
    "\n",
    "for f in ['pid_autherraw_cnt_1','pid_autherraw_cnt_2','pid_autherraw_cnt_3']:\n",
    "    data[f + '_mean'] = data[f].apply(lambda x:np.mean(x))\n",
    "    data[f + '_max'] = data[f].apply(lambda x:np.max(x))\n",
    "    data[f + '_min'] = data[f].apply(lambda x:np.min(x))\n",
    "    data[f + '_std'] = data[f].apply(lambda x:np.std(x))\n",
    "data = data.drop(['autherName','pid_autherraw_cnt_1','pid_autherraw_cnt_2','pid_autherraw_cnt_3'],axis = 1)\n",
    "data.to_feather('feats/autherName_raw_jaccard_sim.feather')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "0fc61d4b-abdf-4de5-8c97-1fff141aa68c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "123"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f185e51-8f24-4fa0-a6f7-681405fb7811",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7df69c4c-1bce-487c-817d-4f0be31419a6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8ed8ed13-f0fe-4660-9e0b-e3733baabd93",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_w2v_feature(data,col1,col2,emb_size,ext='',feature=[]):\n",
    "    print('begin train word2vec')\n",
    "    data = data[col1 +[col2]]\n",
    "    data[col2] = data[col2].astype(str)\n",
    "    tmp = data.groupby(col1)[col2].apply(lambda x:list(x)).reset_index()\n",
    "    sentences = tmp[col2].values.tolist()\n",
    "    print(tmp.head())\n",
    "    del tmp[col2]\n",
    "    if os.path.exists('w2v/{}_{}_feature{}.model'.format('_'.join(col1),col2,ext)):\n",
    "        model = Word2Vec.load('w2v/{}_{}_feature{}.model'.format('_'.join(col1),col2,ext))\n",
    "    else:\n",
    "        model = Word2Vec(sentences, vector_size=emb_size, window=5, min_count=1, workers=8, epochs=10, sg=1, seed=42)\n",
    "        model.save('w2v/{}_{}_feature{}.model'.format('_'.join(col1),col2,ext))\n",
    "    emb_matrix = []\n",
    "    emb_dict = {}\n",
    "    print('begin make feature')\n",
    "    for seq in sentences:\n",
    "        vec = []\n",
    "        for w in seq:\n",
    "            #print(w)\n",
    "            if w in model.wv:\n",
    "                vec.append(model.wv[w])\n",
    "                emb_dict[w] = model.wv[w]\n",
    "        if len(vec) > 0:\n",
    "            emb_matrix.append(np.mean(vec, axis=0))\n",
    "        else:\n",
    "            emb_matrix.append([0] * emb_size)\n",
    "    emb_matrix = np.array(emb_matrix)\n",
    "    for i in range(emb_size):\n",
    "        tmp['{}_{}_emb_{}{}'.format('_'.join(col1), col2, i, ext)] = emb_matrix[:, i]\n",
    "        feature.append('{}_{}_emb_{}{}'.format('_'.join(col1), col2, i,ext))\n",
    "    del model, emb_matrix, sentences\n",
    "    new_emb_martix = []\n",
    "    data_index = []\n",
    "    for v in emb_dict:\n",
    "        data_index.append(v)\n",
    "        tmp_emb = np.array(emb_dict[v])\n",
    "        new_emb_martix.append(tmp_emb)\n",
    "    new_emb_martix = np.array(new_emb_martix)\n",
    "    data = pd.DataFrame()\n",
    "    data[col2] = data_index\n",
    "    for i in range(emb_size):\n",
    "        data['{}_emb_{}_{}'.format(col2, i, ext)] = new_emb_martix[:,i]\n",
    "        feature.append('{}_emb_{}_{}'.format(col2, i, ext))\n",
    "    return tmp,feature,data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "c0cc42fa-4831-4b99-9e6d-054286e1bc10",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2957/2957 [00:00<00:00, 4953.01it/s]\n"
     ]
    }
   ],
   "source": [
    "with open('../IND-WhoIsWho/pid_to_info_all.json', 'r') as file:\n",
    "    pid = json.load(file)\n",
    "    \n",
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "data['keywords'] = data['PID'].apply(lambda x:pid[x]['keywords'])\n",
    "\n",
    "ans = []\n",
    "for pid,keyword in tqdm(data[['autherID','keywords']].values):\n",
    "    if len(keyword)>0 and type(keyword)== list:\n",
    "        for key in keyword:\n",
    "            ans.append([pid,key])\n",
    "df = pd.DataFrame(ans,columns = ['autherID','keyword'])\n",
    "df['keyword'] = df['keyword'].apply(lambda x:x.lower())\n",
    "#df = df.groupby('autherID')['keyword'].agg(list).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "7a9e8a95-2891-4593-98c2-73a6612b16d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "begin train word2vec\n",
      "   autherID                                            keyword\n",
      "0  9Gs8Wj3Y  [bulk density, infiltration characteristics of...\n",
      "1  C97iQ0Fj  [adults, brain neoplasms, magnetic resonance i...\n",
      "2  Fkb16wn7  [visual memory, spatial information, computer ...\n",
      "3  Iki037dt  [vesicoureteral reflux, bladder hypertrophy, c...\n",
      "4  KKiBE172  [traditional pid control, self-adjusting syste...\n",
      "begin make feature\n"
     ]
    }
   ],
   "source": [
    "emb_cols = [\n",
    "    ['autherID', 'keyword'],\n",
    "    \n",
    "    # ...\n",
    "]\n",
    "\n",
    "for f1,f2 in emb_cols:\n",
    "    total_feature_1,feature,total_feature_2 = get_w2v_feature(df,[f1],f2,16,ext='16',feature=[])\n",
    "    #total_feature_1 = reduce_mem(total_feature_1)\n",
    "    #total_feature_2 = reduce_mem(total_feature_2)\n",
    "    #total_feature_2[f2] = total_feature_2[f2].astype(int)\n",
    "    \n",
    "    data = pd.merge(data,total_feature_1,how='left',on=[f1],copy=False)\n",
    "    \n",
    "    ##df = pd.merge(df,total_feature_2,how='left',on=[f2],copy=False)\n",
    "total_feature_1.to_feather('feats/w2v_feats/autherID_keyword_w2v_emb.feather')\n",
    "t_dict = dict(zip(total_feature_2['keyword'].to_list(),total_feature_2[total_feature_2.columns[1:]].values))\n",
    "t_df = data[['autherID','PID','keywords']]\n",
    "t_df.loc[t_df['keywords'].apply(lambda x:len(x)) !=0,'keywords_emb'] = t_df.loc[t_df['keywords'].apply(lambda x:len(x)) !=0,'keywords'].apply(lambda x:np.mean([t_dict[f.lower()] for f in x],axis = 0))\n",
    "t_df.loc[t_df['keywords'].apply(lambda x:len(x)) !=0,[f'keywords_auterID_emb_mean_{i}' for i in range(16)]] = np.array(t_df.loc[t_df['keywords'].apply(lambda x:len(x)) !=0,'keywords_emb'].to_list())\n",
    "t_df[['autherID','PID'] + [f'keywords_auterID_emb_mean_{i}' for i in range(16)]].to_feather('feats/w2v_feats/keyword_autherID_w2v_emb_mean.feather')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "a579c39f-15e4-4826-b904-b1aafaf05e6a",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('../IND-WhoIsWho/pid_to_info_all.json', 'r') as file:\n",
    "    pid = json.load(file)\n",
    "    \n",
    "data = pd.concat([train,test]).reset_index(drop = True)\n",
    "data['venue'] = data['PID'].apply(lambda x:pid[x]['venue'])\n",
    "data['venue'] = data['venue'].fillna('')\n",
    "data['venue'] = data['venue'].apply(lambda x:text_clean(x))\n",
    "data['venue'] = data['venue'].apply(lambda x:x.lower())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "e8f5b48d-501d-47ee-81a4-c44775693869",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "begin train word2vec\n",
      "   autherID                                              venue\n",
      "0  9Gs8Wj3Y  [journal of soil and water conservation, journ...\n",
      "1  Fkb16wn7  [eccv 7, computer vision and pattern recogniti...\n",
      "2  Iki037dt  [annals of oncology, the journal of urology, u...\n",
      "3  KKiBE172  [proceedings of the national academy of scienc...\n",
      "4  WXMYBk3c  [chinese journal of clinicianselectronic editi...\n",
      "begin make feature\n"
     ]
    }
   ],
   "source": [
    "emb_cols = [\n",
    "    ['autherID', 'venue'],\n",
    "    \n",
    "    # ...\n",
    "]\n",
    "\n",
    "for f1,f2 in emb_cols:\n",
    "    total_feature_1,feature,total_feature_2 = get_w2v_feature(data,[f1],f2,8,ext='8',feature=[])\n",
    "\n",
    "total_feature_1.to_feather('feats/w2v_feats/autherID_venue_w2v_emb.feather')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "df087456-cdfc-4b83-a04f-8503d84b11a1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "123"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "123"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e1ff74a0-4fbd-4f87-bce9-578ffd47a04c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bc71d03f-fedd-4a7c-82f7-601c014e83d5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "019d6730-8ecc-4d2e-b17a-88566c29bc46",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 2957/2957 [00:00<00:00, 3881.77it/s]\n"
     ]
    }
   ],
   "source": [
    "data = pd.concat([train,valid,test]).reset_index(drop = True)\n",
    "\n",
    "tmp_df = piddf[['id','authors']]\n",
    "tmp_df['authors_org'] = tmp_df['authors'].apply(lambda x:[f['org'] for f in x])\n",
    "tmp_df['authors_name'] = tmp_df['authors'].apply(lambda x:[f['name'] for f in x])\n",
    "tmp_df.columns = ['PID','authors','authors_org','authors_name']\n",
    "tmp_df = data[['PID','autherID']].merge(tmp_df,on = 'PID',how = 'left')\n",
    "tmp_df['authors_org'] = tmp_df['authors_org'].apply(lambda x:[f.lower() for f in x ])\n",
    "tmp_df['authors_name'] = tmp_df['authors_name'].apply(lambda x:[f.lower() for f in x ])\n",
    "\n",
    "ans = []\n",
    "for auther,keyword,org,PID in tqdm(tmp_df[['autherID','authors_name','authors_org','PID']].values):\n",
    "    if len(keyword)>0 and type(keyword)== list:\n",
    "        for i in range(len(keyword)):\n",
    "            ans.append([auther,keyword[i],org[i],PID])\n",
    "    \n",
    "df = pd.DataFrame(ans,columns = ['autherID','authors_name','authors_org','PID'])\n",
    "df['auther_authors_name_count'] = df.groupby(['autherID','authors_name'])['PID'].transform('count')\n",
    "df['autherName'] = df['autherID'].map(dict(data[['autherID','autherName']].values))\n",
    "df['autherName'] = df['autherName'].apply(lambda x:x.lower())\n",
    "\n",
    "\n",
    "from pypinyin import pinyin, Style\n",
    "def chinese2ping_name(x):\n",
    "    x = pinyin(x,Style.NORMAL)\n",
    "    #print(x)\n",
    "    x1 = x[0][0]\n",
    "    x2 = ''.join([f[0] for f in x[1:]])\n",
    "    #print(x1,x2)\n",
    "    return x2 + ' ' + x1\n",
    "\n",
    "\n",
    "def is_contain_chinese(text):\n",
    "    # 定义匹配中文字符的正则表达式模式\n",
    "    chinese_pattern = re.compile(r'[\\u4e00-\\u9fa5]')\n",
    "    # 使用正则表达式模式搜索文本\n",
    "    match = chinese_pattern.search(text)\n",
    "    # 如果找到匹配的中文字符，则返回True，否则返回False\n",
    "    return match is not None\n",
    "\n",
    "\n",
    "df.loc[df['autherName'].apply(lambda x:'高阳' in x),'autherName'] = 'yang gao'\n",
    "#df.loc[(df['authors_name'].apply(lambda x:len(x) < 4) & (df['authors_name'].apply(lambda x:is_contain_chinese(x))) ),'authors_name'] = df[(df['authors_name'].apply(lambda x:len(x) < 4)) & (df['authors_name'].apply(lambda x:is_contain_chinese(x))),'authors_name'] = df.loc[ (df['authors_name'].apply(lambda x:len(x) < 4)) & (df['authors_name'].apply(lambda x:is_contain_chinese(x))) ,'authors_name'].map(chinese2ping_name)\n",
    "\n",
    "\n",
    "\n",
    "df['authors_name'] = df['authors_name'].apply(lambda x:x.replace('-','').replace(',',''))\n",
    "df['autherName'] = df['autherName'].apply(lambda x:x.replace('-','').replace(',',''))\n",
    "df['authors_name'] = df['authors_name'].apply(lambda x:x.replace('\\xa0', ' ') )\n",
    "df.loc[df['autherName'] == df['authors_name'],'flag'] = 1\n",
    "df.loc[(df['flag']!= 1) &(df['authors_name'].apply(lambda x:x[1] == ' ' if len(x)>2 else False)),'flag'] = df.loc[(df['flag']!= 1) &(df['authors_name'].apply(lambda x:x[1] == ' ' if len(x)>2 else False))].apply(lambda x:int(x['authors_name'][2:] in x['autherName']),axis = 1)\n",
    "df.loc[(df['flag']!= 1) &(df['authors_name'].apply(lambda x:x[1] == '.' if len(x)>2 else False)),'flag'] = df.loc[(df['flag']!= 1) &(df['authors_name'].apply(lambda x:x[1] == '.' if len(x)>2 else False))].apply(lambda x:int(x['authors_name'][2:] in x['autherName']),axis = 1)\n",
    "df.loc[(df['flag']!= 1) &(df['authors_name'].apply(lambda x:' '.join(x.split(' ')[1:]+ x.split(' ')[0:1])) == df['autherName']),'flag'] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "352a7d51-ef91-497e-af6e-a379f58e5bef",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from collections import Counter\n",
    "def swap_name_judge(x1,x2):\n",
    "    x1 = x1.replace('.','')\n",
    "    x2 = x2.replace('.','')\n",
    "    cnt = 0\n",
    "    cnt += sum(list((Counter(x1)-Counter(x2)).values()))\n",
    "    cnt += sum(list((Counter(x2)-Counter(x1)).values()))\n",
    "    \n",
    "    all_ = len(x1) + len(x2)\n",
    "    if all_>5 and cnt / all_<0.2:\n",
    "        return True\n",
    "    return False\n",
    "\n",
    "df.loc[(df['flag']!=1)&(df.apply(lambda x:swap_name_judge(x['authors_name'],x['autherName']),axis = 1)),'flag'] = 1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "6a4e280d-048a-401f-a360-4202f7e3dce9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_feather('temp_data/autherID_authors.feather')\n",
    "#df = pd.read_feather('temp_data/autherID_authors.feather')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ec637953-39bb-4def-8b4e-7a27d1cbf28f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}