diff --git a/python/3_metric comparison.ipynb b/python/3_metric comparison.ipynb
new file mode 100644
index 0000000..6fe0bfc
--- /dev/null
+++ b/python/3_metric comparison.ipynb	
@@ -0,0 +1,2469 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pandas as pd\n",
+    "import numpy as np\n",
+    "import statsmodels.formula.api as smf\n",
+    "import scipy.io as sio\n",
+    "from scipy.stats import entropy"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Load region-product data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>year</th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "      <th>regionsum</th>\n",
+       "      <th>prodsum</th>\n",
+       "      <th>yearsum</th>\n",
+       "      <th>rca</th>\n",
+       "      <th>binrca</th>\n",
+       "      <th>pivec_p</th>\n",
+       "      <th>pivec_m</th>\n",
+       "      <th>pivec_c</th>\n",
+       "      <th>pci</th>\n",
+       "      <th>pci_m</th>\n",
+       "      <th>pci_p</th>\n",
+       "      <th>pci_c</th>\n",
+       "      <th>pcil_m</th>\n",
+       "      <th>pcil_p</th>\n",
+       "      <th>pcil_c</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000e+00</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>0.002745</td>\n",
+       "      <td>0.003898</td>\n",
+       "      <td>-0.058055</td>\n",
+       "      <td>-2.307477</td>\n",
+       "      <td>-0.919122</td>\n",
+       "      <td>-1.095074</td>\n",
+       "      <td>-0.006335</td>\n",
+       "      <td>-0.007716</td>\n",
+       "      <td>-0.004269</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>8.181017e+07</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>0.002745</td>\n",
+       "      <td>0.003898</td>\n",
+       "      <td>-0.058055</td>\n",
+       "      <td>-2.307477</td>\n",
+       "      <td>-0.919122</td>\n",
+       "      <td>-1.095074</td>\n",
+       "      <td>-0.006335</td>\n",
+       "      <td>-0.007716</td>\n",
+       "      <td>-0.004269</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>9682.848633</td>\n",
+       "      <td>1.311828e+08</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.013437</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>0.002745</td>\n",
+       "      <td>0.003898</td>\n",
+       "      <td>-0.058055</td>\n",
+       "      <td>-2.307477</td>\n",
+       "      <td>-0.919122</td>\n",
+       "      <td>-1.095074</td>\n",
+       "      <td>-0.006335</td>\n",
+       "      <td>-0.007716</td>\n",
+       "      <td>-0.004269</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AIA</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000e+00</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>0.002745</td>\n",
+       "      <td>0.003898</td>\n",
+       "      <td>-0.058055</td>\n",
+       "      <td>-2.307477</td>\n",
+       "      <td>-0.919122</td>\n",
+       "      <td>-1.095074</td>\n",
+       "      <td>-0.006335</td>\n",
+       "      <td>-0.007716</td>\n",
+       "      <td>-0.004269</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>2.301677e+06</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>0.002745</td>\n",
+       "      <td>0.003898</td>\n",
+       "      <td>-0.058055</td>\n",
+       "      <td>-2.307477</td>\n",
+       "      <td>-0.919122</td>\n",
+       "      <td>-1.095074</td>\n",
+       "      <td>-0.006335</td>\n",
+       "      <td>-0.007716</td>\n",
+       "      <td>-0.004269</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   year region prod       export     regionsum       prodsum       yearsum  \\\n",
+       "0  1962    ABW  001     0.000000  0.000000e+00  6.782654e+08  1.234760e+11   \n",
+       "1  1962    AFG  001     0.000000  8.181017e+07  6.782654e+08  1.234760e+11   \n",
+       "2  1962    AGO  001  9682.848633  1.311828e+08  6.782654e+08  1.234760e+11   \n",
+       "3  1962    AIA  001     0.000000  0.000000e+00  6.782654e+08  1.234760e+11   \n",
+       "4  1962    ALB  001     0.000000  2.301677e+06  6.782654e+08  1.234760e+11   \n",
+       "\n",
+       "        rca  binrca   pivec_p   pivec_m   pivec_c       pci     pci_m  \\\n",
+       "0  0.000000       0  0.008395  0.002745  0.003898 -0.058055 -2.307477   \n",
+       "1  0.000000       0  0.008395  0.002745  0.003898 -0.058055 -2.307477   \n",
+       "2  0.013437       0  0.008395  0.002745  0.003898 -0.058055 -2.307477   \n",
+       "3  0.000000       0  0.008395  0.002745  0.003898 -0.058055 -2.307477   \n",
+       "4  0.000000       0  0.008395  0.002745  0.003898 -0.058055 -2.307477   \n",
+       "\n",
+       "      pci_p     pci_c    pcil_m    pcil_p    pcil_c  \n",
+       "0 -0.919122 -1.095074 -0.006335 -0.007716 -0.004269  \n",
+       "1 -0.919122 -1.095074 -0.006335 -0.007716 -0.004269  \n",
+       "2 -0.919122 -1.095074 -0.006335 -0.007716 -0.004269  \n",
+       "3 -0.919122 -1.095074 -0.006335 -0.007716 -0.004269  \n",
+       "4 -0.919122 -1.095074 -0.006335 -0.007716 -0.004269  "
+      ]
+     },
+     "execution_count": 2,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "trade = pd.read_parquet('data/cleaned.parquet')\n",
+    "trade.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "R0 = 0.115"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "count    3.335355e+06\n",
+       "mean     3.586094e-01\n",
+       "std      2.806920e-01\n",
+       "min      0.000000e+00\n",
+       "25%      1.172252e-01\n",
+       "50%      3.212149e-01\n",
+       "75%      5.881455e-01\n",
+       "max      1.005534e+00\n",
+       "Name: avgrca_p, dtype: float64"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "# transform RCA with R0 and generate different projections\n",
+    "df = trade.assign(llrca = np.log(1+trade.rca/R0)/np.log(1+1/R0))\n",
+    "df['bin'] = df['export']>0\n",
+    "df['avgrca_part_p'] = df.llrca*df.pivec_p\n",
+    "df['avgrca_part_m'] = df.llrca*df.pivec_m\n",
+    "df['avgrca_part_c'] = df.llrca*df.pivec_c\n",
+    "df['avgrca_p'] = df.groupby(['year','region'])['avgrca_part_p'].transform(sum)\n",
+    "df['avgrca_m'] = df.groupby(['year','region'])['avgrca_part_m'].transform(sum)\n",
+    "df['avgrca_c'] = df.groupby(['year','region'])['avgrca_part_c'].transform(sum)\n",
+    "df['diversity'] = df.groupby(['year','region'])['binrca'].transform(sum)\n",
+    "df.avgrca_p.describe()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>year</th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "      <th>regionsum</th>\n",
+       "      <th>prodsum</th>\n",
+       "      <th>yearsum</th>\n",
+       "      <th>rca</th>\n",
+       "      <th>binrca</th>\n",
+       "      <th>pivec_p</th>\n",
+       "      <th>...</th>\n",
+       "      <th>rct_p</th>\n",
+       "      <th>rct_m</th>\n",
+       "      <th>rct_c</th>\n",
+       "      <th>rct_demean_p</th>\n",
+       "      <th>rct_demean_m</th>\n",
+       "      <th>rct_demean_c</th>\n",
+       "      <th>proj_p</th>\n",
+       "      <th>proj_m</th>\n",
+       "      <th>proj_c</th>\n",
+       "      <th>eci_part</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000e+00</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-0.766895</td>\n",
+       "      <td>-1.179861</td>\n",
+       "      <td>-1.066510</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>8.181017e+07</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-0.766895</td>\n",
+       "      <td>-1.179861</td>\n",
+       "      <td>-1.066510</td>\n",
+       "      <td>-0.000000</td>\n",
+       "      <td>-0.000000</td>\n",
+       "      <td>-0.000000</td>\n",
+       "      <td>-0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>9682.848633</td>\n",
+       "      <td>1.311828e+08</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.013437</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.117628</td>\n",
+       "      <td>0.268855</td>\n",
+       "      <td>0.202484</td>\n",
+       "      <td>-0.649267</td>\n",
+       "      <td>-0.911006</td>\n",
+       "      <td>-0.864026</td>\n",
+       "      <td>-0.000908</td>\n",
+       "      <td>-0.001703</td>\n",
+       "      <td>-0.000864</td>\n",
+       "      <td>-0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AIA</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000e+00</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-0.766895</td>\n",
+       "      <td>-1.179861</td>\n",
+       "      <td>-1.066510</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>2.301677e+06</td>\n",
+       "      <td>6.782654e+08</td>\n",
+       "      <td>1.234760e+11</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "      <td>0.008395</td>\n",
+       "      <td>...</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>-0.766895</td>\n",
+       "      <td>-1.179861</td>\n",
+       "      <td>-1.066510</td>\n",
+       "      <td>-0.000000</td>\n",
+       "      <td>-0.000000</td>\n",
+       "      <td>-0.000000</td>\n",
+       "      <td>-0.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>5 rows × 38 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   year region prod       export     regionsum       prodsum       yearsum  \\\n",
+       "0  1962    ABW  001     0.000000  0.000000e+00  6.782654e+08  1.234760e+11   \n",
+       "1  1962    AFG  001     0.000000  8.181017e+07  6.782654e+08  1.234760e+11   \n",
+       "2  1962    AGO  001  9682.848633  1.311828e+08  6.782654e+08  1.234760e+11   \n",
+       "3  1962    AIA  001     0.000000  0.000000e+00  6.782654e+08  1.234760e+11   \n",
+       "4  1962    ALB  001     0.000000  2.301677e+06  6.782654e+08  1.234760e+11   \n",
+       "\n",
+       "        rca  binrca   pivec_p  ...     rct_p     rct_m     rct_c  \\\n",
+       "0  0.000000       0  0.008395  ...  0.000000  0.000000  0.000000   \n",
+       "1  0.000000       0  0.008395  ...  0.000000  0.000000  0.000000   \n",
+       "2  0.013437       0  0.008395  ...  0.117628  0.268855  0.202484   \n",
+       "3  0.000000       0  0.008395  ...  0.000000  0.000000  0.000000   \n",
+       "4  0.000000       0  0.008395  ...  0.000000  0.000000  0.000000   \n",
+       "\n",
+       "   rct_demean_p  rct_demean_m  rct_demean_c    proj_p    proj_m    proj_c  \\\n",
+       "0     -0.766895     -1.179861     -1.066510  0.000000  0.000000  0.000000   \n",
+       "1     -0.766895     -1.179861     -1.066510 -0.000000 -0.000000 -0.000000   \n",
+       "2     -0.649267     -0.911006     -0.864026 -0.000908 -0.001703 -0.000864   \n",
+       "3     -0.766895     -1.179861     -1.066510  0.000000  0.000000  0.000000   \n",
+       "4     -0.766895     -1.179861     -1.066510 -0.000000 -0.000000 -0.000000   \n",
+       "\n",
+       "   eci_part  \n",
+       "0       0.0  \n",
+       "1      -0.0  \n",
+       "2      -0.0  \n",
+       "3       0.0  \n",
+       "4      -0.0  \n",
+       "\n",
+       "[5 rows x 38 columns]"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df['rct_p'] = np.where(df.avgrca_p>0,df.llrca/df.avgrca_p,0)\n",
+    "df['rct_m'] = np.where(df.avgrca_m>0,df.llrca/df.avgrca_m,0)\n",
+    "df['rct_c'] = np.where(df.avgrca_c>0,df.llrca/df.avgrca_c,0)\n",
+    "df['rct_demean_p'] = df.rct_p-df.groupby('prod').rct_p.transform('mean')\n",
+    "df['rct_demean_m'] = df.rct_m-df.groupby('prod').rct_m.transform('mean')\n",
+    "df['rct_demean_c'] = df.rct_c-df.groupby('prod').rct_c.transform('mean')\n",
+    "df['proj_p'] = np.where(df.avgrca_p>0, df.pci_p * df.llrca * df.pivec_p / df.avgrca_p,0)\n",
+    "df['proj_m'] = np.where(df.avgrca_m>0, df.pci_m * df.llrca * df.pivec_m / df.avgrca_m,0)\n",
+    "df['proj_c'] = np.where(df.avgrca_c>0, df.pci_c * df.llrca * df.pivec_c / df.avgrca_c,0)\n",
+    "df['eci_part'] = np.where(df.diversity>0,df.pci * df.binrca/df.diversity,0)\n",
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>year</th>\n",
+       "      <th>region</th>\n",
+       "      <th>avgrca_p</th>\n",
+       "      <th>avgrca_m</th>\n",
+       "      <th>avgrca_c</th>\n",
+       "      <th>proj_p</th>\n",
+       "      <th>proj_m</th>\n",
+       "      <th>proj_c</th>\n",
+       "      <th>eci</th>\n",
+       "      <th>bin</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>ABW</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AFG</td>\n",
+       "      <td>0.229441</td>\n",
+       "      <td>0.075256</td>\n",
+       "      <td>0.101934</td>\n",
+       "      <td>-1.007255</td>\n",
+       "      <td>-1.725521</td>\n",
+       "      <td>-1.162990</td>\n",
+       "      <td>-0.067015</td>\n",
+       "      <td>39</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AGO</td>\n",
+       "      <td>0.413560</td>\n",
+       "      <td>0.180938</td>\n",
+       "      <td>0.240246</td>\n",
+       "      <td>-0.870960</td>\n",
+       "      <td>-1.279847</td>\n",
+       "      <td>-0.896124</td>\n",
+       "      <td>-0.060397</td>\n",
+       "      <td>134</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>AIA</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>1962</td>\n",
+       "      <td>ALB</td>\n",
+       "      <td>0.211957</td>\n",
+       "      <td>0.075734</td>\n",
+       "      <td>0.103582</td>\n",
+       "      <td>-0.996652</td>\n",
+       "      <td>-1.301794</td>\n",
+       "      <td>-0.791606</td>\n",
+       "      <td>-0.079119</td>\n",
+       "      <td>24</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   year region  avgrca_p  avgrca_m  avgrca_c    proj_p    proj_m    proj_c  \\\n",
+       "0  1962    ABW  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
+       "1  1962    AFG  0.229441  0.075256  0.101934 -1.007255 -1.725521 -1.162990   \n",
+       "2  1962    AGO  0.413560  0.180938  0.240246 -0.870960 -1.279847 -0.896124   \n",
+       "3  1962    AIA  0.000000  0.000000  0.000000  0.000000  0.000000  0.000000   \n",
+       "4  1962    ALB  0.211957  0.075734  0.103582 -0.996652 -1.301794 -0.791606   \n",
+       "\n",
+       "        eci  bin  \n",
+       "0  0.000000    0  \n",
+       "1 -0.067015   39  \n",
+       "2 -0.060397  134  \n",
+       "3  0.000000    0  \n",
+       "4 -0.079119   24  "
+      ]
+     },
+     "execution_count": 6,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "cntryagg = df.groupby(['year','region'])[['avgrca_part_p','avgrca_part_m','avgrca_part_c','proj_p','proj_m','proj_c','eci_part','bin']].sum().reset_index().rename(columns={'avgrca_part_p':'avgrca_p','avgrca_part_m':'avgrca_m','avgrca_part_c':'avgrca_c','eci_part':'eci'})\n",
+    "cntryagg.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## calculate metrics for 2016"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### prepare mcp of 2016"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(249, 235)"
+      ]
+     },
+     "execution_count": 7,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mcpdf = trade[trade.year==2016].pivot(index='region',columns='prod',values='binrca').fillna(0)\n",
+    "mcpdf.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(233, 235)"
+      ]
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mcp = mcpdf.values\n",
+    "ubiquity = mcp.sum(axis=0)\n",
+    "diversity = mcp.sum(axis=1)\n",
+    "ubiquity.min(),diversity.min()\n",
+    "mcp = mcp[diversity>0,:]\n",
+    "mcp.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "kp = mcp.sum(axis=0)\n",
+    "kc = mcp.sum(axis=1)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### fitness"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "qp = np.ones(mcp.shape[1])\n",
+    "fc = np.ones(mcp.shape[0])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(20):\n",
+    "    fc_t = mcp @ qp\n",
+    "    qp_t = 1/(mcp.T @ (1/fc))\n",
+    "    fc = fc_t / fc_t.mean()\n",
+    "    qp = qp_t / qp_t.mean()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### ECI"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(233, 233)"
+      ]
+     },
+     "execution_count": 12,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mr = np.diag(1/kc) @ mcp @ np.diag(1/kp) @ mcp.T\n",
+    "mr.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eigvals2,eigvecs2 = np.linalg.eig(mr)\n",
+    "eigvecs2 = np.real(eigvecs2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eci = np.sign(np.corrcoef(kc,eigvecs2[:,1])[0,1])*eigvecs2[:,1]"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### genepy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "kp_1 = (np.diag(1/kc) @ mcp).sum(axis=0)\n",
+    "wcp = np.diag(1/kc) @ mcp @ np.diag(1/kp_1)\n",
+    "ncc = wcp @ wcp.T\n",
+    "np.fill_diagonal(ncc, 0)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eigvals,eigvecs = np.linalg.eig(ncc)\n",
+    "xc1 = np.absolute(eigvecs[:,0])\n",
+    "xc2 = eigvecs[:,1]\n",
+    "lambda1 = eigvals[0]\n",
+    "lambda2 = eigvals[1]\n",
+    "genepy = np.square(lambda1*np.square(xc1)+lambda2*np.square(xc2))+2*(lambda1**2*np.square(xc1)+lambda2**2*np.square(xc2))"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### production ability"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from girth import twopl_mml\n",
+    "estimates = twopl_mml(mcp.T)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "dict_keys(['Discrimination', 'Difficulty', 'Ability', 'LatentPDF', 'AIC', 'BIC'])"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "estimates.keys()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## fix effects"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "      <th>rca</th>\n",
+       "      <th>regionsum</th>\n",
+       "      <th>prodsum</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>13446</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>96504.796875</td>\n",
+       "      <td>0.267735</td>\n",
+       "      <td>2.851261e+08</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13447</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>98623.320312</td>\n",
+       "      <td>0.102860</td>\n",
+       "      <td>7.584507e+08</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13448</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>1975.171631</td>\n",
+       "      <td>0.000058</td>\n",
+       "      <td>2.698029e+10</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13450</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>72021.726562</td>\n",
+       "      <td>0.027557</td>\n",
+       "      <td>2.067362e+09</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13451</th>\n",
+       "      <td>AND</td>\n",
+       "      <td>001</td>\n",
+       "      <td>30757.896484</td>\n",
+       "      <td>0.303068</td>\n",
+       "      <td>8.028031e+07</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      region prod        export       rca     regionsum       prodsum\n",
+       "13446    ABW  001  96504.796875  0.267735  2.851261e+08  1.843904e+10\n",
+       "13447    AFG  001  98623.320312  0.102860  7.584507e+08  1.843904e+10\n",
+       "13448    AGO  001   1975.171631  0.000058  2.698029e+10  1.843904e+10\n",
+       "13450    ALB  001  72021.726562  0.027557  2.067362e+09  1.843904e+10\n",
+       "13451    AND  001  30757.896484  0.303068  8.028031e+07  1.843904e+10"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fedf = trade[(trade.year==2016)&(trade['export']>0)][['region','prod','export','rca','regionsum','prodsum']].copy()\n",
+    "fedf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "      <th>rca</th>\n",
+       "      <th>regionsum</th>\n",
+       "      <th>prodsum</th>\n",
+       "      <th>ycp</th>\n",
+       "      <th>regionshare</th>\n",
+       "      <th>prodshare</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>13446</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>96504.796875</td>\n",
+       "      <td>0.267735</td>\n",
+       "      <td>2.851261e+08</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "      <td>-0.441469</td>\n",
+       "      <td>3.384636e-04</td>\n",
+       "      <td>5.233722e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13447</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>98623.320312</td>\n",
+       "      <td>0.102860</td>\n",
+       "      <td>7.584507e+08</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "      <td>-0.863858</td>\n",
+       "      <td>1.300326e-04</td>\n",
+       "      <td>5.348616e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13448</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>1975.171631</td>\n",
+       "      <td>0.000058</td>\n",
+       "      <td>2.698029e+10</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "      <td>-2.277953</td>\n",
+       "      <td>7.320795e-08</td>\n",
+       "      <td>1.071190e-07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13450</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>72021.726562</td>\n",
+       "      <td>0.027557</td>\n",
+       "      <td>2.067362e+09</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "      <td>-1.286105</td>\n",
+       "      <td>3.483751e-05</td>\n",
+       "      <td>3.905937e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13451</th>\n",
+       "      <td>AND</td>\n",
+       "      <td>001</td>\n",
+       "      <td>30757.896484</td>\n",
+       "      <td>0.303068</td>\n",
+       "      <td>8.028031e+07</td>\n",
+       "      <td>1.843904e+10</td>\n",
+       "      <td>-0.377422</td>\n",
+       "      <td>3.831313e-04</td>\n",
+       "      <td>1.668086e-06</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      region prod        export       rca     regionsum       prodsum  \\\n",
+       "13446    ABW  001  96504.796875  0.267735  2.851261e+08  1.843904e+10   \n",
+       "13447    AFG  001  98623.320312  0.102860  7.584507e+08  1.843904e+10   \n",
+       "13448    AGO  001   1975.171631  0.000058  2.698029e+10  1.843904e+10   \n",
+       "13450    ALB  001  72021.726562  0.027557  2.067362e+09  1.843904e+10   \n",
+       "13451    AND  001  30757.896484  0.303068  8.028031e+07  1.843904e+10   \n",
+       "\n",
+       "            ycp   regionshare     prodshare  \n",
+       "13446 -0.441469  3.384636e-04  5.233722e-06  \n",
+       "13447 -0.863858  1.300326e-04  5.348616e-06  \n",
+       "13448 -2.277953  7.320795e-08  1.071190e-07  \n",
+       "13450 -1.286105  3.483751e-05  3.905937e-06  \n",
+       "13451 -0.377422  3.831313e-04  1.668086e-06  "
+      ]
+     },
+     "execution_count": 20,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fedf['ycp'] = -np.log(-np.log(fedf.rca/(fedf.rca+1)))\n",
+    "fedf['regionshare'] = fedf.export/fedf.regionsum\n",
+    "fedf['prodshare'] = fedf.export/fedf.prodsum\n",
+    "fedf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "res = smf.ols(formula='ycp ~ region+prod', data=fedf).fit()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>index</th>\n",
+       "      <th>fe</th>\n",
+       "      <th>var</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>region[T.AFG]</td>\n",
+       "      <td>-0.343794</td>\n",
+       "      <td>AFG</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>region[T.AGO]</td>\n",
+       "      <td>-1.024346</td>\n",
+       "      <td>AGO</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>region[T.AIA]</td>\n",
+       "      <td>0.638461</td>\n",
+       "      <td>AIA</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>region[T.ALB]</td>\n",
+       "      <td>0.012500</td>\n",
+       "      <td>ALB</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>region[T.AND]</td>\n",
+       "      <td>0.205859</td>\n",
+       "      <td>AND</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "           index        fe  var\n",
+       "0  region[T.AFG] -0.343794  AFG\n",
+       "1  region[T.AGO] -1.024346  AGO\n",
+       "2  region[T.AIA]  0.638461  AIA\n",
+       "3  region[T.ALB]  0.012500  ALB\n",
+       "4  region[T.AND]  0.205859  AND"
+      ]
+     },
+     "execution_count": 22,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "fecoefdf = pd.DataFrame({'fe':res.params[1:]}).reset_index()\n",
+    "fecoefdf['var'] = fecoefdf['index'].str[-4:-1]\n",
+    "fecoefdf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>fe</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>0.000000</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>-0.343794</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>-1.024346</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>0.012500</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>AND</td>\n",
+       "      <td>0.205859</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  region        fe\n",
+       "0    ABW  0.000000\n",
+       "1    AFG -0.343794\n",
+       "2    AGO -1.024346\n",
+       "3    ALB  0.012500\n",
+       "4    AND  0.205859"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "gamma_c = fedf[['region']].drop_duplicates().merge(fecoefdf[['var','fe']].rename(columns={'var':'region'}),how='left').fillna(0)\n",
+    "gamma_c.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### entropy"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 24,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>13446</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>96504.796875</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13447</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>98623.320312</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13448</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>1975.171631</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13450</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>72021.726562</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13451</th>\n",
+       "      <td>AND</td>\n",
+       "      <td>001</td>\n",
+       "      <td>30757.896484</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      region prod        export\n",
+       "13446    ABW  001  96504.796875\n",
+       "13447    AFG  001  98623.320312\n",
+       "13448    AGO  001   1975.171631\n",
+       "13450    ALB  001  72021.726562\n",
+       "13451    AND  001  30757.896484"
+      ]
+     },
+     "execution_count": 24,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tmpdf = fedf[['region','prod','export']].copy()\n",
+    "tmpdf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 25,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(233, 235)"
+      ]
+     },
+     "execution_count": 25,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tmpdf.region.nunique(),tmpdf['prod'].nunique()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 26,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "      <th>hc</th>\n",
+       "      <th>hp</th>\n",
+       "      <th>xcp</th>\n",
+       "      <th>ycp</th>\n",
+       "      <th>xcpr</th>\n",
+       "      <th>ycpr</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>13446</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>96504.796875</td>\n",
+       "      <td>3.210066</td>\n",
+       "      <td>3.35685</td>\n",
+       "      <td>202099.239581</td>\n",
+       "      <td>217089.386604</td>\n",
+       "      <td>3.205398e-04</td>\n",
+       "      <td>8.516327e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13447</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>98623.320312</td>\n",
+       "      <td>2.447190</td>\n",
+       "      <td>3.35685</td>\n",
+       "      <td>206535.827084</td>\n",
+       "      <td>297092.432299</td>\n",
+       "      <td>1.210198e-04</td>\n",
+       "      <td>1.165481e-05</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13448</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>1975.171631</td>\n",
+       "      <td>0.660831</td>\n",
+       "      <td>3.35685</td>\n",
+       "      <td>4136.381792</td>\n",
+       "      <td>9478.364743</td>\n",
+       "      <td>6.347788e-08</td>\n",
+       "      <td>3.718323e-07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13450</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>72021.726562</td>\n",
+       "      <td>3.520899</td>\n",
+       "      <td>3.35685</td>\n",
+       "      <td>150827.074332</td>\n",
+       "      <td>139627.564405</td>\n",
+       "      <td>3.080489e-05</td>\n",
+       "      <td>5.477532e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13451</th>\n",
+       "      <td>AND</td>\n",
+       "      <td>001</td>\n",
+       "      <td>30757.896484</td>\n",
+       "      <td>3.530259</td>\n",
+       "      <td>3.35685</td>\n",
+       "      <td>64412.834304</td>\n",
+       "      <td>59342.036646</td>\n",
+       "      <td>3.102073e-04</td>\n",
+       "      <td>2.327964e-06</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      region prod        export        hc       hp            xcp  \\\n",
+       "13446    ABW  001  96504.796875  3.210066  3.35685  202099.239581   \n",
+       "13447    AFG  001  98623.320312  2.447190  3.35685  206535.827084   \n",
+       "13448    AGO  001   1975.171631  0.660831  3.35685    4136.381792   \n",
+       "13450    ALB  001  72021.726562  3.520899  3.35685  150827.074332   \n",
+       "13451    AND  001  30757.896484  3.530259  3.35685   64412.834304   \n",
+       "\n",
+       "                 ycp          xcpr          ycpr  \n",
+       "13446  217089.386604  3.205398e-04  8.516327e-06  \n",
+       "13447  297092.432299  1.210198e-04  1.165481e-05  \n",
+       "13448    9478.364743  6.347788e-08  3.718323e-07  \n",
+       "13450  139627.564405  3.080489e-05  5.477532e-06  \n",
+       "13451   59342.036646  3.102073e-04  2.327964e-06  "
+      ]
+     },
+     "execution_count": 26,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tmpdf['hc'] = tmpdf.groupby('region')['export'].transform(entropy)\n",
+    "tmpdf['hp'] = tmpdf.groupby('prod')['export'].transform(entropy)\n",
+    "tmpdf['xcp'] = tmpdf.export*(np.log(233) - tmpdf.hp)\n",
+    "tmpdf['ycp'] = tmpdf.export*(np.log(235) - tmpdf.hc)\n",
+    "tmpdf['xcpr'] = tmpdf.xcp/tmpdf.groupby('region')['xcp'].transform(sum)\n",
+    "tmpdf['ycpr'] = tmpdf.ycp/tmpdf.groupby('prod')['ycp'].transform(sum)\n",
+    "tmpdf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "for i in range(25):\n",
+    "    tmpdf['hc'] = tmpdf.groupby('region')['xcpr'].transform(entropy)\n",
+    "    tmpdf['hp'] = tmpdf.groupby('prod')['ycpr'].transform(entropy)\n",
+    "    tmpdf['xcp'] = tmpdf.export*(np.log(233) - tmpdf.hp)\n",
+    "    tmpdf['ycp'] = tmpdf.export*(np.log(235) - tmpdf.hc)\n",
+    "    tmpdf['xcpr'] = tmpdf.xcp/tmpdf.groupby('region')['xcp'].transform(sum)\n",
+    "    tmpdf['ycpr'] = tmpdf.ycp/tmpdf.groupby('prod')['ycp'].transform(sum)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 28,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>prod</th>\n",
+       "      <th>export</th>\n",
+       "      <th>hc</th>\n",
+       "      <th>hp</th>\n",
+       "      <th>xcp</th>\n",
+       "      <th>ycp</th>\n",
+       "      <th>xcpr</th>\n",
+       "      <th>ycpr</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>13446</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>001</td>\n",
+       "      <td>96504.796875</td>\n",
+       "      <td>3.326746</td>\n",
+       "      <td>3.456288</td>\n",
+       "      <td>192503.017473</td>\n",
+       "      <td>205829.207877</td>\n",
+       "      <td>3.184944e-04</td>\n",
+       "      <td>7.666431e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13447</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>001</td>\n",
+       "      <td>98623.320312</td>\n",
+       "      <td>2.597491</td>\n",
+       "      <td>3.456288</td>\n",
+       "      <td>196728.943723</td>\n",
+       "      <td>282269.251269</td>\n",
+       "      <td>1.268189e-04</td>\n",
+       "      <td>1.051356e-05</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13448</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>001</td>\n",
+       "      <td>1975.171631</td>\n",
+       "      <td>0.607231</td>\n",
+       "      <td>3.456288</td>\n",
+       "      <td>3939.975123</td>\n",
+       "      <td>9584.232724</td>\n",
+       "      <td>5.567062e-08</td>\n",
+       "      <td>3.569797e-07</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13450</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>001</td>\n",
+       "      <td>72021.726562</td>\n",
+       "      <td>3.413015</td>\n",
+       "      <td>3.456288</td>\n",
+       "      <td>143665.394218</td>\n",
+       "      <td>147397.565596</td>\n",
+       "      <td>2.989973e-05</td>\n",
+       "      <td>5.490053e-06</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13451</th>\n",
+       "      <td>AND</td>\n",
+       "      <td>001</td>\n",
+       "      <td>30757.896484</td>\n",
+       "      <td>3.441601</td>\n",
+       "      <td>3.456288</td>\n",
+       "      <td>61354.337568</td>\n",
+       "      <td>62068.964583</td>\n",
+       "      <td>2.951840e-04</td>\n",
+       "      <td>2.311856e-06</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      region prod        export        hc        hp            xcp  \\\n",
+       "13446    ABW  001  96504.796875  3.326746  3.456288  192503.017473   \n",
+       "13447    AFG  001  98623.320312  2.597491  3.456288  196728.943723   \n",
+       "13448    AGO  001   1975.171631  0.607231  3.456288    3939.975123   \n",
+       "13450    ALB  001  72021.726562  3.413015  3.456288  143665.394218   \n",
+       "13451    AND  001  30757.896484  3.441601  3.456288   61354.337568   \n",
+       "\n",
+       "                 ycp          xcpr          ycpr  \n",
+       "13446  205829.207877  3.184944e-04  7.666431e-06  \n",
+       "13447  282269.251269  1.268189e-04  1.051356e-05  \n",
+       "13448    9584.232724  5.567062e-08  3.569797e-07  \n",
+       "13450  147397.565596  2.989973e-05  5.490053e-06  \n",
+       "13451   62068.964583  2.951840e-04  2.311856e-06  "
+      ]
+     },
+     "execution_count": 28,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "tmpdf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>hc</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>13446</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>3.326746</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13447</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>2.597491</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13448</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>0.607231</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>70221</th>\n",
+       "      <td>AIA</td>\n",
+       "      <td>2.766955</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13450</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>3.413015</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "      region        hc\n",
+       "13446    ABW  3.326746\n",
+       "13447    AFG  2.597491\n",
+       "13448    AGO  0.607231\n",
+       "70221    AIA  2.766955\n",
+       "13450    ALB  3.413015"
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "regiondf = tmpdf[['region','hc']].drop_duplicates().sort_values('region')\n",
+    "regiondf.head()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## combine metrics into a result dataframe"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "(233, 8)"
+      ]
+     },
+     "execution_count": 30,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "resdf = pd.DataFrame(\n",
+    "    {\n",
+    "        \"fitness2016\": fc,\n",
+    "        \"eci2016\": eci,\n",
+    "        \"kc\": kc,\n",
+    "        \"xc1\": xc1,\n",
+    "        \"xc2\": xc2,\n",
+    "        \"genepy\": genepy,\n",
+    "        \"ability\": estimates[\"Ability\"],\n",
+    "    },\n",
+    "    index=mcpdf.index[diversity > 0],\n",
+    ").reset_index()\n",
+    "resdf.shape"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 31,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>region</th>\n",
+       "      <th>fitness2016</th>\n",
+       "      <th>eci2016</th>\n",
+       "      <th>kc</th>\n",
+       "      <th>xc1</th>\n",
+       "      <th>xc2</th>\n",
+       "      <th>genepy</th>\n",
+       "      <th>ability</th>\n",
+       "      <th>fe</th>\n",
+       "      <th>hc</th>\n",
+       "      <th>avgrca_p</th>\n",
+       "      <th>avgrca_m</th>\n",
+       "      <th>avgrca_c</th>\n",
+       "      <th>proj_p</th>\n",
+       "      <th>proj_m</th>\n",
+       "      <th>proj_c</th>\n",
+       "      <th>eci</th>\n",
+       "      <th>bin</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ABW</td>\n",
+       "      <td>1.004032</td>\n",
+       "      <td>0.024499</td>\n",
+       "      <td>33</td>\n",
+       "      <td>0.078176</td>\n",
+       "      <td>0.021791</td>\n",
+       "      <td>0.015089</td>\n",
+       "      <td>0.313106</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>3.326746</td>\n",
+       "      <td>0.458487</td>\n",
+       "      <td>0.475485</td>\n",
+       "      <td>0.466518</td>\n",
+       "      <td>0.105968</td>\n",
+       "      <td>0.117298</td>\n",
+       "      <td>0.152733</td>\n",
+       "      <td>0.021736</td>\n",
+       "      <td>192</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>AFG</td>\n",
+       "      <td>0.365737</td>\n",
+       "      <td>-0.096063</td>\n",
+       "      <td>25</td>\n",
+       "      <td>0.042223</td>\n",
+       "      <td>-0.111436</td>\n",
+       "      <td>0.007655</td>\n",
+       "      <td>-0.452356</td>\n",
+       "      <td>-0.343794</td>\n",
+       "      <td>2.597491</td>\n",
+       "      <td>0.511607</td>\n",
+       "      <td>0.234632</td>\n",
+       "      <td>0.286582</td>\n",
+       "      <td>-0.815917</td>\n",
+       "      <td>-1.072287</td>\n",
+       "      <td>-0.849485</td>\n",
+       "      <td>-0.050744</td>\n",
+       "      <td>164</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>AGO</td>\n",
+       "      <td>0.070533</td>\n",
+       "      <td>-0.100995</td>\n",
+       "      <td>7</td>\n",
+       "      <td>0.030913</td>\n",
+       "      <td>-0.070780</td>\n",
+       "      <td>0.003656</td>\n",
+       "      <td>-1.378027</td>\n",
+       "      <td>-1.024346</td>\n",
+       "      <td>0.607231</td>\n",
+       "      <td>0.160169</td>\n",
+       "      <td>0.116978</td>\n",
+       "      <td>0.135088</td>\n",
+       "      <td>-0.437043</td>\n",
+       "      <td>-0.387758</td>\n",
+       "      <td>-0.384177</td>\n",
+       "      <td>-0.039769</td>\n",
+       "      <td>198</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>AIA</td>\n",
+       "      <td>0.604428</td>\n",
+       "      <td>0.060996</td>\n",
+       "      <td>24</td>\n",
+       "      <td>0.064913</td>\n",
+       "      <td>0.025858</td>\n",
+       "      <td>0.010484</td>\n",
+       "      <td>0.221500</td>\n",
+       "      <td>0.638461</td>\n",
+       "      <td>2.766955</td>\n",
+       "      <td>0.281158</td>\n",
+       "      <td>0.304979</td>\n",
+       "      <td>0.266034</td>\n",
+       "      <td>0.237016</td>\n",
+       "      <td>0.157759</td>\n",
+       "      <td>0.369143</td>\n",
+       "      <td>0.037130</td>\n",
+       "      <td>67</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>ALB</td>\n",
+       "      <td>0.733577</td>\n",
+       "      <td>-0.039531</td>\n",
+       "      <td>47</td>\n",
+       "      <td>0.045679</td>\n",
+       "      <td>-0.060442</td>\n",
+       "      <td>0.006066</td>\n",
+       "      <td>0.650443</td>\n",
+       "      <td>0.012500</td>\n",
+       "      <td>3.413015</td>\n",
+       "      <td>0.659942</td>\n",
+       "      <td>0.531238</td>\n",
+       "      <td>0.513222</td>\n",
+       "      <td>-0.114121</td>\n",
+       "      <td>-0.304722</td>\n",
+       "      <td>-0.224567</td>\n",
+       "      <td>0.001374</td>\n",
+       "      <td>206</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "  region  fitness2016   eci2016  kc       xc1       xc2    genepy   ability  \\\n",
+       "0    ABW     1.004032  0.024499  33  0.078176  0.021791  0.015089  0.313106   \n",
+       "1    AFG     0.365737 -0.096063  25  0.042223 -0.111436  0.007655 -0.452356   \n",
+       "2    AGO     0.070533 -0.100995   7  0.030913 -0.070780  0.003656 -1.378027   \n",
+       "3    AIA     0.604428  0.060996  24  0.064913  0.025858  0.010484  0.221500   \n",
+       "4    ALB     0.733577 -0.039531  47  0.045679 -0.060442  0.006066  0.650443   \n",
+       "\n",
+       "         fe        hc  avgrca_p  avgrca_m  avgrca_c    proj_p    proj_m  \\\n",
+       "0  0.000000  3.326746  0.458487  0.475485  0.466518  0.105968  0.117298   \n",
+       "1 -0.343794  2.597491  0.511607  0.234632  0.286582 -0.815917 -1.072287   \n",
+       "2 -1.024346  0.607231  0.160169  0.116978  0.135088 -0.437043 -0.387758   \n",
+       "3  0.638461  2.766955  0.281158  0.304979  0.266034  0.237016  0.157759   \n",
+       "4  0.012500  3.413015  0.659942  0.531238  0.513222 -0.114121 -0.304722   \n",
+       "\n",
+       "     proj_c       eci  bin  \n",
+       "0  0.152733  0.021736  192  \n",
+       "1 -0.849485 -0.050744  164  \n",
+       "2 -0.384177 -0.039769  198  \n",
+       "3  0.369143  0.037130   67  \n",
+       "4 -0.224567  0.001374  206  "
+      ]
+     },
+     "execution_count": 31,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "resdf = resdf.merge(gamma_c).merge(regiondf).merge(cntryagg[(cntryagg.year==2016)&(cntryagg.bin>0)].drop(columns=['year']), how='left')\n",
+    "resdf.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 32,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>kc</th>\n",
+       "      <th>fitness2016</th>\n",
+       "      <th>hc</th>\n",
+       "      <th>ability</th>\n",
+       "      <th>xc1</th>\n",
+       "      <th>avgrca_p</th>\n",
+       "      <th>avgrca_m</th>\n",
+       "      <th>avgrca_c</th>\n",
+       "      <th>bin</th>\n",
+       "      <th>eci2016</th>\n",
+       "      <th>eci</th>\n",
+       "      <th>xc2</th>\n",
+       "      <th>proj_p</th>\n",
+       "      <th>proj_m</th>\n",
+       "      <th>proj_c</th>\n",
+       "      <th>genepy</th>\n",
+       "      <th>fe</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>kc</th>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.908907</td>\n",
+       "      <td>0.922799</td>\n",
+       "      <td>0.910720</td>\n",
+       "      <td>0.716408</td>\n",
+       "      <td>0.931741</td>\n",
+       "      <td>0.967285</td>\n",
+       "      <td>0.969524</td>\n",
+       "      <td>0.698028</td>\n",
+       "      <td>0.420054</td>\n",
+       "      <td>0.470444</td>\n",
+       "      <td>0.225476</td>\n",
+       "      <td>0.406505</td>\n",
+       "      <td>0.397427</td>\n",
+       "      <td>0.378487</td>\n",
+       "      <td>0.625309</td>\n",
+       "      <td>0.324606</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fitness2016</th>\n",
+       "      <td>0.908907</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.794303</td>\n",
+       "      <td>0.802775</td>\n",
+       "      <td>0.866133</td>\n",
+       "      <td>0.744436</td>\n",
+       "      <td>0.886949</td>\n",
+       "      <td>0.868511</td>\n",
+       "      <td>0.551076</td>\n",
+       "      <td>0.564106</td>\n",
+       "      <td>0.583691</td>\n",
+       "      <td>0.500641</td>\n",
+       "      <td>0.504874</td>\n",
+       "      <td>0.481109</td>\n",
+       "      <td>0.499610</td>\n",
+       "      <td>0.845189</td>\n",
+       "      <td>0.358421</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>hc</th>\n",
+       "      <td>0.922799</td>\n",
+       "      <td>0.794303</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.958869</td>\n",
+       "      <td>0.722283</td>\n",
+       "      <td>0.925722</td>\n",
+       "      <td>0.942871</td>\n",
+       "      <td>0.943946</td>\n",
+       "      <td>0.688022</td>\n",
+       "      <td>0.490495</td>\n",
+       "      <td>0.543543</td>\n",
+       "      <td>0.221256</td>\n",
+       "      <td>0.498029</td>\n",
+       "      <td>0.478179</td>\n",
+       "      <td>0.457501</td>\n",
+       "      <td>0.593582</td>\n",
+       "      <td>0.354024</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>ability</th>\n",
+       "      <td>0.910720</td>\n",
+       "      <td>0.802775</td>\n",
+       "      <td>0.958869</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.779477</td>\n",
+       "      <td>0.895495</td>\n",
+       "      <td>0.927337</td>\n",
+       "      <td>0.924193</td>\n",
+       "      <td>0.684027</td>\n",
+       "      <td>0.551558</td>\n",
+       "      <td>0.607099</td>\n",
+       "      <td>0.257679</td>\n",
+       "      <td>0.542558</td>\n",
+       "      <td>0.536644</td>\n",
+       "      <td>0.517418</td>\n",
+       "      <td>0.644847</td>\n",
+       "      <td>0.362611</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>xc1</th>\n",
+       "      <td>0.716408</td>\n",
+       "      <td>0.866133</td>\n",
+       "      <td>0.722283</td>\n",
+       "      <td>0.779477</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.564092</td>\n",
+       "      <td>0.761327</td>\n",
+       "      <td>0.726598</td>\n",
+       "      <td>0.396321</td>\n",
+       "      <td>0.801605</td>\n",
+       "      <td>0.803642</td>\n",
+       "      <td>0.684278</td>\n",
+       "      <td>0.723349</td>\n",
+       "      <td>0.699943</td>\n",
+       "      <td>0.742445</td>\n",
+       "      <td>0.947870</td>\n",
+       "      <td>0.485798</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>avgrca_p</th>\n",
+       "      <td>0.931741</td>\n",
+       "      <td>0.744436</td>\n",
+       "      <td>0.925722</td>\n",
+       "      <td>0.895495</td>\n",
+       "      <td>0.564092</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.937545</td>\n",
+       "      <td>0.961412</td>\n",
+       "      <td>0.808057</td>\n",
+       "      <td>0.250376</td>\n",
+       "      <td>0.305705</td>\n",
+       "      <td>-0.006503</td>\n",
+       "      <td>0.261337</td>\n",
+       "      <td>0.274873</td>\n",
+       "      <td>0.226414</td>\n",
+       "      <td>0.454005</td>\n",
+       "      <td>0.212463</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>avgrca_m</th>\n",
+       "      <td>0.967285</td>\n",
+       "      <td>0.886949</td>\n",
+       "      <td>0.942871</td>\n",
+       "      <td>0.927337</td>\n",
+       "      <td>0.761327</td>\n",
+       "      <td>0.937545</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.995504</td>\n",
+       "      <td>0.749586</td>\n",
+       "      <td>0.474007</td>\n",
+       "      <td>0.538753</td>\n",
+       "      <td>0.264613</td>\n",
+       "      <td>0.498864</td>\n",
+       "      <td>0.492137</td>\n",
+       "      <td>0.472184</td>\n",
+       "      <td>0.655015</td>\n",
+       "      <td>0.314116</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>avgrca_c</th>\n",
+       "      <td>0.969524</td>\n",
+       "      <td>0.868511</td>\n",
+       "      <td>0.943946</td>\n",
+       "      <td>0.924193</td>\n",
+       "      <td>0.726598</td>\n",
+       "      <td>0.961412</td>\n",
+       "      <td>0.995504</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.778343</td>\n",
+       "      <td>0.417803</td>\n",
+       "      <td>0.478739</td>\n",
+       "      <td>0.199634</td>\n",
+       "      <td>0.438848</td>\n",
+       "      <td>0.443048</td>\n",
+       "      <td>0.413391</td>\n",
+       "      <td>0.623274</td>\n",
+       "      <td>0.288118</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>bin</th>\n",
+       "      <td>0.698028</td>\n",
+       "      <td>0.551076</td>\n",
+       "      <td>0.688022</td>\n",
+       "      <td>0.684027</td>\n",
+       "      <td>0.396321</td>\n",
+       "      <td>0.808057</td>\n",
+       "      <td>0.749586</td>\n",
+       "      <td>0.778343</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>-0.012318</td>\n",
+       "      <td>0.112200</td>\n",
+       "      <td>-0.166632</td>\n",
+       "      <td>0.152788</td>\n",
+       "      <td>0.208170</td>\n",
+       "      <td>0.095990</td>\n",
+       "      <td>0.348256</td>\n",
+       "      <td>-0.207158</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>eci2016</th>\n",
+       "      <td>0.420054</td>\n",
+       "      <td>0.564106</td>\n",
+       "      <td>0.490495</td>\n",
+       "      <td>0.551558</td>\n",
+       "      <td>0.801605</td>\n",
+       "      <td>0.250376</td>\n",
+       "      <td>0.474007</td>\n",
+       "      <td>0.417803</td>\n",
+       "      <td>-0.012318</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.952046</td>\n",
+       "      <td>0.809481</td>\n",
+       "      <td>0.856796</td>\n",
+       "      <td>0.804323</td>\n",
+       "      <td>0.876913</td>\n",
+       "      <td>0.696404</td>\n",
+       "      <td>0.680923</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>eci</th>\n",
+       "      <td>0.470444</td>\n",
+       "      <td>0.583691</td>\n",
+       "      <td>0.543543</td>\n",
+       "      <td>0.607099</td>\n",
+       "      <td>0.803642</td>\n",
+       "      <td>0.305705</td>\n",
+       "      <td>0.538753</td>\n",
+       "      <td>0.478739</td>\n",
+       "      <td>0.112200</td>\n",
+       "      <td>0.952046</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.740652</td>\n",
+       "      <td>0.920774</td>\n",
+       "      <td>0.875872</td>\n",
+       "      <td>0.913696</td>\n",
+       "      <td>0.676550</td>\n",
+       "      <td>0.597208</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>xc2</th>\n",
+       "      <td>0.225476</td>\n",
+       "      <td>0.500641</td>\n",
+       "      <td>0.221256</td>\n",
+       "      <td>0.257679</td>\n",
+       "      <td>0.684278</td>\n",
+       "      <td>-0.006503</td>\n",
+       "      <td>0.264613</td>\n",
+       "      <td>0.199634</td>\n",
+       "      <td>-0.166632</td>\n",
+       "      <td>0.809481</td>\n",
+       "      <td>0.740652</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.643015</td>\n",
+       "      <td>0.605885</td>\n",
+       "      <td>0.730221</td>\n",
+       "      <td>0.694924</td>\n",
+       "      <td>0.492865</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>proj_p</th>\n",
+       "      <td>0.406505</td>\n",
+       "      <td>0.504874</td>\n",
+       "      <td>0.498029</td>\n",
+       "      <td>0.542558</td>\n",
+       "      <td>0.723349</td>\n",
+       "      <td>0.261337</td>\n",
+       "      <td>0.498864</td>\n",
+       "      <td>0.438848</td>\n",
+       "      <td>0.152788</td>\n",
+       "      <td>0.856796</td>\n",
+       "      <td>0.920774</td>\n",
+       "      <td>0.643015</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.903356</td>\n",
+       "      <td>0.917782</td>\n",
+       "      <td>0.600577</td>\n",
+       "      <td>0.468510</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>proj_m</th>\n",
+       "      <td>0.397427</td>\n",
+       "      <td>0.481109</td>\n",
+       "      <td>0.478179</td>\n",
+       "      <td>0.536644</td>\n",
+       "      <td>0.699943</td>\n",
+       "      <td>0.274873</td>\n",
+       "      <td>0.492137</td>\n",
+       "      <td>0.443048</td>\n",
+       "      <td>0.208170</td>\n",
+       "      <td>0.804323</td>\n",
+       "      <td>0.875872</td>\n",
+       "      <td>0.605885</td>\n",
+       "      <td>0.903356</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.955276</td>\n",
+       "      <td>0.568372</td>\n",
+       "      <td>0.443293</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>proj_c</th>\n",
+       "      <td>0.378487</td>\n",
+       "      <td>0.499610</td>\n",
+       "      <td>0.457501</td>\n",
+       "      <td>0.517418</td>\n",
+       "      <td>0.742445</td>\n",
+       "      <td>0.226414</td>\n",
+       "      <td>0.472184</td>\n",
+       "      <td>0.413391</td>\n",
+       "      <td>0.095990</td>\n",
+       "      <td>0.876913</td>\n",
+       "      <td>0.913696</td>\n",
+       "      <td>0.730221</td>\n",
+       "      <td>0.917782</td>\n",
+       "      <td>0.955276</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.623007</td>\n",
+       "      <td>0.539185</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>genepy</th>\n",
+       "      <td>0.625309</td>\n",
+       "      <td>0.845189</td>\n",
+       "      <td>0.593582</td>\n",
+       "      <td>0.644847</td>\n",
+       "      <td>0.947870</td>\n",
+       "      <td>0.454005</td>\n",
+       "      <td>0.655015</td>\n",
+       "      <td>0.623274</td>\n",
+       "      <td>0.348256</td>\n",
+       "      <td>0.696404</td>\n",
+       "      <td>0.676550</td>\n",
+       "      <td>0.694924</td>\n",
+       "      <td>0.600577</td>\n",
+       "      <td>0.568372</td>\n",
+       "      <td>0.623007</td>\n",
+       "      <td>1.000000</td>\n",
+       "      <td>0.381975</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>fe</th>\n",
+       "      <td>0.324606</td>\n",
+       "      <td>0.358421</td>\n",
+       "      <td>0.354024</td>\n",
+       "      <td>0.362611</td>\n",
+       "      <td>0.485798</td>\n",
+       "      <td>0.212463</td>\n",
+       "      <td>0.314116</td>\n",
+       "      <td>0.288118</td>\n",
+       "      <td>-0.207158</td>\n",
+       "      <td>0.680923</td>\n",
+       "      <td>0.597208</td>\n",
+       "      <td>0.492865</td>\n",
+       "      <td>0.468510</td>\n",
+       "      <td>0.443293</td>\n",
+       "      <td>0.539185</td>\n",
+       "      <td>0.381975</td>\n",
+       "      <td>1.000000</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                   kc  fitness2016        hc   ability       xc1  avgrca_p  \\\n",
+       "kc           1.000000     0.908907  0.922799  0.910720  0.716408  0.931741   \n",
+       "fitness2016  0.908907     1.000000  0.794303  0.802775  0.866133  0.744436   \n",
+       "hc           0.922799     0.794303  1.000000  0.958869  0.722283  0.925722   \n",
+       "ability      0.910720     0.802775  0.958869  1.000000  0.779477  0.895495   \n",
+       "xc1          0.716408     0.866133  0.722283  0.779477  1.000000  0.564092   \n",
+       "avgrca_p     0.931741     0.744436  0.925722  0.895495  0.564092  1.000000   \n",
+       "avgrca_m     0.967285     0.886949  0.942871  0.927337  0.761327  0.937545   \n",
+       "avgrca_c     0.969524     0.868511  0.943946  0.924193  0.726598  0.961412   \n",
+       "bin          0.698028     0.551076  0.688022  0.684027  0.396321  0.808057   \n",
+       "eci2016      0.420054     0.564106  0.490495  0.551558  0.801605  0.250376   \n",
+       "eci          0.470444     0.583691  0.543543  0.607099  0.803642  0.305705   \n",
+       "xc2          0.225476     0.500641  0.221256  0.257679  0.684278 -0.006503   \n",
+       "proj_p       0.406505     0.504874  0.498029  0.542558  0.723349  0.261337   \n",
+       "proj_m       0.397427     0.481109  0.478179  0.536644  0.699943  0.274873   \n",
+       "proj_c       0.378487     0.499610  0.457501  0.517418  0.742445  0.226414   \n",
+       "genepy       0.625309     0.845189  0.593582  0.644847  0.947870  0.454005   \n",
+       "fe           0.324606     0.358421  0.354024  0.362611  0.485798  0.212463   \n",
+       "\n",
+       "             avgrca_m  avgrca_c       bin   eci2016       eci       xc2  \\\n",
+       "kc           0.967285  0.969524  0.698028  0.420054  0.470444  0.225476   \n",
+       "fitness2016  0.886949  0.868511  0.551076  0.564106  0.583691  0.500641   \n",
+       "hc           0.942871  0.943946  0.688022  0.490495  0.543543  0.221256   \n",
+       "ability      0.927337  0.924193  0.684027  0.551558  0.607099  0.257679   \n",
+       "xc1          0.761327  0.726598  0.396321  0.801605  0.803642  0.684278   \n",
+       "avgrca_p     0.937545  0.961412  0.808057  0.250376  0.305705 -0.006503   \n",
+       "avgrca_m     1.000000  0.995504  0.749586  0.474007  0.538753  0.264613   \n",
+       "avgrca_c     0.995504  1.000000  0.778343  0.417803  0.478739  0.199634   \n",
+       "bin          0.749586  0.778343  1.000000 -0.012318  0.112200 -0.166632   \n",
+       "eci2016      0.474007  0.417803 -0.012318  1.000000  0.952046  0.809481   \n",
+       "eci          0.538753  0.478739  0.112200  0.952046  1.000000  0.740652   \n",
+       "xc2          0.264613  0.199634 -0.166632  0.809481  0.740652  1.000000   \n",
+       "proj_p       0.498864  0.438848  0.152788  0.856796  0.920774  0.643015   \n",
+       "proj_m       0.492137  0.443048  0.208170  0.804323  0.875872  0.605885   \n",
+       "proj_c       0.472184  0.413391  0.095990  0.876913  0.913696  0.730221   \n",
+       "genepy       0.655015  0.623274  0.348256  0.696404  0.676550  0.694924   \n",
+       "fe           0.314116  0.288118 -0.207158  0.680923  0.597208  0.492865   \n",
+       "\n",
+       "               proj_p    proj_m    proj_c    genepy        fe  \n",
+       "kc           0.406505  0.397427  0.378487  0.625309  0.324606  \n",
+       "fitness2016  0.504874  0.481109  0.499610  0.845189  0.358421  \n",
+       "hc           0.498029  0.478179  0.457501  0.593582  0.354024  \n",
+       "ability      0.542558  0.536644  0.517418  0.644847  0.362611  \n",
+       "xc1          0.723349  0.699943  0.742445  0.947870  0.485798  \n",
+       "avgrca_p     0.261337  0.274873  0.226414  0.454005  0.212463  \n",
+       "avgrca_m     0.498864  0.492137  0.472184  0.655015  0.314116  \n",
+       "avgrca_c     0.438848  0.443048  0.413391  0.623274  0.288118  \n",
+       "bin          0.152788  0.208170  0.095990  0.348256 -0.207158  \n",
+       "eci2016      0.856796  0.804323  0.876913  0.696404  0.680923  \n",
+       "eci          0.920774  0.875872  0.913696  0.676550  0.597208  \n",
+       "xc2          0.643015  0.605885  0.730221  0.694924  0.492865  \n",
+       "proj_p       1.000000  0.903356  0.917782  0.600577  0.468510  \n",
+       "proj_m       0.903356  1.000000  0.955276  0.568372  0.443293  \n",
+       "proj_c       0.917782  0.955276  1.000000  0.623007  0.539185  \n",
+       "genepy       0.600577  0.568372  0.623007  1.000000  0.381975  \n",
+       "fe           0.468510  0.443293  0.539185  0.381975  1.000000  "
+      ]
+     },
+     "execution_count": 32,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "resdf[['kc','fitness2016','hc','ability','xc1','avgrca_p','avgrca_m','avgrca_c','bin','eci2016','eci','xc2','proj_p','proj_m','proj_c','genepy','fe']].corr()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 33,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "Index(['region', 'fitness2016', 'eci2016', 'kc', 'xc1', 'xc2', 'genepy',\n",
+       "       'ability', 'fe', 'hc', 'avgrca_p', 'avgrca_m', 'avgrca_c', 'proj_p',\n",
+       "       'proj_m', 'proj_c', 'eci', 'bin'],\n",
+       "      dtype='object')"
+      ]
+     },
+     "execution_count": 33,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "resdf.columns"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 34,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sio.savemat('metric2016.mat', {'struct1':resdf.to_dict(\"list\")})"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3 (ipykernel)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.8.12"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}