diff --git a/demo.ipynb b/demo.ipynb index a6518f4..c7beee5 100644 --- a/demo.ipynb +++ b/demo.ipynb @@ -204,16 +204,7 @@ "execution_count": 6, "id": "c0749134", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/scottlee/code/fairness/tools.py:461: FutureWarning: The default dtype for empty Series will be 'object' instead of 'float64' in a future version. Specify a dtype explicitly to silence this warning.\n", - " stype = type(pd.Series())\n" - ] - } - ], + "outputs": [], "source": [ "shear_stats = tools.clf_metrics(y, y_)" ] @@ -389,10 +380,10 @@ "\n", "group fpr tpr\n", " cat 0.1829 0.4419\n", - " dog 0.1862 0.4103\n", - "sheep 0.1855 0.4304\n", + " dog 0.1835 0.4359\n", + "sheep 0.1774 0.443\n", "\n", - "And loss is 0.2470\n", + "And loss is 0.2430\n", "\n" ] } @@ -502,7 +493,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAENCAYAAAD+CUlOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAtPUlEQVR4nO3de3hU9b0u8HdymdyTSTK3zOQKQZCbWlCQWqwoUjChSrBoEKzVCGUrLdt6UAFhHwXc6mk2uPu0xXq6qYUqWyyYVgMFSo8FKoVWUS4HAuWSuU/umcwkc/ntP2hGAsgkYdasyeT9PI/Pw6xJJl8Xw7z5rVnrHYUQQoCIiOga4uQegIiIoh/DgoiIQmJYEBFRSAwLIiIKiWFBREQhMSyIiCgkycOivb0dZWVlqK+vv+K+48ePo6KiAtOmTcOyZcvg8/mkHoeIiPpB0rD47LPP8PDDD+Ps2bNXvf/ZZ5/FihUrsGPHDgghsGXLFinHISKifpI0LLZs2YKVK1dCq9VecZ/JZILH48HNN98MAJg1axZqa2ulHIeIiPopQcoHX7169VfeZ7fbodFogrc1Gg1sNluvHtfj8eCLL76ARqNBfHz8dc9JRDQY+P1+OBwOjB49GsnJyX36XknD4lqu1jKiUCh69b1ffPEF5s6dG+6RiIgGhU2bNmH8+PF9+h7ZwkKn08HpdAZvOxyOqx6uupruFcmmTZug1+slmY+IaKDzer2wWq1wu91ISUkBAHz3u9/tcVSnt2QLC6PRiKSkJBw+fBjjxo3Dtm3bMHny5F59b/ehJ71ej/z8fCnHJCIacIQQaGhoQEtLC7KysjB8+HBkZ2fDZDIBQL8O30f8Oouqqip8/vnnAIDXX38da9euxfTp0+F2uzF//vxIj0NEFFM8Hg/OnDkDq9WKtLQ0lJaWIicnp9eH+b9KRFYWe/bsCf75zTffDP55xIgReO+99yIxAhFRTAsEAnA6nXA4HIiLi0N+fj6ysrKuOyS6yXYYioiIwsPtdgcvR8jKykJeXh4SEsL78s6wICIaoAKBAOx2O5xOJxISElBYWIjMzExJfhbDgohoAHK5XDCZTOjq6kJ2djb0er2k150xLIiIBhC/3w+bzYbGxkYkJiaiuLgY6enpkv9chgUR0QDR1tYGk8kEn8+H3Nxc6HQ6xMVF5qRWhgURUZTz+XywWCxoaWlBUlISCgsLkZqaGtEZGBZERFFKCIGWlhZYLBYEAgFoNBpoNJqIrSYuxbAgIopCXq8XZrMZbW1tSElJgdFo7HP5XzgxLIiIoogQAk1NTbBarRBCQK/XIzc3N2wX1/UXw4KIKEp0dnbCbDbD5XIhLS0NBoMBSUlJco8FgGFBRCS77uI/m80GhUIBg8GA7Oxs2VcTl2JYEBHJyOPxwGQywe12IyMjAwaDAYmJiXKPdQWGBRGRDAKBABwOB5xOpyTFf+HGsCAiirCOjg6YTCZ0dnZKVvwXbtE9HRFRDAkEArDZbGhoaJC8+C/cGBZERBHQ3t4Ok8kEr9eLnJwc6HQ6SYv/wo1hQUQkIb/fD6vViqamJiiVSpSUlCAtLU3usfqMYUFEJJHW1laYzWb4fD6o1WpotVpZqjrCgWFBRBRmlxf/FRUVISUlRe6xrgvDgogoTC4v/tNqtVCr1QN2NXEphgURURh0dXXBYrFETfFfuDEsiIiuQ7QW/4Ubw4KIqJ86OzthMpnQ0dGBtLQ0GI1GKJVKuceSBMOCiKiPhBBwOp2w2+1QKBQwGo1QqVQxt5q4FMPiOjg9LvyjrQHNXR4YUzNRkpGDlITY/K2CiC5yu90wmUzweDxRXfwXbgyLfjrX3oiqP/83dltOAQDiFXH4P7eV49HSW5GpjJ03tYjoou7iP4fDgfj4eBQUFCAzMzOmVxOXYlj0gxACv677WzAoAMAvAvjhJ9sxUVOEWzWFMk5HROF2afGfSqWCXq+P+uK/cBv4J//KwOZuwy9OfnLV+/6f9UyEpyEiqQQCAVgsFpw5cwaBQABFRUXIz88fdEEBcGXRLwlx8Uj7ivcmMhKj4yMQiej6DPTiv3DjyqIf1MlpeG7slCu2K+PiMUlXHPmBiChs/H4/6uvrcfbsWSgUCpSUlMBgMAzqoAAYFv12r/EGrBk3I7jCKEnPwYf3VmFEllbmyYiov1pbW3Hq1Ck0NzdDrVajtLR0QDbESoGHofpJm5KBZ0bfiQdLboLL2wV1chryUgfGh5gQUU8+nw9msxmtra1ITk6OieK/cGNYXIeEuHgMyciVewwi6ichBJqbm2G1WoPFfxqNZtCcDtsXDAsiGpS6urpgNpvR3t6O1NRUGI1GJCXxBJWvwrAgokFFCIHGxkbYbDYAQF5eHnJycriaCEHSN7hramowY8YMTJ06FZs2bbri/qNHj6KiogIzZ87EggUL0NraKuU4RDTIdXZ24h//+AcsFgtSUlJQWloakw2xUpAsLGw2G6qrq7F582Zs374d7777Lurq6np8zerVq7F48WJ88MEHKCkpwVtvvSXVOEQ0iAkh4HA4UFdXB4/HA6PRiOLi4phtiJWCZGGxf/9+TJw4ESqVCqmpqZg2bRpqa2t7fE0gEIDL5QJwsZwrlj4ohIiig9vtxunTp2Gz2ZCRkYFhw4YhOzubq4k+kuw9C7vdDo1GE7yt1Wpx5MiRHl/z3HPP4bHHHsOaNWuQkpKCLVu2SDUOEQ0ylxb/JSQkoKCgAFlZWXKPNWBJtrIQQlyx7dIk93g8WLZsGTZu3Ig///nPqKysxNKlS6Uah4gGEZfLhbq6OjgcDqhUKpSWljIorpNkYaHT6eB0OoO37XY7tNovr24+efIkkpKSMHbsWADAnDlzcPDgQanGIaJBwO/3w2w24x//+AeEEIO6+C/cJAuLSZMm4cCBA2hsbITb7cbOnTsxefLk4P1FRUWwWq04c+ZiS+vu3bsxZswYqcYhohjX1taGuro6NDY2IicnB6WlpcjIyJB7rJghWdzqdDosWbIE8+fPh9frxezZszF27FhUVVVh8eLFGDNmDNauXYsf/vCHEEIgNzcXa9askWocIopRPp8PVqsVzc3NUCqVKCkpYZ+TBCRdm5WXl6O8vLzHtjfffDP45zvvvBN33nmnlCMQUQxraWmBxWKBz+eDRqOBRqNBXBz7UaXAA3lENOB4vV5YLBYW/0UQw4KIBozu4j+LxQIhBHQ6HdRqNa+ZiACGBRENCF1dXTCZTHC5XCz+kwHDgoiiGov/ogPDgoiilsfjgdlsRkdHB9LT02EwGNjnJBOGBRFFne7iP4fDgbi4OBiNRqhUKq4mZMSwIKKo4na7YTKZ4PF4kJmZCYPBwCuwowD/BogoKgQCAdjtdjidTiQkJKCwsBCZmfxc+2jBsCAi2blcLphMJnR1dSE7Oxt6vR7x8fFyj0WXYFgQkWz8fj9sNhsaGxuRmJiI4uJipKenyz0WXQXDgohk0dbWBrPZDK/Xi9zcXOh0OlZ1RDGGBRFF1KXFf0lJSRgyZAhSU1PlHotCYFgQUUQIIdDa2gqz2Qy/38/ivwGGYUFEkvN6vTCbzWhra0NycjJKSkqQnJws91jUBwwLIpKMEAJNTU2wWq0s/hvgGBZEJAkW/8UWhgURhZUQAg0NDbDZbFAoFDAYDMjOzuZqYoBjWBBR2Hg8HphMJrjdbmRkZCAvL4/FfzGCYUFE1y0QCMDpdAaL//Lz85GVlcXVRAxhWBDRdeno6IDJZEJnZyeysrKQl5fH4r8YxL9RIuoXFv8NLgwLIuqz9vZ2mM1mFv8NIgwLIuo1v98Pq9WKpqYmFv8NMgwLIuqVtrY2mEwm+Hw+Fv8NQgwLIromn88Hi8WClpYWJCUlobCwkMV/gxDDggAAAZ8Hvtaz8LusiEtSISGrGPFJKrnHIhkJIdDS0gKLxYJAIACtVgu1Ws3VxCDFsCD4O1vQfuxXaPx4KRDwAQBSh81G7uTXkJBhlHk6ksOlxX8pKSkwGo0s/hvkGBYEb8NxNP7pmR7bOk69hxTjN5B58/dlmorkcHnxn16vR25uLi+uI3A9SXCf23HV7a1Hfg6/uyHC05BcOjs7cfbsWZjNZqSkpKC0tJQNsRTElQVBkZDyFduTgDg+RWIdi/+oN7iyIKQU3QsornwqZI3/EeKTsmSYiCLF4/HgzJkzsFqtSE9Px7Bhw5CTk8OgoCswLAiJOTdCV74V8al6ABdXGtlffxkp+d+UdzCSTCAQgM1mw+nTp9HV1YWCggIUFhYiMTFR7tEoSvEYAyEuIQmpQ+6D4eF98LsbEKfMQEJGERTxfHrEIhb/UX/wGUJBCRkFSMgokHsMkkj3aqKhoQEJCQkoKipCRkaG3GPRACHpYaiamhrMmDEDU6dOxaZNm664/8yZM5g3bx5mzpyJxx9/HC0tLVKOQzRotbe349SpU2hoaEBOTg6GDRvGoKA+kSwsbDYbqqursXnzZmzfvh3vvvsu6urqgvcLIfD9738fVVVV+OCDD3DjjTdiw4YNUo1DNCj5/X6YTCacPXsWCoUCJSUlMBgMbIilPpPsMNT+/fsxceJEqFQqAMC0adNQW1uLp556CgBw9OhRpKamYvLkyQCAhQsXorW1VapxiAad1tZWmM1m+Hw+qNVqaLVaVnVQv0kWFna7HRqNJnhbq9XiyJEjwdvnz5+HWq3G0qVLcezYMdxwww1YsWKFVOMQDRqXF/8VFRUhJeXq19IQ9ZZkv2YIIa7Ydum52z6fDwcPHsQjjzyCmpoaFBQU4JVXXpFqHKKYJ4RAc3MzTp06hdbWVmi1WgwdOpRBQWEhWVjodDo4nc7gbbvdDq1WG7yt0WhQVFSEMWPGAADKysp6rDyIqPe6urpw7tw51NfXQ6lUYujQoTzsRGEl2TNp0qRJOHDgABobG+F2u7Fz587g+xMAcMstt6CxsREnTpwAAOzZswejRo2SahyimNRd1VFXVweXywW9Xo8hQ4awIZbCTrL3LHQ6HZYsWYL58+fD6/Vi9uzZGDt2LKqqqrB48WKMGTMGP/nJT7B8+XK43W7o9Xq8+uqrUo1DFHM6OzthMpnQ0dGBtLQ0GI1GKJVKuceiGKUQV3tzIcrV19fj7rvvxu7du5Gfny/3OEQRJYSA0+mE3W6HQqFAXl4eVCoV+5wopOt57eQV3EQDiNvthslkgsfjQUZGBgwGA/ucKCIYFkQDQCAQgMPhgMPhQHx8PAoKCpCZmcnVBEUMw4Ioyl1a/KdSqaDX61n8RxHHZxxRlPL7/bDb7WhoaEBiYiKL/0hWDAuiKNTe3g6TyQSv14ucnBzodDr2OZGsGBZEUcTv98NisaC5uRlKpRIlJSVIS0uTeywihgVRtGDxH0UzhgWRzHw+H8xmM1pbW5GcnMziP4pKDAsimXQX/1mtVgQCAWi1Wmg0Gp4OS1GJYUEkg66uLpjNZrS3tyM1NRVGoxFJSUlyj0X0lRgWRBEkhEBjYyNsNhsAIC8vDzk5OVxNUNQLGRZer5d1AkRhcGnxX3p6OgwGA4v/aMAIearFgw8+GIk5iGKWEAIOhwN1dXXo7OyE0WhEUVERg4IGlJAri+TkZFitVuj1+kjMQxRTLi3+y8zMRF5eHlfqNCCFDAu32427774ber0eqampwe01NTWSDkY0kAUCAdjtdjidTiQkJKCgoABZWVlyj0XUbyHDYtmyZZGYgyhmuFwumEwmdHV1sfiPYkbIZ/Btt92GU6dOYd++fYiLi8PkyZNRXFwcgdGIBha/3w+bzYbGxkYW/1HMCfkG929+8xvMnz8fx44dw5EjR1BZWYkPP/wwErMRDRhtbW2oq6tDY2MjcnNzUVpayqCgmBJyZfFf//Vf2LZtG3Q6HQDAbDbjySefxIwZMyQfjija+Xw+WK1WNDc3IykpicV/FLNChkV6enowKADw3HCif2ppaYHFYoHP54NGo4FGo2HxH8WskGExfvx4rFy5EnPnzkV8fDy2b9+O4uJiHD16FAAwatQoyYckiiZerxcWi4XFfzSohAyLt99+G3q9Hh9//HFwm9vtxqeffgqFQoHdu3dLOiBRtOgu/rNYLBBCQKfTQa1Ws6qDBoWvDIvm5mYAwNChQ/H2229DCAGFQgGv14vKykr84Q9/iNSMRLLr6uqCyWSCy+Vi8R8NSl8ZFs888wz27dsHhUKB22+/Pbg9Pj4eU6dOjchwdFF3UH/VbZIOi/+ILvrKsHjrrbcAAM8//zzWrl0bsYGop1WrVqG5uRnV1dVQKBQQQmDJkiVQqVRYtWqV3OPFNI/HA7PZzOI/IvTiPQsGRWQ5XZ1o9fiQpkxAUrwC//+CDe/8358BAKqrq7FkyRKsW7cO33lsAdo9XqQns2co3LqL/xwOB+Li4mA0GqFSqbiaoEGNHQRRwusP4C/nmvD0ts9xxNyGobmp+PHMUVBNX4gHhcC6deuwbt06AEDFo08i//6nkKrkX1+4XV78ZzAYWNVBhF5cwU2RcdTahrt/dgBHzG0AgNMNHfj2L/+Kh242Qlu2qMfXFs16Gq+VjQJ/0Q2fQCAAq9WK06dPw+fzobCwEIWFhQwKon9iWESJLZ+Z4QuIK7av/vgzHPzvF3ts8+1+E61eNxYd2IqACERqxJjlcrlQV1cHp9OJ7OxsDBs2DJmZmXKPRRRV+GtTlLjQ5L5imxACn2/8Gax/2o4RD9yHEyVVeLDxI6xfvx47zSex85ebEadg3vfX5cV/xcXFSE9Pl3ssoqjEsIgS37nFgE1/N/XYplAoMH6oEUnFT8Jfdj9mlwC2c1UoajqD6cNuQkF6tkzTDnxtbW0wm83wer3Izc2FTqdjVQfRNTAsosT4fBUeusWAd/5uDm6bPCQHj0x6FgfPN+G1slF49+ynWHNkE+bP+9+YPe4GGacduHw+HywWC1paWpCUlIQhQ4b0+FAvIro6hkWUyMtMxvr7x+AHdwxBfYsbuowkaNKSsPHgGay64QTafEPxUf1x/HTSbFxob8bWI2YM9x1Gln4UEtINco8f9YQQaG1thdlsht/vZ/EfUR8xLKKIOk0JdZoSE/Dl4aU1ZWNx4e/78b8+eg2v3bMY+Wkq/P7CMexr/RNgPof4offIOPHA4PV6YTab0dbWhpSUFBiNRiQnJ8s9FtGAwl+rolxLlxvL2jLxolGNzNP/DQCYEjBjbOdZrEv/przDRbnuqo5Tp06hvb0der0eQ4YMYVAQ9YOkYVFTU4MZM2Zg6tSp2LRp01d+3d69ezFlyhQpRxmw3D4v1oybgRvHX7zWwva7OXCf3405017B/YWjIXDl6bZ0sfjv7NmzMJvNSE5ORmlpKRtiia6DZIehbDYbqqur8f7770OpVOKhhx7ChAkTUFpa2uPrnE4n/v3f/12qMQY8feqX5/snZpWg6cIfkVJ4NxQKBb6mzpdxsugkhEBDQwNsNhsUCgUMBgOys7MZEkTXSbKVxf79+zFx4kSoVCqkpqZi2rRpqK2tveLrli9fjqeeekqqMWKG+9wuuM/vRuFCGwAFWo+8KfdIUcfj8eDMmTOwWq1IT0/HsGHD2BBLFCaShYXdbodGowne1mq1wZrnbr/61a8wcuRI3HTTTVKNERO6gyL7jjVQKBTIHFsFAAyMfwoEArDb7Th9+jS6urqQn5+PwsJCJCayZJEoXCQLCyGuPJZ+6W94J0+exM6dO7Fo0aIrvo6+JPxd6LQdDgZFt8yxVYAiDr520zW+O/Z1dHTg9OnTsNvtyMzMxLBhw9gQSyQByd6z0Ol0OHToUPC23W6HVqsN3q6trYXD4UBFRQW8Xi/sdjsqKyuxefNmqUYakBTxSqhuW3rV+zLHPB7haaJH92rC6XQiISEBhYWF7HMikpBkK4tJkybhwIEDaGxshNvtxs6dOzF58uTg/YsXL8aOHTuwfft2bNiwAVqtlkFBvdLe3s7iP6IIkywsdDodlixZgvnz5+P+++9HWVkZxo4di6qqKnz++edS/ViKYX6/HyaTCWfPngUAFBcXw2g0Ij4+Xt7BiAYBSa/gLi8vR3l5eY9tb7555Zuy+fn52LNnj5Sj0ADXXdXh8/lY/EckA9Z9UFS7vPivsLCQxX9EMmBYUFQSQqClpQUWiwWBQABarRZqtZqrCSKZMCwo6rD4jyj6MCwoaggh0NTUBKvVCiEE9Ho9cnNzec0EURRgWFBU6OzshNlshsvlQlpaGgwGA5KSkuQei4j+iWFBsmLxH9HAwLAg2Xg8HphMJrjdbmRkZMBgMLDPiShKMSwo4gKBABwOBxwOB+Lj41FQUIDMzEyuJoiiGMOCIqqjowMmkwmdnZ3IyspCXl4eEhL4NCSKdvxXShERCARgs9nQ0NCAhIQEFBUVISMjQ+6xiKiXGBYkufb2dphMJni9XuTk5ECn07HPiWiAYViQZPx+P6xWK5qamqBUKlFSUoK0tDS5xyKifmBYkCQuLf5Tq9XQarWs6iAawBgWFFaXFv8lJyejqKgIKSkpco9FRNeJYUFhcbXiP41Gw9NhiWIEw4KuW1dXF8xmM9rb21n8RxSjGBbUb0IINDY2wmazsfiPKMYxLKhfOjs7YTKZ0NHRgbS0NBiNRiiVSrnHIiKJMCyoT4QQcDqdsNvtUCgUMBqNUKlUXE0QxTiGBfWa2+2GyWSCx+NBZmYm8vLyWPxHNEgwLCgkFv8REcOCrunS4j+VSgW9Xs/iP6JBiP/q6ar8fj/sdjsaGhqQmJjI4j+iQY5hQVdg8R8RXY5hQUF+vx8WiwXNzc0s/iOiHhgWBIDFf0R0bQyLQc7r9cJisaC1tZXFfzRgeb1e1NfXw+PxyD1KVIiPj4dKpYJarQ7bL30Mi0FKCIHm5mZYrVYEAgHodDqo1WqeDksDUn19PTIyMlBcXDzon8NCCHi9XthsNtTX16OwsDAsj8vjDINQV1cXzp07B5PJhKSkJJSWlrIhlgY0j8fDXrJ/UigUUCqVMBqNcLlcYXtcriwGkUuL/wAgLy8POTk5/AdGMYHP457C/Z4jw2KQuLT4Lz09HQaDgcV/RNRrDIsYd2nxX1xcHIv/iKLMnj17cO7cOTz22GNyj3JNDIsYxuI/ouh39OhRuUfoFYZFDAoEArDb7XA6nUhISEBBQQGysrLkHosoqmw+XI8XPjqBC81uFKhSsGb6CFSOyw/LYwsh8Prrr2PXrl2Ij4/HnDlzcOONN6K6uhoejwctLS149tlnMWzYMLzzzjsAAIPBgIqKirD8fClIGhY1NTX46U9/Cq/Xi+9+97uYO3duj/t37dqFN954A0II5OfnY+3atXxRu04ulwsmkwldXV1QqVTIy8tjVQfRZTYfrseT7x1Bh9cPADjf7MaT7x0BgLAERm1tLf72t7+hpqYGXq8XlZWVyM7Oxssvv4yhQ4fiwIEDWLNmDWpqavDQQw8BQFQHBSBhWNhsNlRXV+P999+HUqnEQw89hAkTJqC0tBTAxf6hVatWYevWrdDpdFi3bh3eeOMNLF++XKqRYprf74fNZkNjYyMSExNRXFyM9PR0ucciikovfHQiGBTdOrx+vPDRibCExV//+ldMnz4dSqUSSqUS27dvR2dnJ/74xz+itrYWn332WVhPa40Eya6z2L9/PyZOnAiVSoXU1FRMmzYNtbW1wfu9Xi9WrVoFnU4HABg+fDgsFotU48S0trY21NXVobGxEbm5uSgtLWVQEF3DhWZ3n7b31eU1/vX19aisrMSRI0cwevRoLFy4MCw/J5IkCwu73Q6NRhO8rdVqg+f3A0B2djbuueceABcvqNmwYUPwNvWOz+dDfX09zp07h7i4OAwZMoSHnYh6oUB19Uqbr9reV7feeiv+8Ic/wOv1wu124/HHH8epU6fwgx/8AHfeeSf27dsHv//iyiY+Ph4+ny8sP1dKkh2GEkJcse1qp2u2tbVh0aJFGDFiBB544AGpxok5LS0tMJvN8Pv90Gg00Gg0LP4j6qU100f0eM8CAFIT47Fm+oiwPP7UqVPxxRdfYNasWQgEAnj00Udx/vx53HfffUhPT8fNN98Mj8eDjo4O3HrrrVi6dCnUajXmzZsXlp8vBcnCQqfT4dChQ8HbdrsdWq22x9fY7XY8/vjjmDhxIl544QWpRokplxf/FRcXs/iPqI+635eQ6mwoAFiyZAmWLFnSY9tzzz0X/POqVasAXFyF7NmzJ2w/VyqShcWkSZPwxhtvoLGxESkpKdi5cydeeuml4P1+vx8LFy7E9OnTsWjRIqnGiBndxX8WiwVCCBb/EV2nynH5YQ2HWCfpymLJkiWYP38+vF4vZs+ejbFjx6KqqgqLFy+G1WrFsWPH4Pf7sWPHDgDA6NGjsXr1aqlGGrC6urpgMpngcrmQmpoKo9GIpKQkucciokFE0ussysvLUV5e3mPbm2++CQAYM2YMTpw4IeWPH/C6i/+sVisUCgWL/4hINryCO0p5PB6YTCa43W4W/xGR7BgWUUYIAYfDAYfDgbi4OOTn5yMrK4urCSKSFcMiilxe/GcwGK64uIeISA48MT8KBAIBWK1WnD59Gj6fD4WFhSgsLGRQEMWI5557Du+//77cY1wXvhrJ7NLiv+zsbOj1el6BTURRh2EhExb/Ecmr7cRv0LRvBfxtFxCfUYDsr7+EjBEPh+WxhRB45ZVXsHfvXmi1Wvj9ftx2223YunUrfvnLX0KhUGDUqFFYsWIF0tLS8OGHH2L9+vVISUnByJEj4ff78corr4RllnDhYSgZXF78N2zYMAYFUQS1nfgNGnZ9H/628wAE/G3n0bDr+2g78ZuwPP6OHTtw7Ngx/O53v8O6detw/vx5uFwu/OxnP8Pbb7+NmpoapKSk4D//8z/R2NiINWvWYOPGjdi6dStaWlrCMkO4MSwiyOfz4cKFC1cU/7HTiSiymvatgPB19NgmfB1o2rciLI9/8OBB3HvvvUhMTEROTg4mT54MhUKBu+66C9nZ2QCAOXPm4C9/+QsOHTqEW265BTqdDnFxcbj//vvDMkO48TBUBAgh0NrayuI/oijhb7vQp+19pVAoEAgEgrcTEhJ63AYuvi74fD7ExcVdcV804quVxLxeL86fP48LFy5AqVSitLQ0+BsEEckjPqOgT9v76vbbb0dtbS26urrQ0tKCjz/+GACwZ88eNDc3AwC2bNmCCRMm4Gtf+xo+//xz2O12CCHw4YcfRuV1VVxZSEQIgaamJlitVgghoNfrkZubG5VPAqLBJvvrL6Fh1/d7HIpSJKQi++svXeO7eu+ee+7B559/jrKyMqjVagwdOhTp6elYsGAB5s2bB6/Xi1GjRuHf/u3fkJ6ejuXLl+N73/selEol8vPzkZmZGZY5wolhIQEW/xFFt+6znqQ6Gwq4ekU5ADz44IM9bjc1NeHEiRP44IMPEBcXh5dffhlFRUVhmyNcGBZhJIRAQ0MDbDYbFAoFDAYDsrOzuZogikIZIx4Oazj0l0qlQmtrK8rKyhAfH49Ro0bhO9/5jtxjXYFhESaXFv9lZGTAYDAgMTFR7rGIKMopFAosX75c7jFCYlhcp0AgAKfTyeI/IoppDIvr0NHRAZPJhM7OTmRlZSEvL499TkQUk/jK1g+BQAA2mw0NDQ1ISEhAYWFhVJ69QEQULgyLPmpvb4fZbGbxHxENKgyLXvL7/bBarWhqaoJSqWTxHxENKryMuBdaW1tx6tQpNDU1Qa1Wo7S0lEFBRH0yZcoU1NfXyz1Gv3FlcQ0+nw8WiwUtLS1ISkpCYWEhUlNT5R6LiCjiGBZXIYRAS0sLLBYLAoEAtFot1Go1+5yIYsjm04ex7PBHuOBqRkGaCqvHTUfl0HFheWyr1Yof/ehH6OjoQFxcXPA6ip/85Cc4fvw43G43Xn31Vdx00004d+4cVq1ahebmZiQnJ2PFihUYOXIknE4nXnzxRVitVigUCjzzzDOYNGkS3njjDZw9exbnz59Hc3Mz5syZgyeeeCIsc18Lw+IyXq8XZrMZbW1tSElJgdFoRHJystxjEVEYbT59GAv2vYcOvxcAcN7VjAX73gOAsATGe++9h29+85t44okn8Mknn+Dw4cMAgNLSUqxduxa//vWv8dZbb2H9+vVYunQpXnzxRYwcORJ1dXX4l3/5F+zYsQOrV69GRUUF7r77btjtdlRWVmLbtm0AgJMnT+Kdd95BIBDArFmzcPvtt2PUqFHXPfe1MCz+icV/RIPHssMfBYOiW4ffi2WHPwpLWNx+++14+umncfz4cdx555145JFHsGnTJtxzzz0ALobGjh074HK58MUXX+D555//co6ODjQ1NWH//v04c+YM1q9fD+DLz8MBgLKyMqSlpQG4+F7IX/7yF4ZFJHR2dsJsNsPlciEtLQ1GoxFKpVLusYhIIhdczX3a3lfjxo3D73//e+zduxcffvghfvvb3wJA8DT77l9CA4EAlEoltm/fHvxeq9UKlUqFQCCAjRs3QqVSAQBsNhvUajV27drV43T9QCAQkdP3B/VBeCEEnE4n6urq4Ha7YTAYUFxczKAginEFaao+be+rV199Fdu3b8cDDzyAF198EceOHbvq12VkZKC4uDgYFvv27cPcuXMBABMnTsTmzZsBAHV1dZg5cybcbjcAYNeuXcHPyvjjH/+IO+64IyxzX8ugXVmw+I9o8Fo9bnqP9ywAIDU+EavHTQ/L48+bNw/PPPMMfvvb3yI+Ph4rV67E66+/ftWvfe2117Bq1Sr84he/QGJiIqqrq4Plgi+++CLKy8sBXAyg7lP2k5KSUFlZifb2dixYsAClpaVhmftaBl1YBAIBOBwOOBwOxMfHo6CgAJmZmXxvgmgQ6X5fQqqzofLy8oKrgm4zZswI/nnChAmYMGECAGDo0KF4++23r3gMnU6Hn//851d9/PHjx+Ppp58Oy6y9NajCgsV/RNStcui4sIXDYDAoXikvL/4rKipCRkaG3GMREfVZpFcU3WI+LNrb22EymeD1epGTkwOdTsfiPyKiPorZsLi8+K+kpCR4XjIRxR4hBN97vEQgEAjr48VkWLS2tsJsNsPn80GtVkOr1bKqgyiGJScno6GhgRfS4mJoer1e2Gy2sP6CHFNh4fP5YDab0draiuTkZBQVFSElJUXusYhIYvn5+aivr4fD4ZB7lKiQkJCArKwsqNXq8D1m2B5JRlcr/tNoNIP+NwyiwSIxMRElJSVyjxHTJD02U1NTgxkzZmDq1KnYtGnTFfcfP34cFRUVmDZtGpYtWwafz9fnn9HV1YVz586hvr4eSqUSQ4cOhVarZVAQEYWRZGFhs9lQXV2NzZs3Y/v27Xj33XdRV1fX42ueffZZrFixAjt27IAQAlu2bOnTz2hubkZdXR1cLhfy8vIwZMgQNsQSEUlAssNQ+/fvx8SJE4MlWNOmTUNtbS2eeuopAIDJZILH48HNN98MAJg1axbWr1+PysrKkI/t9/sBXFyZ5OfnQ6/Xw+12w2QySfL/QkQUC6xWK4AvX0P7QrKwsNvt0Gg0wdtarRZHjhz5yvs1Gg1sNluvHrv7TazuDxQhIqLeczgcKCoq6tP3SBYWQogrtl36PkKo+69l9OjR2LRpEzQaDS+wIyLqJb/fD4fDgdGjR/f5eyULC51Oh0OHDgVv2+12aLXaHvc7nc7gbYfD0eP+a0lOTsb48ePDNywR0SDR1xVFN8ne4J40aRIOHDiAxsZGuN1u7Ny5E5MnTw7ebzQakZSUFPy4wW3btvW4n4iIoodCXO14UJjU1NTg5z//ObxeL2bPno2qqipUVVVh8eLFGDNmDE6cOIHly5fD5XJh5MiRWLt2LT94iIgoCkkaFkREFBtYmERERCExLIiIKCSGBRERhcSwICKikKI+LCJRRjhQhNoXu3btwre//W3MnDkTixYtQktLiwxTRkaofdFt7969mDJlSgQni7xQ++LMmTOYN28eZs6ciccff3xQPy+OHj2KiooKzJw5EwsWLEBra6sMU0ZOe3s7ysrKUF9ff8V9fX7tFFHMarWKu+66SzQ1NQmXyyXKy8vFqVOnenzNfffdJ/7+978LIYR4/vnnxaZNm2SYVHqh9kVbW5v4+te/LqxWqxBCiP/4j/8QL730klzjSqo3zwshhHA4HOJb3/qWuOuuu2SYMjJC7YtAICDuvfde8ac//UkIIcRrr70mXn31VbnGlVRvnhcPP/yw2Lt3rxBCiLVr14of//jHcowaEZ9++qkoKysTo0aNEhcuXLji/r6+dkb1yuLSMsLU1NRgGWG3q5URXnp/LAm1L7xeL1atWgWdTgcAGD58OCwWi1zjSirUvui2fPnyYHFlrAq1L44ePYrU1NTgBa8LFy7E3Llz5RpXUr15XgQCAbhcLgCA2+2O6ZbqLVu2YOXKlVdtxujPa2dUh8XVyggvLRu8njLCgSbUvsjOzsY999wDAPB4PNiwYUPwdqwJtS8A4Fe/+hVGjhyJm266KdLjRVSofXH+/Hmo1WosXboU5eXlWLlyJVJTU+UYVXK9eV4899xzWLZsGe644w7s378fDz30UKTHjJjVq1d/ZS1Sf147ozoshIRlhANNb/9f29raUFVVhREjRuCBBx6IxGgRF2pfnDx5Ejt37sSiRYsiOZYsQu0Ln8+HgwcP4pFHHkFNTQ0KCgrwyiuvRHLEiAm1LzweD5YtW4aNGzfiz3/+MyorK7F06dJIjhg1+vPaGdVhcXnZYDjLCAeaUPuie1tlZSVGjBiB1atXR3rEiAm1L2pra+FwOFBRUYEnn3wyuF9iUah9odFoUFRUhDFjxgAAysrKenxUQCwJtS9OnjyJpKQkjB07FgAwZ84cHDx4MOJzRoP+vHZGdViwjPBLofaF3+/HwoULMX36dCxbtixmV1hA6H2xePFi7NixA9u3b8eGDRug1WqxefNmGSeWTqh9ccstt6CxsREnTpwAAOzZswejRo2Sa1xJhdoXRUVFsFqtOHPmDABg9+7dwRAdbPr12hnWt98l8MEHH4j77rtP3HvvvWLDhg1CCCGeeOIJceTIESGEEMePHxcVFRXiW9/6lvjXf/1X0dnZKee4krrWvti5c6cYPny4mDlzZvC/F154QeaJpRPqedHtwoULMX02lBCh98Wnn34qKioqxIwZM8T3vvc94XQ65RxXUqH2xd69e0V5ebkoKysTjz76qDh//ryc40bEXXfdFTwb6npeO1kkSEREIUX1YSgiIooODAsiIgqJYUFERCExLIiIKCSGBRERhcSwILoOK1euxJQpU1BdXS33KESS4qmzRNdhxIgR2Lt3L/R6vdyjEEmKYUHUT5WVlTh8+DBuuOEG1NXVYd68eTh06BDa2trw2GOPobKyEp988glWr16N1NRUdHR04L333oNSqZR7dKI+S5B7AKKBavPmzRg+fDg2btyI2bNnw+PxYOvWrbDZbLj//vsxbtw4AMCpU6ewa9cuGI1GmScm6j++Z0EUJpWVlVAoFNDr9fjGN76Bffv2AQDy8vIYFDTgMSyIwiQh4cuFeiAQQFzcxX9esfr5ETS4MCyIwmTbtm0AALPZjH379sVsAzINTnzPgihM6uvrMWvWLHg8HixfvhxDhgyBw+GQeyyisODZUERhMGXKFKxbt27Qfj4CxT4ehiIiopC4siAiopC4siAiopAYFkREFBLDgoiIQmJYEBFRSAwLIiIKiWFBREQh/Q9lFnjkDOeodQAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAENCAYAAAD+CUlOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8/fFQqAAAACXBIWXMAAAsTAAALEwEAmpwYAAAsoUlEQVR4nO3de3RU9b0+/mdymdyTSTK3zOQKQZCbetCC1GJFAcGEKmDRIFirCHqUlloPKiB0WcBj/R4Oerpasa5TtNDKVyyYc2ygSOm3BSqF1qJcfhCQS+Y+uWcyk8zM/vz+oBkTbpOE2bMnk+e1Vtdi9iSTd7eTefLZe88zKiGEABER0TUkKD0AERHFPoYFERGFxbAgIqKwGBZERBQWw4KIiMJiWBARUViyh0VbWxsqKipQV1d32X3Hjx/H7NmzMW3aNCxfvhyBQEDucYiIqB9kDYt//OMfePjhh3H27Nkr3v/8889j5cqV2LlzJ4QQ2Lp1q5zjEBFRP8kaFlu3bsWqVaug1+svu89iscDn8+Hmm28GAMyaNQs1NTVyjkNERP2UJOeDr1mz5qr3OZ1O6HS60G2dTgeHw9Grx/X5fPjiiy+g0+mQmJh43XMSEQ0GwWAQLpcLo0ePRmpqap++V9awuJYrtYyoVKpefe8XX3yBefPmRXokIqJBYfPmzbj11lv79D2KhYXBYIDb7Q7ddrlcVzxcdSVdK5LNmzfDaDTKMh8R0UDn9/tht9vh9XqRlpYGAPjOd77T46hObykWFmazGSkpKTh8+DDGjRuH7du3Y9KkSb363q5DT0ajEYWFhXKOSUQ04AghUF9fj+bmZuTk5GD48OHIzc2FxWIBgH4dvo/6+ywWLlyIzz//HADw+uuvY926dZg+fTq8Xi8WLFgQ7XGIiOKKz+fDmTNnYLfbkZGRgfLycuTl5fX6MP/VRGVlsWfPntC/33777dC/R4wYgQ8++CAaIxARxTVJkuB2u+FyuZCQkIDCwkLk5ORcd0h0UewwFBERRYbX6w29HSEnJwcFBQVISorsyzvDgohogJIkCU6nE263G0lJSSguLkZ2drYsP4thQUQ0AHk8HlgsFnR2diI3NxdGo1HW950xLIiIBpBgMAiHw4GGhgYkJyejtLQUmZmZsv9chgUR0QDR2toKi8WCQCCA/Px8GAwGJCRE56JWhgURUYwLBAKw2Wxobm5GSkoKiouLkZ6eHtUZGBZERDFKCIHm5mbYbDZIkgSdTgedThe11UR3DAsiohjk9/thtVrR2tqKtLQ0mM3mPpf/RRLDgogohggh0NjYCLvdDiEEjEYj8vPzI/bmuv5iWBARxYiOjg5YrVZ4PB5kZGTAZDIhJSVF6bEAMCyIiBTXVfzncDigUqlgMpmQm5ur+GqiO4YFEZGCfD4fLBYLvF4vsrKyYDKZkJycrPRYl2FYEBEpQJIkuFwuuN1uWYr/Io1hQUQUZe3t7bBYLOjo6JCt+C/SYns6IqI4IkkSHA4H6uvrZS/+izSGBRFRFLS1tcFiscDv9yMvLw8Gg0HW4r9IY1gQEckoGAzCbrejsbERarUaZWVlyMjIUHqsPmNYEBHJpKWlBVarFYFAAFqtFnq9XpGqjkhgWBARRdilxX8lJSVIS0tTeqzrwrAgIoqQS4v/9Ho9tFrtgF1NdMewICKKgM7OTthstpgp/os0hgUR0XWI1eK/SGNYEBH1U0dHBywWC9rb25GRkQGz2Qy1Wq30WLJgWBAR9ZEQAm63G06nEyqVCmazGRqNJu5WE90xLK6D2+fBl631aOr0wZyejbKsPKQlxedfFUR0kdfrhcVigc/ni+niv0hjWPTTubYGLPzz/8UntlMAgERVAv7P1yrxaPltyFbHz0ktIrqoq/jP5XIhMTERRUVFyM7OjuvVRHcMi34QQuBXtX8LBQUABIWE73+6AxN0JbhNV6zgdEQUad2L/zQaDYxGY8wX/0XawL/4VwEObyt+cfLTK973/+xnojwNEclFkiTYbDacOXMGkiShpKQEhYWFgy4oAK4s+iUpIREZVzk3kZUcGx+BSETXZ6AX/0UaVxb9oE3NwAtjJ1+2XZ2QiImG0ugPREQREwwGUVdXh7Nnz0KlUqGsrAwmk2lQBwXAsOi3qeYbsHbcjNAKoywzDx9PXYgROXqFJyOi/mppacGpU6fQ1NQErVaL8vLyAdkQKwcehuonfVoWnht9Jx4suwkefye0qRkoSB8YH2JCRD0FAgFYrVa0tLQgNTU1Lor/Io1hcR2SEhIxJCtf6TGIqJ+EEGhqaoLdbg8V/+l0ukFzOWxfMCyIaFDq7OyE1WpFW1sb0tPTYTabkZLCC1SuhmFBRIOKEAINDQ1wOBwAgIKCAuTl5XE1EYasJ7irq6sxY8YMTJkyBZs3b77s/qNHj2L27NmYOXMmFi1ahJaWFjnHIaJBrqOjA19++SVsNhvS0tJQXl4elw2xcpAtLBwOB9avX48tW7Zgx44deP/991FbW9vja9asWYMlS5bgo48+QllZGd555x25xiGiQUwIAZfLhdraWvh8PpjNZpSWlsZtQ6wcZAuL/fv3Y8KECdBoNEhPT8e0adNQU1PT42skSYLH4wFwsZwrnj4ohIhig9frxenTp+FwOJCVlYVhw4YhNzeXq4k+ku2chdPphE6nC93W6/U4cuRIj6954YUX8Nhjj2Ht2rVIS0vD1q1b5RqHiAaZ7sV/SUlJKCoqQk5OjtJjDViyrSyEEJdt657kPp8Py5cvx6ZNm/DnP/8ZVVVVWLZsmVzjENEg4vF4UFtbC5fLBY1Gg/LycgbFdZItLAwGA9xud+i20+mEXv/Vu5tPnjyJlJQUjB07FgAwd+5cHDx4UK5xiGgQCAaDsFqt+PLLLyGEGNTFf5EmW1hMnDgRBw4cQENDA7xeL3bt2oVJkyaF7i8pKYHdbseZMxdbWj/55BOMGTNGrnGIKM61traitrYWDQ0NyMvLQ3l5ObKyspQeK27IFrcGgwFLly7FggUL4Pf7MWfOHIwdOxYLFy7EkiVLMGbMGKxbtw7f//73IYRAfn4+1q5dK9c4RBSnAoEA7HY7mpqaoFarUVZWxj4nGci6NqusrERlZWWPbW+//Xbo33feeSfuvPNOOUcgojjW3NwMm82GQCAAnU4HnU6HhAT2o8qBB/KIaMDx+/2w2Wws/osihgURDRhdxX82mw1CCBgMBmi1Wr5nIgoYFkQ0IHR2dsJiscDj8bD4TwEMCyKKaSz+iw0MCyKKWT6fD1arFe3t7cjMzITJZGKfk0IYFkQUc7qK/1wuFxISEmA2m6HRaLiaUBDDgohiitfrhcVigc/nQ3Z2NkwmE9+BHQP4X4CIYoIkSXA6nXC73UhKSkJxcTGys/m59rGCYUFEivN4PLBYLOjs7ERubi6MRiMSExOVHou6YVgQkWKCwSAcDgcaGhqQnJyM0tJSZGZmKj0WXQHDgogU0draCqvVCr/fj/z8fBgMBlZ1xDCGBRFFVffiv5SUFAwZMgTp6elKj0VhMCyIKCqEEGhpaYHVakUwGGTx3wDDsCAi2fn9flitVrS2tiI1NRVlZWVITU1VeizqA4YFEclGCIHGxkbY7XYW/w1wDAsikgWL/+ILw4KIIkoIgfr6ejgcDqhUKphMJuTm5nI1McAxLIgoYnw+HywWC7xeL7KyslBQUMDivzjBsCCi6yZJEtxud6j4r7CwEDk5OVxNxBGGBRFdl/b2dlgsFnR0dCAnJwcFBQUs/otD/C9KRP3C4r/BhWFBRH3W1tYGq9XK4r9BhGFBRL0WDAZht9vR2NjI4r9BhmFBRL3S2toKi8WCQCDA4r9BiGFBRNcUCARgs9nQ3NyMlJQUFBcXs/hvEGJYEABACvgQaDmLoMeOhBQNknJKkZiiUXosUpAQAs3NzbDZbJAkCXq9HlqtlquJQYphQQh2NKPt2Lto+NMyQAoAANKHzUH+pJ8gKcus8HSkhO7Ff2lpaTCbzSz+G+QYFgR//XE0/PG5HtvaT32ANPM3kH3zUwpNRUq4tPjPaDQiPz+fb64jcD1J8J7becXtLUfeQtBbH+VpSCkdHR04e/YsrFYr0tLSUF5ezoZYCuHKgqBKSrvK9hQggU+ReMfiP+oNriwIaSVTAdXlT4WcW3+IxJQcBSaiaPH5fDhz5gzsdjsyMzMxbNgw5OXlMSjoMgwLQnLejTBUbkNiuhHAxZVG7td/jLTCbyo7GMlGkiQ4HA6cPn0anZ2dKCoqQnFxMZKTk5UejWIUjzEQEpJSkD7kPpge3oegtx4J6iwkZZVAlcinRzxi8R/1B58hFJKUVYSkrCKlxyCZdK0m6uvrkZSUhJKSEmRlZSk9Fg0Qsh6Gqq6uxowZMzBlyhRs3rz5svvPnDmD+fPnY+bMmXj88cfR3Nws5zhEg1ZbWxtOnTqF+vp65OXlYdiwYQwK6hPZwsLhcGD9+vXYsmULduzYgffffx+1tbWh+4UQeOqpp7Bw4UJ89NFHuPHGG7Fx40a5xiEalILBICwWC86ePQuVSoWysjKYTCY2xFKfyXYYav/+/ZgwYQI0Gg0AYNq0aaipqcEzzzwDADh69CjS09MxadIkAMDixYvR0tIi1zhEg05LSwusVisCgQC0Wi30ej2rOqjfZAsLp9MJnU4Xuq3X63HkyJHQ7fPnz0Or1WLZsmU4duwYbrjhBqxcuVKucYgGjUuL/0pKSpCWduX30hD1lmx/ZgghLtvW/drtQCCAgwcP4pFHHkF1dTWKiorw6quvyjUOUdwTQqCpqQmnTp1CS0sL9Ho9hg4dyqCgiJAtLAwGA9xud+i20+mEXq8P3dbpdCgpKcGYMWMAABUVFT1WHkTUe52dnTh37hzq6uqgVqsxdOhQHnaiiJLtmTRx4kQcOHAADQ0N8Hq92LVrV+j8BADccsstaGhowIkTJwAAe/bswahRo+QahygudVV11NbWwuPxwGg0YsiQIWyIpYiT7ZyFwWDA0qVLsWDBAvj9fsyZMwdjx47FwoULsWTJEowZMwY//elPsWLFCni9XhiNRrz22mtyjUMUdzo6OmCxWNDe3o6MjAyYzWao1Wqlx6I4pRJXOrkQ4+rq6nD33Xfjk08+QWFhodLjEEWVEAJutxtOpxMqlQoFBQXQaDTsc6Kwrue1k+/gJhpAvF4vLBYLfD4fsrKyYDKZ2OdEUcGwIBoAJEmCy+WCy+VCYmIiioqKkJ2dzdUERQ3DgijGdS/+02g0MBqNLP6jqOMzjihGBYNBOJ1O1NfXIzk5mcV/pCiGBVEMamtrg8Vigd/vR15eHgwGA/ucSFEMC6IYEgwGYbPZ0NTUBLVajbKyMmRkZCg9FhHDgihWsPiPYhnDgkhhgUAAVqsVLS0tSE1NZfEfxSSGBZFCuor/7HY7JEmCXq+HTqfj5bAUkxgWRAro7OyE1WpFW1sb0tPTYTabkZKSovRYRFfFsCCKIiEEGhoa4HA4AAAFBQXIy8vjaoJiXtiw8Pv9rBMgioDuxX+ZmZkwmUws/qMBI+ylFg8++GA05iCKW0IIuFwu1NbWoqOjA2azGSUlJQwKGlDCrixSU1Nht9thNBqjMQ9RXOle/JednY2CggKu1GlAChsWXq8Xd999N4xGI9LT00Pbq6urZR2MaCCTJAlOpxNutxtJSUkoKipCTk6O0mMR9VvYsFi+fHk05iCKGx6PBxaLBZ2dnSz+o7gR9hn8ta99DadOncK+ffuQkJCASZMmobS0NAqjEQ0swWAQDocDDQ0NLP6juBP2BPevf/1rLFiwAMeOHcORI0dQVVWFjz/+OBqzEQ0Yra2tqK2tRUNDA/Lz81FeXs6goLgSdmXxy1/+Etu3b4fBYAAAWK1WPPnkk5gxY4bswxHFukAgALvdjqamJqSkpLD4j+JW2LDIzMwMBQUAXhtO9E/Nzc2w2WwIBALQ6XTQ6XQs/qO4FTYsbr31VqxatQrz5s1DYmIiduzYgdLSUhw9ehQAMGrUKNmHJIolfr8fNpuNxX80qIQNi/feew9GoxF/+tOfQtu8Xi8+++wzqFQqfPLJJ7IOSBQruor/bDYbhBAwGAzQarWs6qBB4aph0dTUBAAYOnQo3nvvPQghoFKp4Pf7UVVVhd///vfRmpFIcZ2dnbBYLPB4PCz+o0HpqmHx3HPPYd++fVCpVLj99ttD2xMTEzFlypSoDEcXdQX11W6TfFj8R3TRVcPinXfeAQC8+OKLWLduXdQGop5Wr16NpqYmrF+/HiqVCkIILF26FBqNBqtXr1Z6vLjm8/lgtVpZ/EeEXrzPgkERXW5PB065W/HK4T1o9HTg/7vgwIYNG7B06VIIITDz8QXYsGEDjp+3o83nV3rcuCSEgNPpxOnTp1n8R/RP7CCIEf6ghL+ca8Sz2z/HEWsrCgo8OOLaAePUR/CgENiwYQM2bNgAAJj16JMo/tYipCUEALCULpIuLf4zmUys6iBCL1YWFB1H7a24++cHcMTaCgCw2TKw7UA7vMHNGDH13h5fO+SBxfi39LcgeS4oMWpckiQJdrsdp0+fRiAQQHFxMYqLixkURP/E34QYsfUfVgQk0XOjJx+nfHdj2/s/6rG5ZdMCZP38PSTn3hDFCeNX9+K/3NxcGI1GJCYmKj0WUUxhWMSIC43ey7YJIfD5r95B0x8+w8hpYxC8+XHc9sVb2PjbL5BW/MvQSW/qn0uL/0pLS5GZman0WEQxiWERI759iwmb/27psU2VXQ+dIQN3Pfokku6bgTz3etw48jF06E9Do9EwKK5Da2srrFYr/H4/8vPzYTAYWNVBdA0Mixhxa6EGD91iwm/+br24IcuNkgIJP5q5HofPu/Bv6W/h4IjnsOvEToyYei+mTpik7MADVCAQgM1mQ3NzM1JSUjBkyJAeH+pFRFfGsIgRBdmpeOP+MfjeHUNwvsmDv7adwHfLx+Pdv36Jf0t/C9k3P4X7dDchOacMOV+8id2HEzDGOANZqfxP2BtCCLS0tMBqtSIYDLL4j6iP+EoTQ7QZamgz1BhfkosHUQghBH6Q+jNk3fQUUnQ3AQCmFo+GZNqAIXueRapvJJBarvDUsc/v98NqtaK1tRVpaWkwm81ITU1VeiyiAYVhEcNUKhVy71iLxNTcHtsTklKRP/lNQEgKTTYwCCHQ2NgIu90OIQSMRiPy8/N5roeoH2Rdg1dXV2PGjBmYMmUKNm/efNWv27t3LyZPniznKAPWpUHRJSEpFQnJPNZ+NZ2dnTh79iysVitSU1NRXl7Ohlii6yDbysLhcGD9+vX48MMPoVar8dBDD2H8+PEoL+952MTtduPf//3f5RqDBhkhBOrr6+FwOKBSqWAymZCbm8uQILpOsq0s9u/fjwkTJkCj0SA9PR3Tpk1DTU3NZV+3YsUKPPPMM3KNQYOIz+fDmTNnYLfbkZmZiWHDhrEhlihCZFtZOJ1O6HS60G29Xo8jR470+Jp3330XI0eOxE033STXGDQISJIEt9sNl8uFhIQEFBYWIicnhyFBFEGyhYUQ4rJt3X95T548iV27duGXv/wl7Ha7XGNQnGtvb4fFYkFHRwdycnJQUFDAPiciGcj2W2UwGHDo0KHQbafTCb1eH7pdU1MDl8uF2bNnw+/3w+l0oqqqClu2bJFrJIojkiTB6XTC7XYjKSkJxcXFyM7OVnosorgl2zmLiRMn4sCBA2hoaIDX68WuXbswadJX7zpesmQJdu7ciR07dmDjxo3Q6/UMCuqVtrY21NbWwu12Izc3F8OGDWNQEMlMtrAwGAxYunQpFixYgPvvvx8VFRUYO3YsFi5ciM8//1yuH0txLBgMwmKx4OzZswCA0tJSmM1mNsQSRYGsB3crKytRWVnZY9vbb7992dcVFhZiz549co5CA1xXVUcgEGDxH5ECeCaQYtqlxX/FxcUs/iNSAMOCYpIQAs3NzbDZbJAkCXq9HlqtlqsJIoUwLCjmsPiPKPYwLChmsPiPKHYxLCgmdHR0wGq1wuPxICMjAyaTCSkpKUqPRUT/xLAgRbH4j2hgYFiQYnw+HywWC7xeL7KysmAymZCcnKz0WER0BQwLijpJkuByueByuZCYmIiioiJkZ2dzNUEUwxgWFFUs/iMamPhbSlEhSRIcDgfq6+uRlJSEkpISZGVlKT0WEfUSw4Jk19bWBovFAr/fj7y8PBgMBvY5EQ0wDAuSTTAYhN1uR2NjI9RqNcrKypCRkaH0WETUDwwLkkX34j+tVgu9Xs+qDqIBjGFBEdW9+C81NRUlJSVIS0tTeiwiuk4MC4qIKxX/6XQ6Xg5LFCcYFnTdOjs7YbVa0dbWxuI/ojjFsKB+E0KgoaEBDoeDxX9EcY5hQf3S0dEBi8WC9vZ2ZGRkwGw2Q61WKz0WEcmEYUF9IoSA2+2G0+mESqWC2WyGRqPhaoIozjEsqNe8Xi8sFgt8Ph+ys7NRUFDA4j+iQYJhQWGx+I+IGBZ0Td2L/zQaDYxGI4v/iAYh/tbTFQWDQTidTtTX1yM5OZnFf0SDHMOCLsPiPyK6FMOCQoLBIGw2G5qamlj8R0Q9MCwIAIv/iOjaGBaDnN/vh81mQ0tLC4v/aMDy+/2oq6uDz+dTepSYkJiYCI1GA61WG7E/+hgWg5QQAk1NTbDb7ZAkCQaDAVqtlpfD0oBUV1eHrKwslJaWDvrnsBACfr8fDocDdXV1KC4ujsjj8jjDINTZ2Ylz587BYrEgJSUF5eXlbIilAc3n87GX7J9UKhXUajXMZjM8Hk/EHpcri0Gke/EfABQUFCAvL4+/YBQX+DzuKdLnHBkWg0T34r/MzEyYTCYW/xFRrzEs4lz34r+EhAQW/xHFmD179uDcuXN47LHHlB7lmhgWcYzFf0Sx7+jRo0qP0CsMizgkSRKcTifcbjeSkpJQVFSEnJwcpcciiilbDtfhpd+dwIUmL4o0aVg7fQSqxhVG5LGFEHj99dexe/duJCYmYu7cubjxxhuxfv16+Hw+NDc34/nnn8ewYcPwm9/8BgBgMpkwe/bsiPx8OcgaFtXV1fjZz34Gv9+P73znO5g3b16P+3fv3o0333wTQggUFhZi3bp1fFG7Th6PBxaLBZ2dndBoNCgoKGBVB9Elthyuw5MfHEG7PwgAON/kxZMfHAGAiARGTU0N/va3v6G6uhp+vx9VVVXIzc3Fj3/8YwwdOhQHDhzA2rVrUV1djYceeggAYjooABnDwuFwYP369fjwww+hVqvx0EMPYfz48SgvLwdwsX9o9erV2LZtGwwGAzZs2IA333wTK1askGukuBYMBuFwONDQ0IDk5GSUlpYiMzNT6bGIYtJLvzsRCoou7f4gXvrdiYiExV//+ldMnz4darUaarUaO3bsQEdHB/7whz+gpqYG//jHPyJ6WWs0yPY+i/3792PChAnQaDRIT0/HtGnTUFNTE7rf7/dj9erVMBgMAIDhw4fDZrPJNU5ca21tRW1tLRoaGpCfn4/y8nIGBdE1XGjy9ml7X11a419XV4eqqiocOXIEo0ePxuLFiyPyc6JJtrBwOp3Q6XSh23q9PnR9PwDk5ubinnvuAXDxDTUbN24M3abeCQQCqKurw7lz55CQkIAhQ4bwsBNRLxRprlxpc7XtfXXbbbfh97//Pfx+P7xeLx5//HGcOnUK3/ve93DnnXdi3759CAYvrmwSExMRCAQi8nPlJNthKCHEZduudLlma2srnn76aYwYMQIPPPCAXOPEnebmZlitVgSDQeh0Ouh0Ohb/EfXS2ukjepyzAID05ESsnT4iIo8/ZcoUfPHFF5g1axYkScKjjz6K8+fP47777kNmZiZuvvlm+Hw+tLe347bbbsOyZcug1Woxf/78iPx8OcgWFgaDAYcOHQrddjqd0Ov1Pb7G6XTi8ccfx4QJE/DSSy/JNUpcubT4r7S0lMV/RH3UdV5CrquhAGDp0qVYunRpj20vvPBC6N+rV68GcHEVsmfPnoj9XLnIFhYTJ07Em2++iYaGBqSlpWHXrl145ZVXQvcHg0EsXrwY06dPx9NPPy3XGHGjq/jPZrNBCMHiP6LrVDWuMKLhEO9kXVksXboUCxYsgN/vx5w5czB27FgsXLgQS5Ysgd1ux7FjxxAMBrFz504AwOjRo7FmzRq5RhqwOjs7YbFY4PF4kJ6eDrPZjJSUFKXHIqJBRNb3WVRWVqKysrLHtrfffhsAMGbMGJw4cULOHz/gdRX/2e12qFQqFv8RkWL4Du4Y5fP5YLFY4PV6WfxHRIpjWMQYIQRcLhdcLhcSEhJQWFiInJwcriaISFEMixhyafGfyWS67M09RERK4IX5MUCSJNjtdpw+fRqBQADFxcUoLi5mUBDFiRdeeAEffvih0mNcF74aKax78V9ubi6MRiPfgU1EMYdhoRAW/xEpq/XEr9G4byWCrReQmFWE3K+/gqwRD0fksYUQePXVV7F3717o9XoEg0F87Wtfw7Zt2/Df//3fUKlUGDVqFFauXImMjAx8/PHHeOONN5CWloaRI0ciGAzi1VdfjcgskcLDUAq4tPhv2LBhDAqiKGo98WvU734KwdbzAASCredRv/sptJ74dUQef+fOnTh27Bj+53/+Bxs2bMD58+fh8Xjw85//HO+99x6qq6uRlpaG//qv/0JDQwPWrl2LTZs2Ydu2bWhubo7IDJHGsIiiQCCACxcuXFb8x04nouhq3LcSItDeY5sItKNx38qIPP7BgwcxdepUJCcnIy8vD5MmTYJKpcJdd92F3NxcAMDcuXPxl7/8BYcOHcItt9wCg8GAhIQE3H///RGZIdJ4GCoKhBBoaWlh8R9RjAi2XujT9r5SqVSQJCl0Oykpqcdt4OLrQiAQQEJCwmX3xSK+WsnM7/fj/PnzuHDhAtRqNcrLy0N/QRCRMhKzivq0va9uv/121NTUoLOzE83NzfjTn/4EANizZw+ampoAAFu3bsX48ePxL//yL/j888/hdDohhMDHH38ck++r4spCJkIINDY2wm63QwgBo9GI/Pz8mHwSEA02uV9/BfW7n+pxKEqVlI7cr79yje/qvXvuuQeff/45KioqoNVqMXToUGRmZmLRokWYP38+/H4/Ro0ahR/96EfIzMzEihUr8N3vfhdqtRqFhYXIzs6OyByRxLCQAYv/iGJb11VPcl0NBVy5ohwAHnzwwR63GxsbceLECXz00UdISEjAj3/8Y5SUlERsjkhhWESQEAL19fVwOBxQqVQwmUzIzc3laoIoBmWNeDii4dBfGo0GLS0tqKioQGJiIkaNGoVvf/vbSo91GYZFhHQv/svKyoLJZEJycrLSYxFRjFOpVFixYoXSY4TFsLhOkiTB7Xaz+I+I4hrD4jq0t7fDYrGgo6MDOTk5KCgoYJ8TEcUlvrL1gyRJcDgcqK+vR1JSEoqLi2Py6gUiokhhWPRRW1sbrFYri/+IaFBhWPRSMBiE3W5HY2Mj1Go1i/+IaFDh24h7oaWlBadOnUJjYyO0Wi3Ky8sZFETUJ5MnT0ZdXZ3SY/QbVxbXEAgEYLPZ0NzcjJSUFBQXFyM9PV3psYiIoo5hcQVCCDQ3N8Nms0GSJOj1emi1WvY5EcWRLacPY/nh3+GCpwlFGRqsGTcdVUPHReSx7XY7fvjDH6K9vR0JCQmh91H89Kc/xfHjx+H1evHaa6/hpptuwrlz57B69Wo0NTUhNTUVK1euxMiRI+F2u/Hyyy/DbrdDpVLhueeew8SJE/Hmm2/i7NmzOH/+PJqamjB37lw88cQTEZn7WhgWl/D7/bBarWhtbUVaWhrMZjNSU1OVHouIImjL6cNYtO8DtAf9AIDzniYs2vcBAEQkMD744AN885vfxBNPPIFPP/0Uhw8fBgCUl5dj3bp1+NWvfoV33nkHb7zxBpYtW4aXX34ZI0eORG1tLf71X/8VO3fuxJo1azB79mzcfffdcDqdqKqqwvbt2wEAJ0+exG9+8xtIkoRZs2bh9ttvx6hRo6577mthWPwTi/+IBo/lh38XCoou7UE/lh/+XUTC4vbbb8ezzz6L48eP484778QjjzyCzZs345577gFwMTR27twJj8eDL774Ai+++OJXc7S3o7GxEfv378eZM2fwxhtvAPjq83AAoKKiAhkZGQAungv5y1/+wrCIho6ODlitVng8HmRkZMBsNkOtVis9FhHJ5IKnqU/b+2rcuHH43//9X+zduxcff/wxfvvb3wJA6DL7rj9CJUmCWq3Gjh07Qt9rt9uh0WggSRI2bdoEjUYDAHA4HNBqtdi9e3ePy/UlSYrK5fuD+iC8EAJutxu1tbXwer0wmUwoLS1lUBDFuaIMTZ+299Vrr72GHTt24IEHHsDLL7+MY8eOXfHrsrKyUFpaGgqLffv2Yd68eQCACRMmYMuWLQCA2tpazJw5E16vFwCwe/fu0Gdl/OEPf8Add9wRkbmvZdCuLFj8RzR4rRk3vcc5CwBIT0zGmnHTI/L48+fPx3PPPYff/va3SExMxKpVq/D6669f8Wt/8pOfYPXq1fjFL36B5ORkrF+/PlQu+PLLL6OyshLAxQDqumQ/JSUFVVVVaGtrw6JFi1BeXh6Rua9l0IWFJElwuVxwuVxITExEUVERsrOzeW6CaBDpOi8h19VQBQUFoVVBlxkzZoT+PX78eIwfPx4AMHToULz33nuXPYbBYMBbb711xce/9dZb8eyzz0Zk1t4aVGHB4j8i6lI1dFzEwmEwGBSvlJcW/5WUlCArK0vpsYiI+izaK4oucR8WbW1tsFgs8Pv9yMvLg8FgYPEfEVEfxW1YXFr8V1ZWFroumYjijxCC5x67kSQpoo8Xl2HR0tICq9WKQCAArVYLvV7Pqg6iOJaamor6+nq+kRYXQ9Pv98PhcET0D+S4CotAIACr1YqWlhakpqaipKQEaWlpSo9FRDIrLCxEXV0dXC6X0qPEhKSkJOTk5ECr1UbuMSP2SAq6UvGfTqcb9H9hEA0WycnJKCsrU3qMuCbrsZnq6mrMmDEDU6ZMwebNmy+7//jx45g9ezamTZuG5cuXIxAI9PlndHZ24ty5c6irq4NarcbQoUOh1+sZFEREESRbWDgcDqxfvx5btmzBjh078P7776O2trbH1zz//PNYuXIldu7cCSEEtm7d2qef0dTUhNraWng8HhQUFGDIkCFsiCUikoFsh6H279+PCRMmhEqwpk2bhpqaGjzzzDMAAIvFAp/Ph5tvvhkAMGvWLLzxxhuoqqoK+9jBYBDAxZVJYWEhjEYjvF4vLBaLLP9fiIjigd1uB/DVa2hfyBYWTqcTOp0udFuv1+PIkSNXvV+n08HhcPTqsbtOYnV9oAgREfWey+VCSUlJn75HtrAQQly2rft5hHD3X8vo0aOxefNm6HQ6vsGOiKiXgsEgXC4XRo8e3efvlS0sDAYDDh06FLrtdDqh1+t73O92u0O3XS5Xj/uvJTU1FbfeemvkhiUiGiT6uqLoItsJ7okTJ+LAgQNoaGiA1+vFrl27MGnSpND9ZrMZKSkpoY8b3L59e4/7iYgodqjElY4HRUh1dTXeeust+P1+zJkzBwsXLsTChQuxZMkSjBkzBidOnMCKFSvg8XgwcuRIrFu3jh88REQUg2QNCyIiig8sTCIiorAYFkREFBbDgoiIwmJYEBFRWDEfFtEoIxwowu2L3bt341vf+hZmzpyJp59+Gs3NzQpMGR3h9kWXvXv3YvLkyVGcLPrC7YszZ85g/vz5mDlzJh5//PFB/bw4evQoZs+ejZkzZ2LRokVoaWlRYMroaWtrQ0VFBerq6i67r8+vnSKG2e12cdddd4nGxkbh8XhEZWWlOHXqVI+vue+++8Tf//53IYQQL774oti8ebMCk8ov3L5obW0VX//614XdbhdCCPGf//mf4pVXXlFqXFn15nkhhBAul0vce++94q677lJgyugIty8kSRJTp04Vf/zjH4UQQvzkJz8Rr732mlLjyqo3z4uHH35Y7N27VwghxLp168R//Md/KDFqVHz22WeioqJCjBo1Sly4cOGy+/v62hnTK4vuZYTp6emhMsIuVyoj7H5/PAm3L/x+P1avXg2DwQAAGD58OGw2m1LjyircvuiyYsWKUHFlvAq3L44ePYr09PTQG14XL16MefPmKTWurHrzvJAkCR6PBwDg9XrjuqV669atWLVq1RWbMfrz2hnTYXGlMsLuZYPXU0Y40ITbF7m5ubjnnnsAAD6fDxs3bgzdjjfh9gUAvPvuuxg5ciRuuummaI8XVeH2xfnz56HVarFs2TJUVlZi1apVSE9PV2JU2fXmefHCCy9g+fLluOOOO7B//3489NBD0R4zatasWXPVWqT+vHbGdFgIGcsIB5re/n9tbW3FwoULMWLECDzwwAPRGC3qwu2LkydPYteuXXj66aejOZYiwu2LQCCAgwcP4pFHHkF1dTWKiorw6quvRnPEqAm3L3w+H5YvX45Nmzbhz3/+M6qqqrBs2bJojhgz+vPaGdNhcWnZYCTLCAeacPuia1tVVRVGjBiBNWvWRHvEqAm3L2pqauByuTB79mw8+eSTof0Sj8LtC51Oh5KSEowZMwYAUFFR0eOjAuJJuH1x8uRJpKSkYOzYsQCAuXPn4uDBg1GfMxb057UzpsOCZYRfCbcvgsEgFi9ejOnTp2P58uVxu8ICwu+LJUuWYOfOndixYwc2btwIvV6PLVu2KDixfMLti1tuuQUNDQ04ceIEAGDPnj0YNWqUUuPKKty+KCkpgd1ux5kzZwAAn3zySShEB5t+vXZG9PS7DD766CNx3333ialTp4qNGzcKIYR44oknxJEjR4QQQhw/flzMnj1b3HvvveIHP/iB6OjoUHJcWV1rX+zatUsMHz5czJw5M/S/l156SeGJ5RPuedHlwoULcX01lBDh98Vnn30mZs+eLWbMmCG++93vCrfbreS4sgq3L/bu3SsqKytFRUWFePTRR8X58+eVHDcq7rrrrtDVUNfz2skiQSIiCiumD0MREVFsYFgQEVFYDAsiIgqLYUFERGExLIiIKCyGBdF1WLVqFSZPnoz169crPQqRrHjpLNF1GDFiBPbu3Quj0aj0KESyYlgQ9VNVVRUOHz6MG264AbW1tZg/fz4OHTqE1tZWPPbYY6iqqsKnn36KNWvWID09He3t7fjggw+gVquVHp2oz5KUHoBooNqyZQuGDx+OTZs2Yc6cOfD5fNi2bRscDgfuv/9+jBs3DgBw6tQp7N69G2azWeGJifqP5yyIIqSqqgoqlQpGoxHf+MY3sG/fPgBAQUEBg4IGPIYFUYQkJX21UJckCQkJF3+94vXzI2hwYVgQRcj27dsBAFarFfv27YvbBmQanHjOgihC6urqMGvWLPh8PqxYsQJDhgyBy+VSeiyiiODVUEQRMHnyZGzYsGHQfj4CxT8ehiIiorC4siAiorC4siAiorAYFkREFBbDgoiIwmJYEBFRWAwLIiIKi2FBRERh/f8+aUZP+Ue+BgAAAABJRU5ErkJggg==\n", "text/plain": [ "
" ] @@ -634,11 +625,11 @@ "Post-adjustment group rates are \n", "\n", "group fpr tpr\n", - " cat 0.3894 0.814\n", - " dog 0.3936 0.7692\n", - "sheep 0.3629 0.8481\n", + " cat 0.3835 0.8605\n", + " dog 0.3883 0.8205\n", + "sheep 0.4032 0.7848\n", "\n", - "And loss is 0.3540\n", + "And loss is 0.3560\n", "\n" ] } @@ -940,7 +931,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 25, "id": "93f0abdd", "metadata": {}, "outputs": [ @@ -960,7 +951,7 @@ " [0.0493, 0.463 , 0.4877]]])" ] }, - "execution_count": 26, + "execution_count": 25, "metadata": {}, "output_type": "execute_result" } @@ -979,7 +970,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 26, "id": "b122dd89", "metadata": {}, "outputs": [ @@ -999,7 +990,7 @@ " [0.2045, 0.5864, 0.209 ]]])" ] }, - "execution_count": 27, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -1023,7 +1014,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 27, "id": "4967951b", "metadata": {}, "outputs": [ @@ -1052,7 +1043,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 28, "id": "1fe3e133", "metadata": {}, "outputs": [ @@ -1082,7 +1073,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 29, "id": "561efb4c", "metadata": {}, "outputs": [ @@ -1134,7 +1125,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 30, "id": "2b690e1e", "metadata": {}, "outputs": [ @@ -1170,7 +1161,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 31, "id": "564a1001", "metadata": {}, "outputs": [ @@ -1227,39 +1218,6 @@ "source": [ "Note that the the adjusted FPRs and TPRs are different from our prior attempt at equalized oddds, and the loss is much higher (64% instead of 50%). These changes come from the variance in the LP parameters induced by sampling and show that the performance of adjusted predictor may be much worse on average than its theoretical optimum." ] - }, - { - "cell_type": "code", - "execution_count": 58, - "id": "51842eca", - "metadata": {}, - "outputs": [ - { - "ename": "IndexError", - "evalue": "index 2 is out of bounds for axis 0 with size 2", - "output_type": "error", - "traceback": [ - "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mIndexError\u001b[0m Traceback (most recent call last)", - "\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[1;32m 1\u001b[0m \u001b[0mmulti_b\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madjust\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgoal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'odds'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0mgrid\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtools\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfd_point\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmulti_b\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m", - "\u001b[0;32m~/code/fairness/tools.py\u001b[0m in \u001b[0;36mfd_point\u001b[0;34m(b, new, cols)\u001b[0m\n", - "\u001b[0;32m~/code/fairness/tools.py\u001b[0m in \u001b[0;36m\u001b[0;34m(.0)\u001b[0m\n", - "\u001b[0;31mIndexError\u001b[0m: index 2 is out of bounds for axis 0 with size 2" - ] - } - ], - "source": [ - "multi_b.adjust(goal='odds')\n", - "grid = tools.fd_point(multi_b)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "e3cf1287", - "metadata": {}, - "outputs": [], - "source": [] } ], "metadata": { diff --git a/tools.py b/tools.py index 4552b70..4f11f4b 100644 --- a/tools.py +++ b/tools.py @@ -1109,173 +1109,6 @@ def balancing_stats(b, cv=False, cols=None): return out_df -def fd_grid(b, - loss='micro', - goal='odds', - step=0.01, - max=1.0, - round=2, - absval=True, - cv=False, - shuffle=False, - seed=None): - '''Returns a grid of slack-vs.-loss metrics for making F-D plots. - ''' - out = [] - max_ineqs = np.arange(0, max + step, step) - - for m in max_ineqs: - b.adjust_new(goal=goal, - loss=loss, - slack=m, - cv=cv, - shuffle=shuffle, - seed=seed) - out.append(fd_point(b)) - - out = pd.concat(out, axis=0) - - out['max_ineq'] = max_ineqs - out['adj'] = 1 - - # Getting the pre-adjustment values - point = fd_point(b, new=False) - point['adj'] = 0 - out = pd.concat([out, point], axis=0) - - return out - - -def fd_point(b, new=True, cols=None): - '''Returns a single point for the unadjusted fairness vs. - discrimination for a predictor. - ''' - # Setting things up - p_y_a = b.p_y_a - combos = list(combinations(range(b.n_groups), 2)) - tpr_diffs = [] - fpr_diffs = [] - j_diffs = [] - acc_diffs = [] - parity_diffs = [] - cp_diffs = [] - - # Deciding whether to use pre- or post-adjustment values - if new: - rocs = b.rocs - cp_mats = b.new_cp_mats - brier_score = b.brier_score - else: - rocs = b.old_rocs - cp_mats = b.cp_mats - brier_score = b.old_brier_score - - for c in combos: - tprs = rocs[c, :, 1] - fprs = rocs[c, :, 0] - js = tprs + (1 - fprs) - 1 - accs = np.array([np.dot(p_y_a[i], - tprs[i]) - for i in c]) - counts = np.array([np.dot(p_y_a[i], - cp_mats[i]) - for i in c]) - cp_diffs.append(np.abs(np.diff(cp_mats, - axis=0))) - tpr_diffs.append(np.abs(np.diff(tprs, axis=0))) - j_diffs.append(np.abs(np.diff(js, axis=0))) - acc_diffs.append(np.abs(np.diff(accs)[0])) - parity_diffs.append(np.abs(np.diff(counts, axis=0))) - - tpr = np.max([a.max() for a in tpr_diffs]) - mean_tpr = np.mean([a.mean() for a in tpr_diffs]) - j = np.max([a.max() for a in j_diffs]) - mean_j = np.max([a.mean() for a in j_diffs]) - acc = np.max([a.max() for a in acc_diffs]) - mean_acc = np.max([a.mean() for a in acc_diffs]) - parity = np.max([a.max() for a in parity_diffs]) - mean_parity = np.max([a.mean() for a in parity_diffs]) - cp = np.max([a.max() for a in cp_diffs]) - mean_cp = np.mean([a.mean() for a in cp_diffs]) - - macro = b.macro_loss - micro = b.loss - - out = pd.DataFrame([micro, macro, brier_score, - acc, mean_acc, tpr, - mean_tpr, j, mean_j, - parity, mean_parity, cp, - mean_cp]).transpose() - out.columns = [ - 'micro_loss', 'macro_loss', 'brier_score', - 'acc', 'mean_acc', 'tpr', 'mean_tpr', - 'j', 'mean_j', 'parity', 'mean_parity', - 'cp', 'mean_cp' - ] - - if cols: - out = out[cols] - - return out - - -def fd_plot(grid, - goal='odds', - disc='macro', - ax=None, - plot_original=True, - label_axes=False, - show=False): - # Separating original and adjusted values - pre = np.where(grid.adj == 0)[0] - post = np.where(grid.adj == 1)[0] - - # Setting the measure of fairness for the x-axis - if 'opportunity' in goal: - x = grid.mean_tpr.values - x_name = 'max mean TPR diff' - elif 'odds' in goal: - x = grid.mean_j.values - x_name = 'max mean J diff' - elif 'acc' in goal: - x = grid.acc.values - x_name = 'max accuracy diff' - elif 'parity' in goal: - x = grid.mean_parity.values - elif 'strict' in goal: - x = grid.mean_cp.values - - # Setting the measure of discrimination for the y-axis - if disc == 'micro': - y = grid.micro_loss.values - y_name = 'micro loss' - elif disc == 'macro': - y = grid.macro_loss.values - y_name = 'macro loss' - elif disc == 'brier': - y = grid.brier_score.values - y_name = 'Brier score' - - # Making the plot - lp = sns.lineplot(x=x[post], y=y[post], ax=ax) - - # Optionally adding the original point - if plot_original: - sns.scatterplot(x=x[pre], - y=y[pre], - ax=lp, - marker='x', - color='black') - - if label_axes: - lp.set(xlabel=x_name, ylabel=y_name) - - if show: - plt.show() - - return - - def cv_predict(y, y_, a, goal='strict', loss='macro',