ratings-hockey.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd  \n",
    "import numpy as np  \n",
    "import matplotlib.pyplot as plt  \n",
    "from sklearn.linear_model import LinearRegression,Ridge,ElasticNet,ElasticNetCV,LassoCV,SGDRegressor,RidgeCV\n",
    "from collections import defaultdict\n",
    "import json\n",
    "import statsmodels.api as sm\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, PolynomialFeatures\n",
    "import os, sys\n",
    "\n",
    "import fnmatch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xso = []\n",
    "y = []\n",
    "\n",
    "pos_Xs = defaultdict(list)\n",
    "pos_y = defaultdict(list) \n",
    "pos_min = defaultdict(list) \n",
    "\n",
    "for file in sorted(os.listdir('.')):\n",
    "    if fnmatch.fnmatch(file, 'ZGMH_League*.json'):\n",
    "        print(file)\n",
    "        data = json.load(open(file,'rt',encoding='utf-8-sig'))\n",
    "        player_ratings = {}\n",
    "        player_pos = {}\n",
    "\n",
    "        valid_pos = set()\n",
    "        for p in data['players']:\n",
    "            for r in p['ratings']:\n",
    "                player_ratings[(p['pid'],r['season'])] ={k:v for k,v in r.items() if type(v) == int and k != 'season'}\n",
    "                player_pos[(p['pid'],r['season'])] =r['pos']\n",
    "\n",
    "                valid_pos.add(r['pos'])\n",
    "        for p in data['players']:\n",
    "            for r in p['stats']:\n",
    "                if r['min'] > 0:\n",
    "                    pos = player_pos[(p['pid'],r['season'])]\n",
    "                    pAV = r['ops'] + r['dps'] + r['gps']\n",
    "                    if pAV:\n",
    "                        pos_Xs[pos].append(player_ratings[(p['pid'],r['season'])])\n",
    "                        pos_y[pos].append(float(pAV)/r['min'])\n",
    "                        pos_min[pos].append(r['min'])\n",
    "\n",
    "        valid_pos = sorted(list(valid_pos))\n",
    "        valid_col = [k for k in list(player_ratings.values())[0] if k not in ['min','pos','ovr','pot','injuryIndex']]\n",
    "        print(valid_pos)\n",
    "\n",
    "        for g in data['games']:\n",
    "            season = g['season']\n",
    "            if g['won']['tid'] == g['teams'][0]['tid']: #home team won\n",
    "                s = g['won']['pts'] - g['lost']['pts'] \n",
    "            else:\n",
    "                s =  g['lost']['pts'] - g['won']['pts']\n",
    "            y.append(s)\n",
    "            r2 = []\n",
    "            for i in range(2):\n",
    "                team_r = []\n",
    "                t = g['teams'][i]['players']\n",
    "                for p in t:\n",
    "                    pos = p['pos']\n",
    "                    mp = p['min']\n",
    "                    r = {k:mp*v for k,v in player_ratings[(p['pid'],season)].items()}\n",
    "                    r['pos'] = pos\n",
    "                    r['min'] = mp\n",
    "                    if mp > 0:\n",
    "                        team_r.append(r)\n",
    "                # dummies to make sure we have players at every pos\n",
    "                for p in valid_pos:\n",
    "                    r = {k:0 for k in r}\n",
    "                    r['pos'] = p\n",
    "                    r['min'] = 1e-9\n",
    "                    team_r.append(r)\n",
    "\n",
    "                # should be alphabetical\n",
    "                res = pd.DataFrame(team_r).groupby('pos').sum()\n",
    "                res2 = res.divide(res['min'],'rows')\n",
    "                ra = np.array(res2[valid_col])\n",
    "                r2.append(ra)\n",
    "            Xso.append(np.array(r2[0] - r2[1]).flatten())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# predict Point Shares per position\n",
    "for pos in valid_pos:\n",
    "    print(pos)\n",
    "    #Xs = np.nan_to_num(pos_Xs[pos],0)\n",
    "    fx = StandardScaler()\n",
    "    y2 = np.array(pos_y[pos]).astype(float)\n",
    "    X3 = sm.add_constant(pd.DataFrame(fx.fit_transform(pd.DataFrame(pos_Xs[pos])[valid_col]),columns=valid_col))\n",
    "    filt = np.array(pos_min[pos]) > sorted(pos_min[pos])[len(pos_min[pos])//2]\n",
    "    est = sm.OLS(y2[filt], X3[filt])\n",
    "    est2 = est.fit()\n",
    "    print(est2.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# predict Point Shares per position\n",
    "\n",
    "for pos in valid_pos:\n",
    "    #Xs = np.nan_to_num(pos_Xs[pos],0)\n",
    "    fx = StandardScaler()\n",
    "    y2 = np.array(pos_y[pos]).astype(float)\n",
    "    X3 = pd.DataFrame(fx.fit_transform(pd.DataFrame(pos_Xs[pos])[valid_col]),columns=valid_col)\n",
    "    filt = np.array(pos_min[pos]) > sorted(pos_min[pos])[len(pos_min[pos])//2]\n",
    "    est = ElasticNetCV([0.001,.1,.4,.5,.7,.725,.75,.775,.8,.9,.95,.99,1],cv=3,positive=True,max_iter=3e4)\n",
    "    est.fit(X3[filt],y2[filt],)\n",
    "    print(pos,est.score(X3[filt],y2[filt]))\n",
    "    print(pos+' : { ')\n",
    "    res = sorted([(i,r) for r,i in zip(valid_col,est.coef_) if i > 0],reverse=True)\n",
    "    for i,r in res:\n",
    "        if i > 0:\n",
    "            print('\\t{}: [{:.1f}, 1],'.format(r,i/res[-1][0]))\n",
    "    print('},')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y.mean(),y.std()\n",
    "#X3.shape,y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(Xso),len(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.hist(y,20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "exclude = {\n",
    "    ('glk'):['G'],\n",
    "}\n",
    "#exclude = {}\n",
    "exp_lbl = sum([[str(p) + '_' + str(s) for s in valid_col] for p in valid_pos],[])\n",
    "filt_lbl = []\n",
    "filt_vec = []\n",
    "for n in exp_lbl:\n",
    "    inc = True\n",
    "    pos,rate = n.split('_')\n",
    "    for k,v in exclude.items():\n",
    "        if pos not in v and rate in k:\n",
    "            inc=False\n",
    "    filt_vec.append(inc)       \n",
    "    if inc:\n",
    "        print(n)\n",
    "        filt_lbl.append(n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "\n",
    "Xs = np.nan_to_num(Xso,0)\n",
    "fx = StandardScaler()\n",
    "X2 = fx.fit_transform(Xs[:,filt_vec] )\n",
    "y = np.array(y).astype(float)\n",
    "\n",
    "reg = ElasticNetCV([.1,.4,.5,.7,.725,.75,.775,.8,.9],cv=3,n_alphas=250,positive=True,max_iter=2e4)#(alpha=0.1,l1_ratio=0.7)#CV(cv=10)#ElasticNetCV(.7,cv=10,)\n",
    "#reg = ElasticNet(1e-3,0.01,positive=True,max_iter=2e4)#(alpha=0.1,l1_ratio=0.7)#CV(cv=10)#ElasticNetCV(.7,cv=10,)\n",
    "\n",
    "#reg = lgb.LGBMRegressor()\n",
    "reg.fit(X2,y)\n",
    "print(X2.shape,reg.score(X2,y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "plt.style.use('seaborn-white')\n",
    "plt.scatter(reg.predict(X2),y,s=5,alpha=0.1)\n",
    "#plt.ylim(-60,60)\n",
    "#plt.xlim(-60,60)\n",
    "plt.xlabel('predicted margin')\n",
    "plt.ylabel('actual margin')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#sorted([(i,n) for i,n in zip(reg.feature_importances_,exp_lbl) if 'LB' in n],reverse=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "X3f = pd.DataFrame(X2,columns=filt_lbl)\n",
    "X3 = sm.add_constant(X3f)\n",
    "est = sm.RLM(y, X3)\n",
    "est2 = est.fit()\n",
    "print(est2.summary())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print('home field adv is {:.1f} points'.format(reg.intercept_))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "future_use = defaultdict(dict)\n",
    "for pos in sorted(list(valid_pos)):\n",
    "    res = sorted([(r,n,p) for n,r,p in zip(filt_lbl,est2.params[1:],est2.pvalues[1:]) if pos+ '_' in n],reverse=True)\n",
    "    print(pos+' : { ')\n",
    "    for p in res:\n",
    "        if np.linalg.norm(p[0]) > 1e-3 and p[2] < 0.05:\n",
    "            key = p[1].split('_')[1]\n",
    "            future_use[pos][key]  = p[0]\n",
    "            print('\\t{}: [{:.3f}, 1],'.format(key,p[0]))\n",
    "    print('},')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_to_use = [k for k,v in (est2.pvalues < 0.1).items() if v if k != 'const']\n",
    "reg_small = ElasticNetCV(positive=True,cv=3)\n",
    "reg_small.fit(Xs[:,[list(filt_lbl).index(r) for r in ratings_to_use]],y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = sorted([(abs(v),k) for k,v in zip(ratings_to_use,reg_small.coef_)],reverse=True)\n",
    "for v,k in res:\n",
    "    print('{}\\t{}\\t{:.3f}'.format('',k,v))\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid_pos = set([k.split('_')[0] for k in ratings_to_use])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "future_use = defaultdict(dict)\n",
    "for pos in sorted(list(valid_pos)):\n",
    "    res = sorted([(r,n) for n,r in zip(ratings_to_use,reg_small.coef_) if pos+ '_' in n and r > 0],reverse=True)\n",
    "    print(pos+' : { ')\n",
    "    for p in res:\n",
    "        if np.linalg.norm(p[0]) > 1e-3:\n",
    "            #future_use[row[0]][res[i][1]] = row[1][res[i][1]]\n",
    "            print('\\t{}: [{:.3f}, 1],'.format(p[1].split('_')[1],p[0]))\n",
    "    print('},')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rts = defaultdict(list)\n",
    "for file in sorted(os.listdir('.')):\n",
    "    if fnmatch.fnmatch(file, 'ZGMH_League*.json'):\n",
    "        print(file)\n",
    "        data = json.load(open(file,'rt',encoding='utf-8-sig'))\n",
    "\n",
    "\n",
    "        valid_pos = set()\n",
    "        for p in data['players']:\n",
    "            for r in p['ratings']:\n",
    "                r2 = sum([future_use[r['pos']][k]*v for k,v in r.items() if type(v) == int and k != 'season' and k in future_use[r['pos']]])\n",
    "                rts[r['pos']].append((r['ovr'],r2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "i=1\n",
    "for pos, results in rts.items():\n",
    "    print(pos)\n",
    "    plt.subplot(3,4,i)\n",
    "    plt.title(pos)\n",
    "    A = np.array(results)\n",
    "    plt.scatter(A[:,0],A[:,1],s=5,alpha=0.5)\n",
    "    plt.xlabel('current OVR')\n",
    "    plt.yticks([],[])\n",
    "    i+=1\n",
    "plt.suptitle('Alt OVR comp',size=18,weight='bold',y=1.02)\n",
    "plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rts = defaultdict(list)\n",
    "\n",
    "player_ratings = {}\n",
    "player_pos = {}\n",
    "player_r1 = {}\n",
    "player_r2 = {}\n",
    "\n",
    "\n",
    "for file in sorted(os.listdir('.')):\n",
    "    if fnmatch.fnmatch(file, 'ZGMH_League*.json'):\n",
    "        print(file)\n",
    "        data = json.load(open(file,'rt',encoding='utf-8-sig'))\n",
    "\n",
    "\n",
    "        valid_pos = set()\n",
    "        for p in data['players']:\n",
    "            for r in p['ratings']:\n",
    "                r2 = sum([future_use[r['pos']][k]*v for k,v in r.items() if type(v) == int and k != 'season' and k in future_use[r['pos']]])\n",
    "                \n",
    "                key = (file,p['pid'],r['season'])\n",
    "                player_ratings[key] ={k:v for k,v in r.items() if type(v) in [int,dict,str] and k != 'season'}\n",
    "                player_pos[key] =r['pos']\n",
    "                player_r1[key] = r['ovr']\n",
    "                player_r2[key] = r2\n",
    "\n",
    "\n",
    "                valid_pos.add(r['pos'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "future_use"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "base = json.load(open('base_hk.json','rt',encoding='utf-8-sig'))\n",
    "print(len(base['players']))\n",
    "for p in base['players']:\n",
    "    if p['tid'] in [31,30]:\n",
    "        p['tid'] = -3\n",
    "\n",
    "print(len(base['players']))\n",
    "del base['teams'][-1]['depth']\n",
    "del base['teams'][-2]['depth']\n",
    "\n",
    "max_pid = max([_['pid'] for _ in base['players']])\n",
    "base['gameAttributes']['userTid'] = [{'start': None, 'value': 31}]\n",
    "base['gameAttributes']['userTids'] = [31]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "count = {'C':5, 'D':7, 'W':10,'G':3}\n",
    "\n",
    "min_ovr = {'C':55, 'D':55, 'W':55,'G':63}\n",
    "alt_t = []\n",
    "\n",
    "for p,N in count.items():\n",
    "    mo = min_ovr[p]\n",
    "    pc = {k:v for k,v in player_r1.items() if v < mo and player_pos[k] == p}\n",
    "    alts = sorted([(player_r2[k],k) for k in pc],reverse=True)\n",
    "    done_set = set()\n",
    "    i = 0\n",
    "    icnt = len(alt_t)\n",
    "    while len(alt_t) < icnt + N:\n",
    "        _, k = alts[i]\n",
    "        if k not in done_set:\n",
    "            player = {'ratings':[player_ratings[k]],'pos':p}\n",
    "            alt_t.append(player)\n",
    "            done_set.add(k)\n",
    "        i+=1\n",
    "for p in alt_t:\n",
    "    p['born'] =  {'year': 1994, 'loc': 'USA'}\n",
    "    p['pid'] = max_pid\n",
    "    p['tid'] = 31\n",
    "    p['ovrs'] = {}\n",
    "    p['pots'] = {}\n",
    "    #p['ratings'][0]['pos'] = p['pos']\n",
    "    #del p['ratings'][0]['ovr']\n",
    "    #del p['ratings'][0]['pot']\n",
    "    #del p['ratings'][0]['pos']\n",
    "    #del p['pos']\n",
    "\n",
    "    max_pid += 1\n",
    "    base['players'].append(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "min_ovr = {'C':63, 'D':63, 'W':63,'G':63}\n",
    "\n",
    "\n",
    "alt_t = []\n",
    "\n",
    "for p,N in count.items():\n",
    "    mo = min_ovr[p]\n",
    "    pc = {k:v for k,v in player_r1.items() if v > mo and player_pos[k] == p}\n",
    "    alts = sorted([(player_r2[k],k) for k in pc])#,reverse=True)\n",
    "    done_set = set()\n",
    "    i = 0\n",
    "    icnt = len(alt_t)\n",
    "    while len(alt_t) < icnt + N:\n",
    "        _, k = alts[i]\n",
    "        if k not in done_set:\n",
    "            player = {'ratings':[player_ratings[k]],'pos':p}\n",
    "            alt_t.append(player)\n",
    "            done_set.add(k)\n",
    "        i+=1\n",
    "for p in alt_t:\n",
    "    p['born'] =  {'year': 1994, 'loc': 'USA'}\n",
    "    p['pid'] = max_pid\n",
    "    p['tid'] = 30\n",
    "    p['ovrs'] = {}\n",
    "    p['pots'] = {}\n",
    "    #p['ratings'][0]['pos'] = p['pos']\n",
    "    #del p['ratings'][0]['ovr']\n",
    "    #del p['ratings'][0]['pot']\n",
    "    #del p['pos']\n",
    "\n",
    "    max_pid += 1\n",
    "    base['players'].append(p)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('overunder_h.json','wt') as fp:\n",
    "    json.dump(base,fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "p"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}