Merge pull request #337 from alan-turing-institute/develop

AIrsenal 0.4.0

.gitignore

@@ -4,6 +4,10 @@
 *FD_API_KEY
 *FPL_LOGIN
 *FPL_PASSWORD
+*AIrsenalDBFile
+*AIrsenalDBUri
+*AIrsenalDBUser
+*AIrsenalDBPassword
 
 # compiled stan models
 stan_model/

.travis.yml

@@ -6,18 +6,25 @@ env:
 
 jobs:
   include:
-    - name: "Python 3.7.4 on Xenial Linux"
-      python: 3.7           # this works for Linux but is ignored on macOS or Windows
+    - name: "Python 3.7"
+      python: 3.7
       install:
         - pip3 install --upgrade pip
         - pip3 install .
 
-    - name: "Python 3.6 on Xenial Linux"
-      python: 3.6           # this works for Linux but is ignored on macOS or Windows
+    - name: "Python 3.8"
+      python: 3.8
       install:
         - pip3 install --upgrade pip
         - pip3 install .
-
+
+    - name: "Python 3.9"
+      python: 3.9
+      install:
+        - pip3 install --upgrade pip
+        - pip3 install numpy Cython
+        - pip3 install .
+
     - name: "Python 3.7.4 on macOS"
       os: osx
       osx_image: xcode11.2  # Python 3.7.4 running on macOS 10.14.4
@@ -44,7 +51,7 @@ jobs:
         - conda activate test_env
         - conda install libpython m2w64-toolchain -c msys2
         - conda install numpy cython -c conda-forge
-        - conda install pystan -c conda-forge
+        - conda install pystan=2.19.1.1 -c conda-forge
       install:
         - python -m pip install --upgrade pip
         - python -m pip install .

README.md

@@ -76,6 +76,8 @@ Once you've installed the module, you will need to set the following parameters:
 
 4. `FPL_PASSWORD`: your FPL password (this is only required to get FPL league standings).
 
+5. `AIrsenalDBFile`: Local path to where you would like to store the AIrsenal sqlite3 database. If not set a temporary directory will be used by default (`/tmp/data.db` on Unix systems).
+
 The values for these should be defined either in environment variables with the names given above, or as files in the `airsenal/data` directory with the names given above. For example, to set your team ID you can create the file `airsenal/data/FPL_TEAM_ID` (with no file extension) and its contents should be your team ID and nothing else. So the contents of the file would just be something like:
 ```
 1234567
@@ -89,7 +91,6 @@ If you installed AIrsenal with conda, you should always make sure the `airsenale
 conda activate airsenalenv
 ```
 
-
 Note: Most the commands below can be run with the `--help` flag to see additional options and information.
 
 ### 1. Creating the database
@@ -144,7 +145,7 @@ Instead of running the commands above individually you can use:
 ```shell
 airsenal_run_pipeline
 ```
-This will delete and recreate the database and then run the points predictions and transfer optimization.
+This will update the database and then run the points predictions and transfer optimization.
 
 ## Docker
 

airsenal/__init__.py

@@ -7,7 +7,7 @@
 # AIrsenal package version. When merging changes to master:
 # - increment 2nd digit for new features
 # - increment 3rd digit for bug fixes
-__version__ = "0.3.0"
+__version__ = "0.4.0"
 
 # Cross-platform temporary directory
 if os.name == "posix":

airsenal/data/fifa_team_ratings_1718.csv

@@ -0,0 +1,21 @@
+team_name,att,mid,defn,ovr
+Chelsea,85,85,82,84
+Manchester United,86,83,81,83
+Arsenal,86,82,80,83
+Manchester City,84,85,81,83
+Tottenham Hotspur,86,82,81,82
+Liverpool,83,82,80,82
+Everton,79,82,79,80
+West Ham United,80,79,78,79
+Leicester City,79,79,76,78
+Southampton,78,78,77,78
+Stoke City,78,77,77,77
+Watford,77,76,76,77
+Crystal Palace,81,76,75,76
+West Bromwich Albion,80,75,76,76
+Swansea City,75,77,75,76
+Bournemouth,76,74,75,75
+Burnley,74,75,75,75
+Newcastle United,74,75,75,75
+Brighton & Hove Albion,74,74,73,74
+Huddersfield Town,73,73,72,73

airsenal/framework/bpl_interface.py

@@ -70,14 +70,26 @@ def get_ratings_df(season, dbsession):
     return df
 
 
-def create_and_fit_team_model(df, df_X, teams=CURRENT_TEAMS):
+def create_and_fit_team_model(df, df_X, teams=CURRENT_TEAMS, n_attempts=3):
     """
     Get the team-level stan model, which can give probabilities of
     each potential scoreline in a given fixture.
     """
     model_team = bpl.BPLModel(df, X=df_X)
 
-    model_team.fit()
+    for i in range(n_attempts):
+        print(f"attempt {i + 1} of {n_attempts}...", end=" ", flush=True)
+        try:
+            model_team.fit()
+            print("SUCCESS!")
+            break
+        except RuntimeError:
+            print("FAILED.")
+            if i + 1 == n_attempts:
+                raise
+            else:
+                continue
+
     # check if each team is known to the model, and if not, add it using FIFA rankings
     for team in teams:
         if team not in model_team.team_indices.keys():

airsenal/framework/db_config.py

@@ -1,27 +1,68 @@
 """
 Database can be either an sqlite file or a postgress server
 """
-
 import os
 from airsenal import TMPDIR
 
-# Default connection string points to a local sqlite file in
-# airsenal/data/data.db
-
-DB_CONNECTION_STRING = "sqlite:///{}/data.db".format(TMPDIR)
+config_path = os.path.join(os.path.dirname(__file__), "..", "data")
+AIrsenalDBFile_path = os.path.join(config_path, "AIrsenalDBFile")
+AIrsenalDBUri_path = os.path.join(config_path, "AIrsenalDBUri")
+AIrsenalDBUser_path = os.path.join(config_path, "AIrsenalDBUser")
+AIrsenalDBPassword_path = os.path.join(config_path, "AIrsenalDBPassword")
 
 # Check that we're not trying to set location for both sqlite and postgres
-if "AIrsenalDBFile" in os.environ.keys() and "AIrsenalDBUri" in os.environ.keys():
+if ("AIrsenalDBFile" in os.environ.keys() or os.path.exists(AIrsenalDBFile_path)) and (
+    "AIrsenalDBUri" in os.environ.keys() or os.path.exists(AIrsenalDBUri_path)
+):
     raise RuntimeError("Please choose only ONE of AIrsenalDBFile and AIrsenalDBUri")
 
-# location of sqlite file overridden by an env var
+# sqlite database in a local file with path specified by:
+# - AIrsenalDBFile environment variable
+# - airsenal/data/AIrsenalDBFile file
+# - platform-dependent temporary directory (default)
 if "AIrsenalDBFile" in os.environ.keys():
-    DB_CONNECTION_STRING = "sqlite:///{}".format(os.environ["AIrsenalDBFile"])
+    AIrsenalDBFile = os.environ["AIrsenalDBFile"]
+elif os.path.exists(AIrsenalDBFile_path):
+    AIrsenalDBFile = open(AIrsenalDBFile_path).read().strip()
+else:
+    AIrsenalDBFile = os.path.join(TMPDIR, "data.db")
+
+DB_CONNECTION_STRING = "sqlite:///{}".format(AIrsenalDBFile)
+
+# postgres database specified by: AIrsenalDBUri, AIrsenalDBUser, AIrsenalDBPassword
+# defined either as:
+# - environment variables
+# - Files in airsenal/data/
+if "AIrsenalDBUri" in os.environ.keys() or os.path.exists(AIrsenalDBUri_path):
+    if "AIrsenalDBUser" in os.environ.keys():
+        AIrsenalDBUser = os.environ["AIrsenalDBUser"]
+    elif os.path.exists(AIrsenalDBUser_path):
+        AIrsenalDBUser = open(AIrsenalDBUser_path).read().strip()
+    else:
+        raise RuntimeError(
+            "AIrsenalDBUser must be defined when using a postgres database"
+        )
+
+    if "AIrsenalDBUser" in os.environ.keys():
+        AIrsenalDBPassword = os.environ["AIrsenalDBPassword"]
+    elif os.path.exists(AIrsenalDBPassword_path):
+        AIrsenalDBPassword = open(AIrsenalDBPassword_path).read().strip()
+    else:
+        raise RuntimeError(
+            "AIrsenalDBPassword must be defined when using a postgres database"
+        )
+
+    if "AIrsenalDBUri" in os.environ.keys():
+        AIrsenalDBUri = os.environ["AIrsenalDBUri"]
+    elif os.path.exists(AIrsenalDBUri_path):
+        AIrsenalDBUri = open(AIrsenalDBUri_path).read().strip()
+    else:
+        raise RuntimeError(
+            "AIrsenalDBUri must be defined when using a postgres database"
+        )
 
-# location of postgres server
-if "AIrsenalDBUri" in os.environ.keys():
     DB_CONNECTION_STRING = "postgres://{}:{}@{}/airsenal".format(
-        os.environ["AIrsenalDBUser"],
-        os.environ["AIrsenalDBPassword"],
-        os.environ["AIrsenalDBUri"],
+        AIrsenalDBUser,
+        AIrsenalDBPassword,
+        AIrsenalDBUri,
     )

airsenal/framework/multiprocessing_utils.py

@@ -34,13 +34,13 @@ def __init__(self, n=0):
         self.count = multiprocessing.Value("i", n)
 
     def increment(self, n=1):
-        """ Increment the counter by n (default = 1) """
+        """Increment the counter by n (default = 1)"""
         with self.count.get_lock():
             self.count.value += n
 
     @property
     def value(self):
-        """ Return the value of the counter """
+        """Return the value of the counter"""
         return self.count.value
 
 
@@ -70,9 +70,9 @@ def get(self, *args, **kwargs):
         return super().get(*args, **kwargs)
 
     def qsize(self):
-        """ Reliable implementation of multiprocessing.Queue.qsize() """
+        """Reliable implementation of multiprocessing.Queue.qsize()"""
         return self.size.value
 
     def empty(self):
-        """ Reliable implementation of multiprocessing.Queue.empty() """
+        """Reliable implementation of multiprocessing.Queue.empty()"""
         return not self.qsize()

airsenal/framework/optimization_utils.py

@@ -114,7 +114,7 @@ def get_starting_squad(fpl_team_id=None):
     # chip is activated
     transactions = (
         session.query(Transaction)
-        .order_by(Transaction.id)
+        .order_by(Transaction.gameweek, Transaction.id)
         .filter_by(fpl_team_id=fpl_team_id)
         .filter_by(free_hit=0)
         .all()
@@ -672,12 +672,13 @@ def make_new_squad(
     return best_squad
 
 
-def fill_suggestion_table(baseline_score, best_strat, season):
+def fill_suggestion_table(baseline_score, best_strat, season, fpl_team_id):
     """
     Fill the optimized strategy into the table
     """
     timestamp = str(datetime.now())
     best_score = best_strat["total_score"]
+
     points_gain = best_score - baseline_score
     for in_or_out in [("players_out", -1), ("players_in", 1)]:
         for gameweek, players in best_strat[in_or_out[0]].items():
@@ -689,6 +690,8 @@ def fill_suggestion_table(baseline_score, best_strat, season):
                 ts.points_gain = points_gain
                 ts.timestamp = timestamp
                 ts.season = season
+                ts.fpl_team_id = fpl_team_id
+                ts.chip_played = best_strat["chips_played"][gameweek]
                 session.add(ts)
     session.commit()
 

airsenal/framework/player.py

@@ -38,6 +38,9 @@ def __init__(
         self.predicted_points = {}
         self.sub_position = None
 
+    def __str__(self):
+        return self.name
+
     def calc_predicted_points(self, method):
         """
         get expected points from the db.
@@ -55,10 +58,6 @@ def get_predicted_points(self, gameweek, method):
         if method not in self.predicted_points.keys():
             self.calc_predicted_points(method)
         if gameweek not in self.predicted_points[method].keys():
-            print(
-                "No prediction available for {} week {}".format(
-                    self.data.name, gameweek
-                )
-            )
+            print("No prediction available for {} week {}".format(self.name, gameweek))
             return 0.0
         return self.predicted_points[method][gameweek]

airsenal/framework/prediction_utils.py

@@ -17,7 +17,6 @@
     NEXT_GAMEWEEK,
     get_fixtures_for_player,
     get_recent_minutes_for_player,
-    get_return_gameweek_for_player,
     get_max_matches_per_player,
     get_player,
     get_player_from_api_id,
@@ -71,7 +70,7 @@ def get_player_history_df(
     for counter, player in enumerate(players):
         print(
             "Filling history dataframe for {}: {}/{} done".format(
-                player.name, counter, len(players)
+                player, counter, len(players)
             )
         )
         results = player.scores
@@ -87,11 +86,7 @@ def get_player_history_df(
 
             match_id = row.result_id
             if not match_id:
-                print(
-                    " Couldn't find result for {} {} {}".format(
-                        row.fixture.home_team, row.fixture.away_team, row.fixture.date
-                    )
-                )
+                print(" Couldn't find result for {}".format(row.fixture))
                 continue
             minutes = row.minutes
             goals = row.goals
@@ -272,7 +267,7 @@ def calc_predicted_points_for_player(
     df_cards,
     season,
     gw_range=None,
-    fixtures_behind=3,
+    fixtures_behind=None,
     tag="",
     dbsession=session,
 ):
@@ -284,13 +279,19 @@ def calc_predicted_points_for_player(
     if isinstance(player, int):
         player = get_player(player, dbsession=dbsession)
 
-    message = "Points prediction for player {}".format(player.name)
+    message = "Points prediction for player {}".format(player)
 
     if not gw_range:
         # by default, go for next three matches
         gw_range = list(
             range(NEXT_GAMEWEEK, min(NEXT_GAMEWEEK + 3, 38))
         )  # don't go beyond gw 38!
+
+    if fixtures_behind is None:
+        # default to getting recent minutes from the same number of matches we're
+        # predicting for
+        fixtures_behind = len(gw_range)
+
     team = player.team(
         season, gw_range[0]
     )  # assume player stays with same team from first gameweek in range
@@ -335,7 +336,7 @@ def calc_predicted_points_for_player(
             # calculate appearance points, defending points, attacking points.
             points = 0.0
 
-        elif is_injured_or_suspended(player.fpl_api_id, gameweek, season, dbsession):
+        elif player.is_injured_or_suspended(season, gw_range[0], gameweek):
             # Points for fixture will be zero if suspended or injured
             points = 0.0
 
@@ -458,27 +459,6 @@ def get_all_fitted_player_models(player_model, season, gameweek, dbsession=sessi
     return df_positions
 
 
-def is_injured_or_suspended(player_api_id, gameweek, season, dbsession=session):
-    """
-    Query the API for 'chance of playing next round', and if this
-    is <=50%, see if we can find a return date.
-    """
-    if season != CURRENT_SEASON:  # no API info for past seasons
-        return False
-    # check if a player is injured or suspended
-    pdata = fetcher.get_player_summary_data()[player_api_id]
-    if (
-        "chance_of_playing_next_round" in pdata.keys()
-        and pdata["chance_of_playing_next_round"] is not None
-        and pdata["chance_of_playing_next_round"] <= 50
-    ):
-        # check if we have a return date
-        return_gameweek = get_return_gameweek_for_player(player_api_id, dbsession)
-        if return_gameweek is None or return_gameweek > gameweek:
-            return True
-    return False
-
-
 def fill_ep(csv_filename, dbsession=session):
     """
     fill the database with FPLs ep_next prediction, and also