Merge pull request #42 from MLDERES/dev

Refactor for projections/actual

.github/workflows/ci.yml

@@ -43,6 +43,4 @@ jobs:
     - name: Run flake8
       run: flake8
     - name: tests
-      run: |
-        pip install .[tests]
-        pytest
+      run: pytest

.ipython/profile_default/startup/00-kedro-init.py

@@ -1,7 +1,6 @@
 import logging.config
 import sys
 from pathlib import Path
-
 from IPython.core.magic import needs_local_scope, register_line_magic
 
 # Find the project root (./../../../)

CHANGELOG.md

@@ -0,0 +1,15 @@
+# Changelog
+
+## [Version] - 2020-09-06
+
+### [Unreleased]
+
+
+### Changed
+
+[0.4.0] - 2020-09-06
+
+### Changed
+
+- Refactored in preparation for weekly projections/results
+- !Breaking removed the possibility of there being a "default" pipeline

Makefile

@@ -25,22 +25,22 @@ notebook_convert:
 ## Clean up the raw data files - delete all files in 01_raw
 ## (data will need to be re-downloaded)
 clean-raw:
-	rm -fv data/01_raw/*
+	find data/01_raw -type f -name '*.csv' -delete
 
 ## Clean intermediate files - delete all files in 02_intermediate
 clean-intermediate:
-	rm -fv data/02_intermediate/*
+	find data/02_intermediate -type f -name '*.csv' -delete
 
 ## Clean primary data - delete all files in 03_primary
 clean-primary:
-	rm -fv data/03_primary/*
+	find data/03_primary -type f -name '*.csv' -delete
 
 ## Remove files associated with data engineering (primary/intermediate)
 clean-de: clean-primary clean-intermediate
 
 ## delete data features - delete all files in 04_feature
 clean-features:
-	rm -fv data/04_feature/*
+	find data/04_feature -type f -name '*.csv' -delete
 
 ## delete all generated data (features, primary, intermediate, raw)
 clean-data: clean-features clean-primary clean-intermediate clean-raw
@@ -67,6 +67,7 @@ pre-check: unittest
 	black .
 	flake8
 	mypy
+
 ## Update version for a patch (do this after pulling)
 patch: 
 	bump2version --allow-dirty --verbose patch

conf/base/catalog.yml

@@ -9,6 +9,9 @@
 # https://kedro.readthedocs.io/en/stable/04_user_guide/04_data_catalog.html
 _csv: &csv
   type: pandas.CSVDataSet
+  filepath: data
+  load_args:
+    parse_dates: True
 
 _remote: &remote_CSV
   type: phantasyfootballer.io.CachedRemoteCSVDataSet
@@ -18,115 +21,110 @@ _remote: &remote_CSV
   expiration: 24H
   year: 2020
 
-pfr_rushing:
-  type: phantasyfootballer.io.CSVRemoteDataSet
-  data_source : 'phantasyfootballer.data_providers.pro_fb_ref.fetch_rushing_data'
-  date_range_type : "year"
-  start_year : 2019
-
-pfr_passing:
-  type: phantasyfootballer.io.CSVRemoteDataSet
-  data_source : 'phantasyfootballer.data_providers.pro_fb_ref.fetch_passing_data'
-  date_range_type : "year"
-  start_year : 2019
-
-pfr_receiving:
-  type: phantasyfootballer.io.CSVRemoteDataSet
-  data_source : 'phantasyfootballer.data_providers.pro_fb_ref.fetch_receiving_data'
-  date_range_type : "year"
-  start_year : 2019
-
-fp_ecr_raw:
-  type: pandas.CSVDataSet
-  filepath: https://raw.githubusercontent.com/DynastyProcess/data/master/files/db_fpecr.csv
-  load_args:
-    parse_dates: True
-
 dp_ecr_local:
   <<: *csv
-  filepath: data/01_raw/dynasty_process_ecr.csv
+  filepath: data/01_raw/projected.annual/ecr/dynasty_process_ecr.csv
 
-fp_projections_remote:
+projections.annual.fp-remote:
   <<: *remote_CSV
   data_source: 'phantasyfootballer.data_providers.fantasy_pros.fetch_projections'
-  local_filepath: data/01_raw/fantasy_pros_projection.csv
+  local_filepath: data/01_raw/projected.annual/sources/fantasy_pros.csv
 
-cbs_projections_remote:
-  <<: *remote_CSV
-  data_source: 'phantasyfootballer.data_providers.cbs_sports.fetch_projections'
-  local_filepath: data/01_raw/cbs_sports_projection.csv
-
-
-fp_projections_local:
+projections.annual.fp-local:
   <<: *csv
-  filepath: data/01_raw/fantasy_pros_projection.csv
+  filepath: data/01_raw/projected.annual/sources/fantasy_pros.csv
   #versioned: True
 
-cbs_projections_local:
+projections.annual.cbs-remote:
+  <<: *remote_CSV
+  data_source: 'phantasyfootballer.data_providers.cbs_sports.fetch_projections'
+  local_filepath: data/01_raw/projected.annual/sources/cbs.csv
+
+projections.annual.cbs-local:
   <<: *csv
-  filepath: data/01_raw/cbs_sports_projection.csv
+  filepath: data/01_raw/projected.annual/sources/cbs.csv
   #versioned: True
 
-projections:
+projections.annual:
   <<: *csv
-  filepath: data/02_intermediate/projections.csv
+  filepath: data/01_raw/projected.annual/2020_projections.csv
   #versioned: True
 
 scoring.ppr:
   <<: *csv
-  filepath: data/03_primary/scoring_ppr.csv
+  filepath: data/03_primary/projected.annual/scoring_ppr.csv
   load_args:
     index_col: player
 
 scoring.half_ppr:
   <<: *csv
-  filepath: data/03_primary/scoring_half_ppr.csv
+  filepath: data/03_primary/projected.annual/scoring_half_ppr.csv
   load_args:
     index_col: player
 
 scoring.standard:
   <<: *csv
-  filepath: data/03_primary/scoring_std.csv
+  filepath: data/03_primary/projected.annual/scoring_std.csv
   load_args:
     index_col: player
 
 scoring.custom:
   <<: *csv
-  filepath: data/03_primary/scoring_custom.csv
+  filepath: data/03_primary/projected.annual/scoring_custom.csv
   load_args:
     index_col: player
 
-historical_yearly_ext:
+results.annual.source:
   type: "PartitionedDataSet"
   dataset: "pandas.CSVDataSet"
   path: data/00_external/yearly/
   filename_suffix: ".csv"
 
-historical_yearly_raw:
+results.annual.raw:
   <<: *csv
-  filepath: data/01_raw/historical.csv
+  filepath: data/01_raw/results.annual/2020.csv
+
+results.weekly.raw:
+  type: "PartitionedDataSet"
+  dataset: "pandas.CSVDataSet"
+  path: data/00_external/yearly/
+  filename_suffix: ".csv"
+
+results.weekly.remote:
+  <<: *remote_CSV
+  data_source: 'phantasyfootballer.data_providers.pro_fb_ref.fetch_weekly_results'
+  local_filepath: data/01_raw/results.weekly/sources/pro_results.csv
+
+results.weekly:
+  <<: *remote_CSV
+  data_source: 'phantasyfootballer.data_providers.pro_fb_ref.fetch_weekly_results'
+  local_filepath: data/01_raw/results.weekly/sources/pro_results.csv
+
 
 scoring.historical.half_ppr:
   <<: *csv
-  filepath: data/03_primary/hist_scoring_half_ppr.csv
+  filepath: data/03_primary/results.annual/hist_scoring_half_ppr.csv
   load_args:
     index_col: player
 
 scoring.historical.ppr:
   <<: *csv
-  filepath: data/03_primary/hist_scoring_ppr.csv
+  filepath: data/03_primary/results.annual/hist_scoring_ppr.csv
   load_args:
     index_col: player
 
 scoring.historical.standard:
   <<: *csv
-  filepath: data/03_primary/hist_scoring_std.csv
+  filepath: data/03_primary/results.annual/hist_scoring_std.csv
   load_args:
     index_col: player
 
 scoring.historical.custom:
   <<: *csv
-  filepath: data/03_primary/hist_scoring_custom.csv
+  filepath: data/03_primary/results.annual/hist_scoring_custom.csv
   load_args:
     index_col: player
 
+annual_projections:
+  <<: *csv
+  filepath: data/02_intermediate/average.csv

conf/base/parameters.yml

@@ -48,3 +48,6 @@ player_name_alias:
         - Mitch Trubisky
     Gardner Minshew:
         - Gardner Minshew II
+
+current_nfl_week: 99
+current_nfl_year: 2020

conf/base/pipelines/data_engineering/catalog.yml

@@ -1,5 +0,0 @@
-# This is a boilerplate catalog config generated for pipeline 'data_engineering'
-# using Kedro 0.16.4.
-#
-# Documentation for this file format can be found in "The Data Catalog"
-# Link: https://kedro.readthedocs.io/en/stable/04_user_guide/04_data_catalog.html

conf/base/pipelines/data_engineering/parameters.yml

@@ -1,5 +1 @@
-# This is a boilerplate parameters config generated for pipeline 'data_engineering'
-# using Kedro 0.16.4.
-#
-# Documentation for this file format can be found in
-# https://kedro.readthedocs.io/en/stable/04_user_guide/03_configuration.html#parameters
+

docs/source/conf.py

@@ -15,10 +15,11 @@
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
 import re
+from recommonmark.transform import AutoStructify
 
 from kedro.framework.cli.utils import find_stylesheets  # noqa: F401
+
 from phantasyfootballer import __version__ as release
-from recommonmark.transform import AutoStructify
 
 # -- Project information -----------------------------------------------------
 

kedro_cli.py

@@ -32,8 +32,8 @@
 from itertools import chain
 from pathlib import Path
 from typing import Dict, Iterable, Tuple
-
 import click
+
 from kedro.framework.cli import main as kedro_main
 from kedro.framework.cli.catalog import catalog as catalog_group
 from kedro.framework.cli.jupyter import jupyter as jupyter_group

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.5
+current_version = 0.4.0
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<build>\d+))?
@@ -24,6 +24,13 @@ include_trailing_comma = True
 force_grid_wrap = 0
 use_parentheses = True
 line_length = 88
+known_kedro = kedro
+known_first_party = phantasyfootballer
+sections = FUTURE, STDLIB, THIRDPARTY, KEDRO, FIRSTPARTY, LOCALFOLDER
+no_lines_before = FUTURE, STDLIB, THIRDPARTY, LOCALFOLDER
+
+[black]
+exclude = setup.cfg
 
 [mypy]
 files = src

src/phantasyfootballer/__init__.py

@@ -28,5 +28,5 @@
 """phantasyfootballer
 """
 
-__version__ = "0.3.5"
+__version__ = "0.4.0"
 from .common import *