From c7521eb3ea2d385a00eadb3756cd03f6e2522e6f Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 28 Apr 2021 11:28:10 -0700
Subject: [PATCH 01/60] ci: add coverage report to pytest

---
 pytest.ini |  1 +
 tox.ini    | 10 ++++++----
 2 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/pytest.ini b/pytest.ini
index 5fc46bb50..4c08f3262 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -1,4 +1,5 @@
 [pytest]
+addopts = --cov=powersimdata
 markers =
     integration: marks tests that require external dependencies (deselect with '-m "not integration"')
     db: marks tests that connect to a local database
diff --git a/tox.ini b/tox.ini
index cf914c7f2..5a9f4784c 100644
--- a/tox.ini
+++ b/tox.ini
@@ -3,20 +3,22 @@ envlist = pytest-local, format, flake8
 skipsdist = true
 
 [testenv]
-passenv = 
+passenv =
     CPPFLAGS
     LDFLAGS
-deps = 
+deps =
     pytest: pipenv
+    pytest: coverage
+    pytest: pytest-cov
     {format,checkformatting}: black
     {format,checkformatting}: isort
     flake8: flake8
     flake8: pep8-naming
 commands =
     pytest: pipenv sync --dev
-    ci: pytest -m 'not ssh' 
+    ci: pytest -m 'not ssh'
     local: pytest -m 'not integration'
-    integration: pytest 
+    integration: pytest
     format: black .
     format: isort .
     checkformatting: black . --check --diff

From 32b68fe980d34848150f6efaf1de734484a1d3b8 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 29 Apr 2021 14:21:54 -0700
Subject: [PATCH 02/60] chore: add dev packages to project wide install

---
 .github/workflows/test.yml |   2 +-
 Pipfile                    |   4 +-
 Pipfile.lock               | 366 ++++++++++++++++++++-----------------
 requirements.txt           |   6 +-
 tox.ini                    |   2 -
 5 files changed, 206 insertions(+), 174 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 4372013dd..bf58af51e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -9,7 +9,7 @@ jobs:
     strategy:
         fail-fast: false
         matrix:
-          python-version: [3.7, 3.8]
+          python-version: [3.7, 3.8, 3.9]
 
     name: Python ${{ matrix.python-version }}
     steps:
diff --git a/Pipfile b/Pipfile
index e598387ed..f784c092d 100644
--- a/Pipfile
+++ b/Pipfile
@@ -5,12 +5,14 @@ verify_ssl = true
 
 [dev-packages]
 black = "*"
+pytest = "*"
+coverage = "*"
+pytest-cov = "*"
 
 [packages]
 numpy = "~=1.20"
 pandas = "~=1.2"
 paramiko = "==2.7.2"
-pytest = "==5.4.3"
 scipy = "~=1.5"
 tqdm = "==4.29.1"
 psycopg2 = "~=2.8.5"
diff --git a/Pipfile.lock b/Pipfile.lock
index aad2b6521..f590f9fa6 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "27069ab7060eb806bbbc50330c27f6f85e7bfd159f756f2a3030137ec3c18702"
+            "sha256": "84a6fc7c5ebb2d3bcdc557de8e5df841f7cdef2894cde2ecbf73b1331a73fd4b"
         },
         "pipfile-spec": 6,
         "requires": {},
@@ -14,13 +14,6 @@
         ]
     },
     "default": {
-        "attrs": {
-            "hashes": [
-                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
-                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
-            ],
-            "version": "==20.3.0"
-        },
         "bcrypt": {
             "hashes": [
                 "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29",
@@ -31,6 +24,7 @@
                 "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1",
                 "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==3.2.0"
         },
         "certifi": {
@@ -87,6 +81,7 @@
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
                 "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.0.0"
         },
         "cryptography": {
@@ -104,6 +99,7 @@
                 "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
                 "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==3.4.7"
         },
         "idna": {
@@ -111,75 +107,60 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
-        "more-itertools": {
-            "hashes": [
-                "sha256:5652a9ac72209ed7df8d9c15daf4e1aa0e3d2ccd3c87f8265a0673cd9cbc9ced",
-                "sha256:c5d6da9ca3ff65220c3bfd2a8db06d698f05d4d2b9be57e1deb2be5a45019713"
-            ],
-            "version": "==8.7.0"
-        },
         "numpy": {
             "hashes": [
-                "sha256:032be656d89bbf786d743fee11d01ef318b0781281241997558fa7950028dd29",
-                "sha256:104f5e90b143dbf298361a99ac1af4cf59131218a045ebf4ee5990b83cff5fab",
-                "sha256:125a0e10ddd99a874fd357bfa1b636cd58deb78ba4a30b5ddb09f645c3512e04",
-                "sha256:12e4ba5c6420917571f1a5becc9338abbde71dd811ce40b37ba62dec7b39af6d",
-                "sha256:13adf545732bb23a796914fe5f891a12bd74cf3d2986eed7b7eba2941eea1590",
-                "sha256:2d7e27442599104ee08f4faed56bb87c55f8b10a5494ac2ead5c98a4b289e61f",
-                "sha256:3bc63486a870294683980d76ec1e3efc786295ae00128f9ea38e2c6e74d5a60a",
-                "sha256:3d3087e24e354c18fb35c454026af3ed8997cfd4997765266897c68d724e4845",
-                "sha256:4ed8e96dc146e12c1c5cdd6fb9fd0757f2ba66048bf94c5126b7efebd12d0090",
-                "sha256:60759ab15c94dd0e1ed88241fd4fa3312db4e91d2c8f5a2d4cf3863fad83d65b",
-                "sha256:65410c7f4398a0047eea5cca9b74009ea61178efd78d1be9847fac1d6716ec1e",
-                "sha256:66b467adfcf628f66ea4ac6430ded0614f5cc06ba530d09571ea404789064adc",
-                "sha256:7199109fa46277be503393be9250b983f325880766f847885607d9b13848f257",
-                "sha256:72251e43ac426ff98ea802a931922c79b8d7596480300eb9f1b1e45e0543571e",
-                "sha256:89e5336f2bec0c726ac7e7cdae181b325a9c0ee24e604704ed830d241c5e47ff",
-                "sha256:89f937b13b8dd17b0099c7c2e22066883c86ca1575a975f754babc8fbf8d69a9",
-                "sha256:9c94cab5054bad82a70b2e77741271790304651d584e2cdfe2041488e753863b",
-                "sha256:9eb551d122fadca7774b97db8a112b77231dcccda8e91a5bc99e79890797175e",
-                "sha256:a1d7995d1023335e67fb070b2fae6f5968f5be3802b15ad6d79d81ecaa014fe0",
-                "sha256:ae61f02b84a0211abb56462a3b6cd1e7ec39d466d3160eb4e1da8bf6717cdbeb",
-                "sha256:b9410c0b6fed4a22554f072a86c361e417f0258838957b78bd063bde2c7f841f",
-                "sha256:c26287dfc888cf1e65181f39ea75e11f42ffc4f4529e5bd19add57ad458996e2",
-                "sha256:c91ec9569facd4757ade0888371eced2ecf49e7982ce5634cc2cf4e7331a4b14",
-                "sha256:ecb5b74c702358cdc21268ff4c37f7466357871f53a30e6f84c686952bef16a9"
+                "sha256:2428b109306075d89d21135bdd6b785f132a1f5a3260c371cee1fae427e12727",
+                "sha256:377751954da04d4a6950191b20539066b4e19e3b559d4695399c5e8e3e683bf6",
+                "sha256:4703b9e937df83f5b6b7447ca5912b5f5f297aba45f91dbbbc63ff9278c7aa98",
+                "sha256:471c0571d0895c68da309dacee4e95a0811d0a9f9f532a48dc1bea5f3b7ad2b7",
+                "sha256:61d5b4cf73622e4d0c6b83408a16631b670fc045afd6540679aa35591a17fe6d",
+                "sha256:6c915ee7dba1071554e70a3664a839fbc033e1d6528199d4621eeaaa5487ccd2",
+                "sha256:6e51e417d9ae2e7848314994e6fc3832c9d426abce9328cf7571eefceb43e6c9",
+                "sha256:719656636c48be22c23641859ff2419b27b6bdf844b36a2447cb39caceb00935",
+                "sha256:780ae5284cb770ade51d4b4a7dce4faa554eb1d88a56d0e8b9f35fca9b0270ff",
+                "sha256:878922bf5ad7550aa044aa9301d417e2d3ae50f0f577de92051d739ac6096cee",
+                "sha256:924dc3f83de20437de95a73516f36e09918e9c9c18d5eac520062c49191025fb",
+                "sha256:97ce8b8ace7d3b9288d88177e66ee75480fb79b9cf745e91ecfe65d91a856042",
+                "sha256:9c0fab855ae790ca74b27e55240fe4f2a36a364a3f1ebcfd1fb5ac4088f1cec3",
+                "sha256:9cab23439eb1ebfed1aaec9cd42b7dc50fc96d5cd3147da348d9161f0501ada5",
+                "sha256:a8e6859913ec8eeef3dbe9aed3bf475347642d1cdd6217c30f28dee8903528e6",
+                "sha256:aa046527c04688af680217fffac61eec2350ef3f3d7320c07fd33f5c6e7b4d5f",
+                "sha256:abc81829c4039e7e4c30f7897938fa5d4916a09c2c7eb9b244b7a35ddc9656f4",
+                "sha256:bad70051de2c50b1a6259a6df1daaafe8c480ca98132da98976d8591c412e737",
+                "sha256:c73a7975d77f15f7f68dacfb2bca3d3f479f158313642e8ea9058eea06637931",
+                "sha256:d15007f857d6995db15195217afdbddfcd203dfaa0ba6878a2f580eaf810ecd6",
+                "sha256:d76061ae5cab49b83a8cf3feacefc2053fac672728802ac137dd8c4123397677",
+                "sha256:e8e4fbbb7e7634f263c5b0150a629342cc19b47c5eba8d1cd4363ab3455ab576",
+                "sha256:e9459f40244bb02b2f14f6af0cd0732791d72232bbb0dc4bab57ef88e75f6935",
+                "sha256:edb1f041a9146dcf02cd7df7187db46ab524b9af2515f392f337c7cbbf5b52cd"
             ],
             "index": "pypi",
-            "version": "==1.20.1"
-        },
-        "packaging": {
-            "hashes": [
-                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
-                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
-            ],
-            "version": "==20.9"
+            "version": "==1.20.2"
         },
         "pandas": {
             "hashes": [
-                "sha256:05ca6bda50123158eb15e716789083ca4c3b874fd47688df1716daa72644ee1c",
-                "sha256:08b6bbe74ae2b3e4741a744d2bce35ce0868a6b4189d8b84be26bb334f73da4c",
-                "sha256:14ed84b463e9b84c8ff9308a79b04bf591ae3122a376ee0f62c68a1bd917a773",
-                "sha256:214ae60b1f863844e97c87f758c29940ffad96c666257323a4bb2a33c58719c2",
-                "sha256:230de25bd9791748b2638c726a5f37d77a96a83854710110fadd068d1e2c2c9f",
-                "sha256:26b4919eb3039a686a86cd4f4a74224f8f66e3a419767da26909dcdd3b37c31e",
-                "sha256:4d33537a375cfb2db4d388f9a929b6582a364137ea6c6b161b0166440d6ffe36",
-                "sha256:69a70d79a791fa1fd5f6e84b8b6dec2ec92369bde4ab2e18d43fc8a1825f51d1",
-                "sha256:8ac028cd9a6e1efe43f3dc36f708263838283535cc45430a98b9803f44f4c84b",
-                "sha256:a50cf3110a1914442e7b7b9cef394ef6bed0d801b8a34d56f4c4e927bbbcc7d0",
-                "sha256:c43d1beb098a1da15934262009a7120aac8dafa20d042b31dab48c28868eb5a4",
-                "sha256:c76a108272a4de63189b8f64086bbaf8348841d7e610b52f50959fbbf401524f",
-                "sha256:cbad4155028b8ca66aa19a8b13f593ebbf51bfb6c3f2685fe64f04d695a81864",
-                "sha256:e3c250faaf9979d0ec836d25e420428db37783fa5fed218da49c9fc06f80f51c",
-                "sha256:e61a089151f1ed78682aa77a3bcae0495cf8e585546c26924857d7e8a9960568",
-                "sha256:e9bbcc7b5c432600797981706f5b54611990c6a86b2e424329c995eea5f9c42b",
-                "sha256:fbddbb20f30308ba2546193d64e18c23b69f59d48cdef73676cbed803495c8dc",
-                "sha256:fc351cd2df318674669481eb978a7799f24fd14ef26987a1aa75105b0531d1a1"
+                "sha256:167693a80abc8eb28051fbd184c1b7afd13ce2c727a5af47b048f1ea3afefff4",
+                "sha256:2111c25e69fa9365ba80bbf4f959400054b2771ac5d041ed19415a8b488dc70a",
+                "sha256:298f0553fd3ba8e002c4070a723a59cdb28eda579f3e243bc2ee397773f5398b",
+                "sha256:2b063d41803b6a19703b845609c0b700913593de067b552a8b24dd8eeb8c9895",
+                "sha256:2cb7e8f4f152f27dc93f30b5c7a98f6c748601ea65da359af734dd0cf3fa733f",
+                "sha256:52d2472acbb8a56819a87aafdb8b5b6d2b3386e15c95bde56b281882529a7ded",
+                "sha256:612add929bf3ba9d27b436cc8853f5acc337242d6b584203f207e364bb46cb12",
+                "sha256:649ecab692fade3cbfcf967ff936496b0cfba0af00a55dfaacd82bdda5cb2279",
+                "sha256:68d7baa80c74aaacbed597265ca2308f017859123231542ff8a5266d489e1858",
+                "sha256:8d4c74177c26aadcfb4fd1de6c1c43c2bf822b3e0fc7a9b409eeaf84b3e92aaa",
+                "sha256:971e2a414fce20cc5331fe791153513d076814d30a60cd7348466943e6e909e4",
+                "sha256:9db70ffa8b280bb4de83f9739d514cd0735825e79eef3a61d312420b9f16b758",
+                "sha256:b730add5267f873b3383c18cac4df2527ac4f0f0eed1c6cf37fcb437e25cf558",
+                "sha256:bd659c11a4578af740782288cac141a322057a2e36920016e0fc7b25c5a4b686",
+                "sha256:c601c6fdebc729df4438ec1f62275d6136a0dd14d332fc0e8ce3f7d2aadb4dd6",
+                "sha256:d0877407359811f7b853b548a614aacd7dea83b0c0c84620a9a643f180060950"
             ],
             "index": "pypi",
-            "version": "==1.2.2"
+            "version": "==1.2.4"
         },
         "paramiko": {
             "hashes": [
@@ -189,13 +170,6 @@
             "index": "pypi",
             "version": "==2.7.2"
         },
-        "pluggy": {
-            "hashes": [
-                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
-                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
-            ],
-            "version": "==0.13.1"
-        },
         "psycopg2": {
             "hashes": [
                 "sha256:00195b5f6832dbf2876b8bf77f12bdce648224c89c880719c745b90515233301",
@@ -217,18 +191,12 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
-        "py": {
-            "hashes": [
-                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
-                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
-            ],
-            "version": "==1.10.0"
-        },
         "pycparser": {
             "hashes": [
                 "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
                 "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.20"
         },
         "pynacl": {
@@ -252,28 +220,15 @@
                 "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff",
                 "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.4.0"
         },
-        "pyparsing": {
-            "hashes": [
-                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
-                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
-            ],
-            "version": "==2.4.7"
-        },
-        "pytest": {
-            "hashes": [
-                "sha256:5c0db86b698e8f170ba4582a492248919255fcd4c79b1ee64ace34301fb589a1",
-                "sha256:7979331bfcba207414f5e1263b5a0f8f521d0f457318836a7355531ed1a4c7d8"
-            ],
-            "index": "pypi",
-            "version": "==5.4.3"
-        },
         "python-dateutil": {
             "hashes": [
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
                 "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.1"
         },
         "pytz": {
@@ -293,34 +248,35 @@
         },
         "scipy": {
             "hashes": [
-                "sha256:0c8a51d33556bf70367452d4d601d1742c0e806cd0194785914daf19775f0e67",
-                "sha256:0e5b0ccf63155d90da576edd2768b66fb276446c371b73841e3503be1d63fb5d",
-                "sha256:2481efbb3740977e3c831edfd0bd9867be26387cacf24eb5e366a6a374d3d00d",
-                "sha256:33d6b7df40d197bdd3049d64e8e680227151673465e5d85723b3b8f6b15a6ced",
-                "sha256:5da5471aed911fe7e52b86bf9ea32fb55ae93e2f0fac66c32e58897cfb02fa07",
-                "sha256:5f331eeed0297232d2e6eea51b54e8278ed8bb10b099f69c44e2558c090d06bf",
-                "sha256:5fa9c6530b1661f1370bcd332a1e62ca7881785cc0f80c0d559b636567fab63c",
-                "sha256:6725e3fbb47da428794f243864f2297462e9ee448297c93ed1dcbc44335feb78",
-                "sha256:68cb4c424112cd4be886b4d979c5497fba190714085f46b8ae67a5e4416c32b4",
-                "sha256:794e768cc5f779736593046c9714e0f3a5940bc6dcc1dba885ad64cbfb28e9f0",
-                "sha256:83bf7c16245c15bc58ee76c5418e46ea1811edcc2e2b03041b804e46084ab627",
-                "sha256:8e403a337749ed40af60e537cc4d4c03febddcc56cd26e774c9b1b600a70d3e4",
-                "sha256:a15a1f3fc0abff33e792d6049161b7795909b40b97c6cc2934ed54384017ab76",
-                "sha256:a423533c55fec61456dedee7b6ee7dce0bb6bfa395424ea374d25afa262be261",
-                "sha256:a5193a098ae9f29af283dcf0041f762601faf2e595c0db1da929875b7570353f",
-                "sha256:bd50daf727f7c195e26f27467c85ce653d41df4358a25b32434a50d8870fc519",
-                "sha256:c4fceb864890b6168e79b0e714c585dbe2fd4222768ee90bc1aa0f8218691b11",
-                "sha256:e79570979ccdc3d165456dd62041d9556fb9733b86b4b6d818af7a0afc15f092",
-                "sha256:f46dd15335e8a320b0fb4685f58b7471702234cba8bb3442b69a3e1dc329c345"
+                "sha256:01b38dec7e9f897d4db04f8de4e20f0f5be3feac98468188a0f47a991b796055",
+                "sha256:10dbcc7de03b8d635a1031cb18fd3eaa997969b64fdf78f99f19ac163a825445",
+                "sha256:19aeac1ad3e57338723f4657ac8520f41714804568f2e30bd547d684d72c392e",
+                "sha256:1b21c6e0dc97b1762590b70dee0daddb291271be0580384d39f02c480b78290a",
+                "sha256:1caade0ede6967cc675e235c41451f9fb89ae34319ddf4740194094ab736b88d",
+                "sha256:23995dfcf269ec3735e5a8c80cfceaf384369a47699df111a6246b83a55da582",
+                "sha256:2a799714bf1f791fb2650d73222b248d18d53fd40d6af2df2c898db048189606",
+                "sha256:3274ce145b5dc416c49c0cf8b6119f787f0965cd35e22058fe1932c09fe15d77",
+                "sha256:33d1677d46111cfa1c84b87472a0274dde9ef4a7ef2e1f155f012f5f1e995d8f",
+                "sha256:44d452850f77e65e25b1eb1ac01e25770323a782bfe3a1a3e43847ad4266d93d",
+                "sha256:9e3302149a369697c6aaea18b430b216e3c88f9a61b62869f6104881e5f9ef85",
+                "sha256:a75b014d3294fce26852a9d04ea27b5671d86736beb34acdfc05859246260707",
+                "sha256:ad7269254de06743fb4768f658753de47d8b54e4672c5ebe8612a007a088bd48",
+                "sha256:b30280fbc1fd8082ac822994a98632111810311a9ece71a0e48f739df3c555a2",
+                "sha256:b79104878003487e2b4639a20b9092b02e1bad07fc4cf924b495cf413748a777",
+                "sha256:d449d40e830366b4c612692ad19fbebb722b6b847f78a7b701b1e0d6cda3cc13",
+                "sha256:d647757373985207af3343301d89fe738d5a294435a4f2aafb04c13b4388c896",
+                "sha256:f68eb46b86b2c246af99fcaa6f6e37c7a7a413e1084a794990b877f2ff71f7b6",
+                "sha256:fdf606341cd798530b05705c87779606fcdfaf768a8129c348ea94441da15b04"
             ],
             "index": "pypi",
-            "version": "==1.6.1"
+            "version": "==1.6.3"
         },
         "six": {
             "hashes": [
                 "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
                 "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.15.0"
         },
         "tqdm": {
@@ -336,15 +292,8 @@
                 "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
                 "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
             ],
-            "index": "pypi",
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
             "version": "==1.26.4"
-        },
-        "wcwidth": {
-            "hashes": [
-                "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784",
-                "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83"
-            ],
-            "version": "==0.2.5"
         }
     },
     "develop": {
@@ -355,20 +304,95 @@
             ],
             "version": "==1.4.4"
         },
+        "attrs": {
+            "hashes": [
+                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
+                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.3.0"
+        },
         "black": {
             "hashes": [
-                "sha256:1c02557aa099101b9d21496f8a914e9ed2222ef70336404eeeac8edba836fbea"
+                "sha256:bff7067d8bc25eb21dcfdbc8c72f2baafd9ec6de4663241a52fb904b304d391f",
+                "sha256:fc9bcf3b482b05c1f35f6a882c079dc01b9c7795827532f4cc43c0ec88067bbc"
             ],
             "index": "pypi",
-            "version": "==20.8b1"
+            "version": "==21.4b2"
         },
         "click": {
             "hashes": [
                 "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
                 "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==7.1.2"
         },
+        "coverage": {
+            "hashes": [
+                "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c",
+                "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6",
+                "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45",
+                "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a",
+                "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03",
+                "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529",
+                "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a",
+                "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a",
+                "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2",
+                "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6",
+                "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759",
+                "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53",
+                "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a",
+                "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4",
+                "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff",
+                "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502",
+                "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793",
+                "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb",
+                "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905",
+                "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821",
+                "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b",
+                "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81",
+                "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0",
+                "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b",
+                "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3",
+                "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184",
+                "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701",
+                "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a",
+                "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82",
+                "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638",
+                "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5",
+                "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083",
+                "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6",
+                "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90",
+                "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465",
+                "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a",
+                "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3",
+                "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e",
+                "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066",
+                "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf",
+                "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b",
+                "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae",
+                "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669",
+                "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873",
+                "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b",
+                "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6",
+                "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb",
+                "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160",
+                "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c",
+                "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079",
+                "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d",
+                "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6"
+            ],
+            "index": "pypi",
+            "version": "==5.5"
+        },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
         "mypy-extensions": {
             "hashes": [
                 "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
@@ -376,6 +400,14 @@
             ],
             "version": "==0.4.3"
         },
+        "packaging": {
+            "hashes": [
+                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
+                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.9"
+        },
         "pathspec": {
             "hashes": [
                 "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd",
@@ -383,6 +415,46 @@
             ],
             "version": "==0.8.1"
         },
+        "pluggy": {
+            "hashes": [
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.13.1"
+        },
+        "py": {
+            "hashes": [
+                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
+                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.10.0"
+        },
+        "pyparsing": {
+            "hashes": [
+                "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
+                "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
+            ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==2.4.7"
+        },
+        "pytest": {
+            "hashes": [
+                "sha256:671238a46e4df0f3498d1c3270e5deb9b32d25134c99b7d75370a68cfbe9b634",
+                "sha256:6ad9c7bdf517a808242b998ac20063c41532a570d088d77eec1ee12b0b5574bc"
+            ],
+            "index": "pypi",
+            "version": "==6.2.3"
+        },
+        "pytest-cov": {
+            "hashes": [
+                "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7",
+                "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da"
+            ],
+            "index": "pypi",
+            "version": "==2.11.1"
+        },
         "regex": {
             "hashes": [
                 "sha256:01afaf2ec48e196ba91b37451aa353cb7eda77efe518e481707e0515025f0cd5",
@@ -434,50 +506,8 @@
                 "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                 "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
             ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.10.2"
-        },
-        "typed-ast": {
-            "hashes": [
-                "sha256:07d49388d5bf7e863f7fa2f124b1b1d89d8aa0e2f7812faff0a5658c01c59aa1",
-                "sha256:14bf1522cdee369e8f5581238edac09150c765ec1cb33615855889cf33dcb92d",
-                "sha256:240296b27397e4e37874abb1df2a608a92df85cf3e2a04d0d4d61055c8305ba6",
-                "sha256:36d829b31ab67d6fcb30e185ec996e1f72b892255a745d3a82138c97d21ed1cd",
-                "sha256:37f48d46d733d57cc70fd5f30572d11ab8ed92da6e6b28e024e4a3edfb456e37",
-                "sha256:4c790331247081ea7c632a76d5b2a265e6d325ecd3179d06e9cf8d46d90dd151",
-                "sha256:5dcfc2e264bd8a1db8b11a892bd1647154ce03eeba94b461effe68790d8b8e07",
-                "sha256:7147e2a76c75f0f64c4319886e7639e490fee87c9d25cb1d4faef1d8cf83a440",
-                "sha256:7703620125e4fb79b64aa52427ec192822e9f45d37d4b6625ab37ef403e1df70",
-                "sha256:8368f83e93c7156ccd40e49a783a6a6850ca25b556c0fa0240ed0f659d2fe496",
-                "sha256:84aa6223d71012c68d577c83f4e7db50d11d6b1399a9c779046d75e24bed74ea",
-                "sha256:85f95aa97a35bdb2f2f7d10ec5bbdac0aeb9dafdaf88e17492da0504de2e6400",
-                "sha256:8db0e856712f79c45956da0c9a40ca4246abc3485ae0d7ecc86a20f5e4c09abc",
-                "sha256:9044ef2df88d7f33692ae3f18d3be63dec69c4fb1b5a4a9ac950f9b4ba571606",
-                "sha256:963c80b583b0661918718b095e02303d8078950b26cc00b5e5ea9ababe0de1fc",
-                "sha256:987f15737aba2ab5f3928c617ccf1ce412e2e321c77ab16ca5a293e7bbffd581",
-                "sha256:9ec45db0c766f196ae629e509f059ff05fc3148f9ffd28f3cfe75d4afb485412",
-                "sha256:9fc0b3cb5d1720e7141d103cf4819aea239f7d136acf9ee4a69b047b7986175a",
-                "sha256:a2c927c49f2029291fbabd673d51a2180038f8cd5a5b2f290f78c4516be48be2",
-                "sha256:a38878a223bdd37c9709d07cd357bb79f4c760b29210e14ad0fb395294583787",
-                "sha256:b4fcdcfa302538f70929eb7b392f536a237cbe2ed9cba88e3bf5027b39f5f77f",
-                "sha256:c0c74e5579af4b977c8b932f40a5464764b2f86681327410aa028a22d2f54937",
-                "sha256:c1c876fd795b36126f773db9cbb393f19808edd2637e00fd6caba0e25f2c7b64",
-                "sha256:c9aadc4924d4b5799112837b226160428524a9a45f830e0d0f184b19e4090487",
-                "sha256:cc7b98bf58167b7f2db91a4327da24fb93368838eb84a44c472283778fc2446b",
-                "sha256:cf54cfa843f297991b7388c281cb3855d911137223c6b6d2dd82a47ae5125a41",
-                "sha256:d003156bb6a59cda9050e983441b7fa2487f7800d76bdc065566b7d728b4581a",
-                "sha256:d175297e9533d8d37437abc14e8a83cbc68af93cc9c1c59c2c292ec59a0697a3",
-                "sha256:d746a437cdbca200622385305aedd9aef68e8a645e385cc483bdc5e488f07166",
-                "sha256:e683e409e5c45d5c9082dc1daf13f6374300806240719f95dc783d1fc942af10"
-            ],
-            "version": "==1.4.2"
-        },
-        "typing-extensions": {
-            "hashes": [
-                "sha256:7cb407020f00f7bfc3cb3e7881628838e69d8f3fcab2f64742a5e76b2f841918",
-                "sha256:99d4073b617d30288f569d3f13d2bd7548c3a7e4c8de87db09a9d29bb3a4a60c",
-                "sha256:dafc7639cde7f1b6e1acc0f457842a83e722ccca8eef5270af2d74792619a89f"
-            ],
-            "version": "==3.7.4.3"
         }
     }
 }
diff --git a/requirements.txt b/requirements.txt
index 3ae0a3a13..4f7d00a3b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,11 @@
-black==20.8b1
 numpy~=1.20
 pandas~=1.2
 paramiko==2.7.2
 psycopg2~=2.8.5
-pytest==5.4.3
 scipy~=1.5
 tqdm==4.29.1
 requests~=2.25
+black
+pytest
+coverage
+pytest-cov
diff --git a/tox.ini b/tox.ini
index 5a9f4784c..876c0de48 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,8 +8,6 @@ passenv =
     LDFLAGS
 deps =
     pytest: pipenv
-    pytest: coverage
-    pytest: pytest-cov
     {format,checkformatting}: black
     {format,checkformatting}: isort
     flake8: flake8

From 6431627132e09fbaf384db730703468c076a26cf Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 29 Apr 2021 15:18:03 -0700
Subject: [PATCH 03/60] chore: add .coveragerc and filter report

---
 .coveragerc | 6 ++++++
 1 file changed, 6 insertions(+)
 create mode 100644 .coveragerc

diff --git a/.coveragerc b/.coveragerc
new file mode 100644
index 000000000..ac59cf57c
--- /dev/null
+++ b/.coveragerc
@@ -0,0 +1,6 @@
+[run]
+branch = True
+omit =
+    **/test_*.py
+    **/__init__.py
+

From 7fbad851057beebaed3217e0c7f4d87efd49736f Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Mon, 3 May 2021 10:58:32 -0700
Subject: [PATCH 04/60] feat: allow specification of congestion metric for
 scale_congested_mesh_branches

---
 powersimdata/design/transmission/upgrade.py | 28 ++++++++++++++-------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/powersimdata/design/transmission/upgrade.py b/powersimdata/design/transmission/upgrade.py
index c9b6e2df4..1f47450a9 100644
--- a/powersimdata/design/transmission/upgrade.py
+++ b/powersimdata/design/transmission/upgrade.py
@@ -197,12 +197,16 @@ def scale_congested_mesh_branches(
     upgrade_n=100,
     allow_list=None,
     deny_list=None,
-    quantile=0.95,
+    congestion_metric="quantile",
+    cost_metric="branches",
+    quantile=None,
     increment=1,
-    method="branches",
 ):
-    """Use a reference scenario as a baseline for branch scaling, and further
-    increment branch scaling based on observed congestion duals.
+    """Use a reference scenario to detect congested mesh branches (based on values of
+    the shadow price of congestion), and increase the capacity of a subset of them.
+    Branches are ranked by a ratio of (congestion metric) / (cost metric), and the top N
+    branches are selected for upgrading, where N is specified by ``upgraade_n``, with
+    each upgraded by their base capacity multiplied by ``increment``.
 
     :param powersimdata.input.change_table.ChangeTable change_table: the
         change table instance we are operating on.
@@ -211,10 +215,15 @@ def scale_congested_mesh_branches(
     :param int upgrade_n: the number of branches to upgrade.
     :param list/set/tuple/None allow_list: only select from these branch IDs.
     :param list/set/tuple/None deny_list: never select any of these branch IDs.
-    :param float quantile: the quantile to use to judge branch congestion.
-    :param [float/int] increment: branch increment, relative to original
+    :param str congestion_metric: numerator method: 'quantile' or 'mean'.
+    :param str cost_metric: denominator method: 'branches', 'cost', 'MW', or
+        'MWmiles'.
+    :param float quantile: if ``congestion_metric`` == 'quantile', this is the quantile
+        to use to judge branch congestion (otherwise it is unused). If None, a default
+        value of 0.95 is used, i.e. we evaluate the shadow price for the worst 5% of
+        hours.
+    :param float/int increment: branch increment, relative to original
         capacity.
-    :param str method: prioritization method: 'branches', 'MW', or 'MWmiles'.
     :return: (*None*) -- the change_table is modified in-place.
     """
     # To do: better type checking of inputs.
@@ -223,10 +232,11 @@ def scale_congested_mesh_branches(
     branches_to_upgrade = _identify_mesh_branch_upgrades(
         ref_scenario,
         upgrade_n=upgrade_n,
-        quantile=quantile,
-        method=method,
+        congestion_metric=congestion_metric,
+        cost_metric=cost_metric,
         allow_list=allow_list,
         deny_list=deny_list,
+        quantile=quantile,
     )
     _increment_branch_scaling(
         change_table, branches_to_upgrade, ref_scenario, value=increment

From e45655ce843877499f25f574a918123c3441cb97 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Mon, 3 May 2021 12:04:36 -0700
Subject: [PATCH 05/60] feat: add 'mean' congestion metric logic

---
 powersimdata/design/transmission/upgrade.py | 97 ++++++++++++++-------
 1 file changed, 64 insertions(+), 33 deletions(-)

diff --git a/powersimdata/design/transmission/upgrade.py b/powersimdata/design/transmission/upgrade.py
index 1f47450a9..f816af7a6 100644
--- a/powersimdata/design/transmission/upgrade.py
+++ b/powersimdata/design/transmission/upgrade.py
@@ -246,24 +246,32 @@ def scale_congested_mesh_branches(
 def _identify_mesh_branch_upgrades(
     ref_scenario,
     upgrade_n=100,
-    quantile=0.95,
     allow_list=None,
     deny_list=None,
-    method="branches",
+    congestion_metric="quantile",
+    cost_metric="branches",
+    quantile=None,
 ):
     """Identify the N most congested branches in a previous scenario, based on
-    the quantile value of congestion duals. A quantile value of 0.95 obtains
-    the branches with highest dual in top 5% of hours.
+    the quantile value of congestion duals, where N is specified by ``upgrade_n``. A
+    quantile value of 0.95 obtains the branches with highest dual in top 5% of hours.
 
     :param powersimdata.scenario.scenario.Scenario ref_scenario: the reference
         scenario to be used to determine the most congested branches.
     :param int upgrade_n: the number of branches to upgrade.
-    :param float quantile: the quantile to use to judge branch congestion.
     :param list/set/tuple/None allow_list: only select from these branch IDs.
     :param list/set/tuple/None deny_list: never select any of these branch IDs.
-    :param str method: prioritization method: 'branches', 'MW', or 'MWmiles'.
-    :raises ValueError: if 'method' not recognized, or not enough branches to
-        upgrade.
+    :param str congestion_metric: numerator method: 'quantile' or 'mean'.
+    :param str cost_metric: denominator method: 'branches', 'cost', 'MW', or
+        'MWmiles'.
+    :param float quantile: if ``congestion_metric`` == 'quantile', this is the quantile
+        to use to judge branch congestion (otherwise it is unused). If None, a default
+        value of 0.95 is used, i.e. we evaluate the shadow price for the worst 5% of
+        hours.
+    :raises ValueError: if ``congestion_metric`` or ``cost_metric`` is not recognized,
+        ``congestion_metric`` == 'mean' but a ``quantile`` is specified, or
+        ``congestion_metric`` == 'quantile' but there are not enough branches which are
+        congested at the desired frequency based on the ``quantile`` specified.
     :return: (*set*) -- A set of ints representing branch indices.
     """
 
@@ -271,39 +279,61 @@ def _identify_mesh_branch_upgrades(
     cong_significance_cutoff = 1e-6  # $/MWh
     # If we rank by MW-miles, what 'length' do we give to zero-length branches?
     zero_length_value = 1  # miles
-
-    # Validate method input
-    allowed_methods = ("branches", "MW", "MWmiles", "cost")
-    if method not in allowed_methods:
-        allowed_list = ", ".join(allowed_methods)
-        raise ValueError(f"method must be one of: {allowed_list}")
+    # If the quantile is not provided, what should we default to?
+    default_quantile = 0.95
+
+    # Validate congestion_metric input
+    allowed_congestion_metrics = ("mean", "quantile")
+    if congestion_metric not in allowed_congestion_metrics:
+        allowed_list = ", ".join(allowed_congestion_metrics)
+        raise ValueError(f"congestion_metric must be one of: {allowed_list}")
+    if congestion_metric == "mean" and quantile is not None:
+        raise ValueError("quantile cannot be specified if congestion_metric is 'mean'")
+    if congestion_metric == "quantile" and quantile is None:
+        quantile = default_quantile
+
+    # Validate cost_metric input
+    allowed_cost_metrics = ("branches", "MW", "MWmiles", "cost")
+    if cost_metric not in allowed_cost_metrics:
+        allowed_list = ", ".join(allowed_cost_metrics)
+        raise ValueError(f"cost_metric must be one of: {allowed_list}")
 
     # Get raw congestion dual values, add them
     ref_cong_abs = ref_scenario.state.get_congu() + ref_scenario.state.get_congl()
     all_branches = set(ref_cong_abs.columns.tolist())
-    # Create validated composite allow list
+    # Create validated composite allow list, and filter shadow price data frame
     composite_allow_list = _construct_composite_allow_list(
         all_branches, allow_list, deny_list
     )
-
-    # Parse 2-D array to vector of quantile values
     ref_cong_abs = ref_cong_abs.filter(items=composite_allow_list)
-    quantile_cong_abs = ref_cong_abs.quantile(quantile)
-    # Filter out insignificant values
-    significance_bitmask = quantile_cong_abs > cong_significance_cutoff
-    quantile_cong_abs = quantile_cong_abs.where(significance_bitmask).dropna()
+
+    if congestion_metric == "mean":
+        congestion_metric_values = ref_cong_abs.mean()
+    if congestion_metric == "quantile":
+        congestion_metric_values = ref_cong_abs.quantile(quantile)
+
+    # Filter out 'insignificant' values
+    congestion_metric_values = congestion_metric_values.where(
+        congestion_metric_values > cong_significance_cutoff
+    ).dropna()
     # Filter based on composite allow list
-    congested_indices = list(quantile_cong_abs.index)
+    congested_indices = list(congestion_metric_values.index)
 
     # Ensure that we have enough congested branches to upgrade
-    num_congested = len(quantile_cong_abs)
+    num_congested = len(congested_indices)
     if num_congested < upgrade_n:
         err_msg = "not enough congested branches: "
         err_msg += f"{upgrade_n} desired, but only {num_congested} congested."
+        if congestion_metric == "quantile":
+            err_msg += (
+                f" The quantile used is {quantile}; increasing this value will increase"
+                " the number of branches which qualify as having 'frequent-enough'"
+                " congestion and can be selected for upgrades."
+            )
         raise ValueError(err_msg)
 
-    # Calculate selected metric for congested branches
-    if method == "cost":
+    # Calculate selected cost metric for congested branches
+    if cost_metric == "cost":
         # Calculate costs for an upgrade dataframe containing only composite_allow_list
         base_grid = Grid(
             ref_scenario.info["interconnect"], ref_scenario.info["grid_model"]
@@ -312,7 +342,7 @@ def _identify_mesh_branch_upgrades(
         upgrade_costs = _calculate_ac_inv_costs(base_grid, sum_results=False)
         # Merge the individual line/transformer data into a single Series
         merged_upgrade_costs = pd.concat([v for v in upgrade_costs.values()])
-    if method in ("MW", "MWmiles"):
+    if cost_metric in ("MW", "MWmiles"):
         ref_grid = ref_scenario.state.get_grid()
         branch_ratings = ref_grid.branch.loc[congested_indices, "rateA"]
         # Calculate 'original' branch capacities, since that's our increment
@@ -325,20 +355,21 @@ def _identify_mesh_branch_upgrades(
             {i: (branch_ct[i] if i in branch_ct else 1) for i in congested_indices}
         )
         branch_ratings = branch_ratings / branch_prev_scaling
-    if method == "MW":
-        branch_metric = quantile_cong_abs / branch_ratings
-    elif method == "MWmiles":
+    # Then, apply this metric
+    if cost_metric == "MW":
+        branch_metric = congestion_metric_values / branch_ratings
+    elif cost_metric == "MWmiles":
         branch_lengths = ref_grid.branch.loc[congested_indices].apply(
             lambda x: haversine((x.from_lat, x.from_lon), (x.to_lat, x.to_lon)), axis=1
         )
         # Replace zero-length branches by designated default, don't divide by 0
         branch_lengths = branch_lengths.replace(0, value=zero_length_value)
-        branch_metric = quantile_cong_abs / (branch_ratings * branch_lengths)
-    elif method == "cost":
-        branch_metric = quantile_cong_abs / merged_upgrade_costs
+        branch_metric = congestion_metric_values / (branch_ratings * branch_lengths)
+    elif cost_metric == "cost":
+        branch_metric = congestion_metric_values / merged_upgrade_costs
     else:
         # By process of elimination, all that's left is method 'branches'
-        branch_metric = quantile_cong_abs
+        branch_metric = congestion_metric_values
 
     # Sort by our metric, grab indexes for N largest values (tail), return
     ranked_branches = set(branch_metric.sort_values().tail(upgrade_n).index)

From dbe52785ba3daa963bb9b73483690fce9f9b8235 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Mon, 3 May 2021 12:17:24 -0700
Subject: [PATCH 06/60] test: update tests for new call signature

---
 .../design/transmission/tests/test_upgrade.py  | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/powersimdata/design/transmission/tests/test_upgrade.py b/powersimdata/design/transmission/tests/test_upgrade.py
index 882325425..ea1a39e39 100644
--- a/powersimdata/design/transmission/tests/test_upgrade.py
+++ b/powersimdata/design/transmission/tests/test_upgrade.py
@@ -270,14 +270,14 @@ def test_identify_mesh_branch_upgrades_quantile90(self):
     def test_identify_mesh_MW_n_3(self):  # noqa: N802
         expected_return = {101, 102, 103}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=3, method="MW"
+            self.mock_scenario, upgrade_n=3, cost_metric="MW"
         )
         self.assertEqual(branches, expected_return)
 
     def test_identify_mesh_MW_n_2(self):  # noqa: N802
         expected_return = {101, 102}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=2, method="MW"
+            self.mock_scenario, upgrade_n=2, cost_metric="MW"
         )
         self.assertEqual(branches, expected_return)
 
@@ -285,7 +285,7 @@ def test_identify_mesh_MW_n_2_allow_list(self):  # noqa: N802
         expected_return = {102, 103}
         allow_list = {102, 103, 104}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=2, method="MW", allow_list=allow_list
+            self.mock_scenario, upgrade_n=2, cost_metric="MW", allow_list=allow_list
         )
         self.assertEqual(branches, expected_return)
 
@@ -293,14 +293,14 @@ def test_identify_mesh_MW_n_2_deny_list(self):  # noqa: N802
         expected_return = {101, 103}
         deny_list = [102, 105]
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=2, method="MW", deny_list=deny_list
+            self.mock_scenario, upgrade_n=2, cost_metric="MW", deny_list=deny_list
         )
         self.assertEqual(branches, expected_return)
 
     def test_identify_mesh_MW_n_1(self):  # noqa: N802
         expected_return = {102}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=1, method="MW"
+            self.mock_scenario, upgrade_n=1, cost_metric="MW"
         )
         self.assertEqual(branches, expected_return)
 
@@ -309,21 +309,21 @@ def test_identify_mesh_MW_n_1(self):  # noqa: N802
     def test_identify_mesh_MWmiles_n_3(self):  # noqa: N802
         expected_return = {101, 102, 103}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=3, method="MWmiles"
+            self.mock_scenario, upgrade_n=3, cost_metric="MWmiles"
         )
         self.assertEqual(branches, expected_return)
 
     def test_identify_mesh_MWmiles_n_2(self):  # noqa: N802
         expected_return = {101, 102}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=2, method="MWmiles"
+            self.mock_scenario, upgrade_n=2, cost_metric="MWmiles"
         )
         self.assertEqual(branches, expected_return)
 
     def test_identify_mesh_MWmiles_n_1(self):  # noqa: N802
         expected_return = {101}
         branches = _identify_mesh_branch_upgrades(
-            self.mock_scenario, upgrade_n=1, method="MWmiles"
+            self.mock_scenario, upgrade_n=1, cost_metric="MWmiles"
         )
         self.assertEqual(branches, expected_return)
 
@@ -331,7 +331,7 @@ def test_identify_mesh_MWmiles_n_1(self):  # noqa: N802
     def test_identify_mesh_bad_method(self):
         with self.assertRaises(ValueError):
             _identify_mesh_branch_upgrades(
-                self.mock_scenario, upgrade_n=2, method="does not exist"
+                self.mock_scenario, upgrade_n=2, cost_metric="does not exist"
             )
 
 

From 1684ef84300aa6fd8b20e8562dd2caf4cd52e3bd Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Mon, 3 May 2021 12:28:16 -0700
Subject: [PATCH 07/60] test: add test for 'mean' congestion metric

---
 .../design/transmission/tests/test_upgrade.py | 43 ++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/powersimdata/design/transmission/tests/test_upgrade.py b/powersimdata/design/transmission/tests/test_upgrade.py
index ea1a39e39..1b587e955 100644
--- a/powersimdata/design/transmission/tests/test_upgrade.py
+++ b/powersimdata/design/transmission/tests/test_upgrade.py
@@ -213,7 +213,7 @@ def setUp(self):
         columns = mock_branch["branch_id"]
         congu = pd.DataFrame(congu_data, index=range(num_hours), columns=columns)
         congl = pd.DataFrame(congl_data, index=range(num_hours), columns=columns)
-        # Populate with dummy data
+        # Populate with dummy data, added in different hours for thorough testing
         # Branch 101 will have frequent, low congestion
         congu[101].iloc[-15:] = 1
         # Branch 102 will have less frequent, but greater congestion
@@ -221,6 +221,8 @@ def setUp(self):
         # Branch 103 will have only occassional congestion, but very high
         congu[103].iloc[10:13] = 20
         congl[103].iloc[20:23] = 30
+        # Branch 105 will have extremely high congestion in only one hour
+        congl[105].iloc[49] = 9000
         # Build dummy change table
         ct = {"branch": {"branch_id": {b: 1 for b in branch_indices}}}
 
@@ -327,6 +329,45 @@ def test_identify_mesh_MWmiles_n_1(self):  # noqa: N802
         )
         self.assertEqual(branches, expected_return)
 
+    def test_identify_mesh_mean(self):
+        # Not enough branches
+        with self.assertRaises(ValueError):
+            _identify_mesh_branch_upgrades(self.mock_scenario, congestion_metric="mean")
+
+    def test_identify_mesh_mean_n_4_specify_quantile(self):
+        with self.assertRaises(ValueError):
+            _identify_mesh_branch_upgrades(
+                self.mock_scenario, congestion_metric="mean", upgrade_n=4, quantile=0.99
+            )
+
+    def test_identify_mesh_mean_n_4(self):
+        expected_return = {101, 102, 103, 105}
+        branches = _identify_mesh_branch_upgrades(
+            self.mock_scenario, congestion_metric="mean", upgrade_n=4
+        )
+        self.assertEqual(branches, expected_return)
+
+    def test_identify_mesh_mean_n_3(self):
+        expected_return = {102, 103, 105}
+        branches = _identify_mesh_branch_upgrades(
+            self.mock_scenario, congestion_metric="mean", upgrade_n=3
+        )
+        self.assertEqual(branches, expected_return)
+
+    def test_identify_mesh_mean_n_2(self):
+        expected_return = {103, 105}
+        branches = _identify_mesh_branch_upgrades(
+            self.mock_scenario, congestion_metric="mean", upgrade_n=2
+        )
+        self.assertEqual(branches, expected_return)
+
+    def test_identify_mesh_mean_n_1(self):
+        expected_return = {105}
+        branches = _identify_mesh_branch_upgrades(
+            self.mock_scenario, congestion_metric="mean", upgrade_n=1
+        )
+        self.assertEqual(branches, expected_return)
+
     # What about a made-up method?
     def test_identify_mesh_bad_method(self):
         with self.assertRaises(ValueError):

From cae25519182cbf24337570fac324278ea162bb92 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 30 Apr 2021 17:16:54 -0700
Subject: [PATCH 08/60] docs: format README

---
 powersimdata/design/investment/README.md | 88 +++++++++++++++---------
 1 file changed, 54 insertions(+), 34 deletions(-)

diff --git a/powersimdata/design/investment/README.md b/powersimdata/design/investment/README.md
index 0a1c2b28c..ec6be228a 100644
--- a/powersimdata/design/investment/README.md
+++ b/powersimdata/design/investment/README.md
@@ -1,13 +1,17 @@
-This adds cost calculations for all new (added since base_grid) transmission, generation, and storage capacity.
+This adds cost calculations for all new (added since base_grid) transmission,
+generation, and storage capacity.
 
-`create_mapping_files.py` contains functions that help create shapefiles and map lat/lon points to regions from shapefiles. 
-`investment_costs.py` uses these mappings to find the regional cost multipliers (for both generation/storage and ac transmission).
+***create_mapping_files.py*** contains functions that help create shapefiles and map
+lat/lon points to regions from shapefiles.
+***investment_costs.py*** uses these mappings to find the regional cost multipliers
+(for both generation/storage and ac transmission).
 
 ### Fundamental calculations
-There are 3 separate cost calculations: one for dclines, one for branches (AC transmission), one for plants.
+There are 3 separate cost calculations: one for HVDC lines, one for branches (AC
+transmission) and one for plants.
 
-#### branches: calculate_ac_inv_costs
 
+#### branches: calculate_ac_inv_costs
 For all capacity added on a **line**, the investment cost is:
 `Cost ($today) = rateA (MW) * lengthMi (miles) * costMWmi ($2010/MW-mi) * mult (regional cost multiplier) * inflation($2010 to today)`
 Then all lines are summed.
@@ -15,30 +19,33 @@ Then all lines are summed.
 For all capacity added on a **transformer**, the investment cost is:
 `Cost ($today) = rateA (MW) * perMWcost ($2020/MW) * mult (regional cost multiplier) * inflation($2020 to today)`
 
-#### dclines: calculate_dc_inv_costs
 
-For all capacity added on a dcline, the investment cost is:
+#### dclines: calculate_dc_inv_costs
+For all capacity added on a HVDC line, the investment cost is:
 `Cost ($today) = Pmax (MW) * (lengthMi (miles) * costMWmi ($2015/MW-mi) * inflation(2015 to today) + 2 * terminal_cost_per_MW ($2020) * inflation($2020 to today))`
 Then all line costs are summed.
 
-#### plant: calculate_gen_inv_costs
 
+#### plant: calculate_gen_inv_costs
 For all capacity added on a plant, the investment cost is:
 `Cost ($today) = Pmax (MW) * CAPEX ($2018/MW) * reg_cap_cost_mult (regional capital cost multiplier) * inflation($2018 to today)`
 Then all costs are summed by technology (so you can ignore the non-renewables/storage if you want).
 
 
 ### Methods
-
 #### branches: calculate_ac_inv_costs
-
-- Find new upgrades for each line/transformer (change in MW rating): `grid.branch.rateA - base_grid.branch.rateA`
+- Find new upgrades for each line/transformer (change in MW rating):
+  `grid.branch.rateA - base_grid.branch.rateA`
 - Drop unchanged branches.
 - Map branches to corresponding kV.
 - Separate lines and transformers (TransformerWindings dropped).
-- Lines: Find closest kV present in the corresponding cost table using `select_kv` . Labeled "kV_cost." 
-- Lines: Find MW closest to the line's rateA that exists within the corresponding kV_cost sub-table. Then use that costMWmi base cost value.
-- Lines: Import bus to NEEM mapping file. Check that no buses you need are missing. Check that buses have the same lat/lon values as the pre-made mapping file. If any issues, re-map NEEM regions to those points.
+- Lines: Find closest kV present in the corresponding cost table using `select_kv`.
+  Labeled "kV_cost."
+- Lines: Find MW closest to the line's rateA that exists within the corresponding
+  kV_cost sub-table. Then use that costMWmi base cost value.
+- Lines: Import bus to NEEM mapping file. Check that no buses you need are missing.
+  Check that buses have the same lat/lon values as the pre-made mapping file. If any
+  issues, re-map NEEM regions to those points.
 - Lines: Map regional multipliers onto lines' to_bus and from_bus by NEEM region.
 - Lines: Regional multiplier is then the average of the 2 buses' regional multipliers.
 - Lines: Inflation is applied to scale from 2010 dollars to present.
@@ -48,9 +55,10 @@ Then all costs are summed by technology (so you can ignore the non-renewables/st
 - Transformers: Regional multiplier is then applied.
 - Transformers: Inflation is applied to scale from 2020 dollars to present.
 
-#### dcline: calculate_dc_inv_costs
 
-- Find new capacity for each dcline (change in capacity): `grid.dcline.Pmax - base_grid.dcline.Pmax`
+#### HVDC line: calculate_dc_inv_costs
+- Find new capacity for each dcline (change in capacity):
+  `grid.dcline.Pmax - base_grid.dcline.Pmax`
 - Drop dcline if no changes.
 - Map using buses to get the from/to lat/lon's of line.
 - Find line length. Find MWmi.
@@ -58,47 +66,59 @@ Then all costs are summed by technology (so you can ignore the non-renewables/st
 - Add per-MW terminal costs for each side of the new line.
 - Apply inflation to scale to present dollars.
 
-#### plant: calculate_gen_inv_costs
 
-- Find new capacity for each plant (change in generation capacity): `grid.plant.Pmax - base_grid.plant.Pmax`
+#### plant: calculate_gen_inv_costs
+- Find new capacity for each plant (change in generation capacity):
+  `grid.plant.Pmax - base_grid.plant.Pmax`
 - Drop plants < 0.1 MW.
 - Drop "dfo" and "other" because no ATB cost data available.
-- Load in base CAPEX costs from ATB data and select cost year/cost_case out of "Conservative", "Moderate", and "Advanced."
-- Select (arbitrary) TechDetail (sub-technology) because we can't differentiate between these yet in our plants.
+- Load in base CAPEX costs from ATB data and select cost year/cost_case out of
+  "Conservative", "Moderate", and "Advanced."
+- Select (arbitrary) TechDetail (sub-technology) because we can't differentiate between
+  these yet in our plants.
 - Map base costs to plants by "Technology".
 - Map plant location to ReEDS region (for regional cost multipliers).
-- If a technology is wind or wind_offshore or concentrated solar power, regions are in rs (wind resource region), so keep rsas the region to map. If technology is another tech, regions are in rb (BA region), so keep rb as the region to map.
-- Map ["Technology", "r" (region)] to ReEDS regional capital cost multipliers. Keep (arbitrary) subclasses for renewables.
+- If a technology is wind or wind_offshore or concentrated solar power, regions are in
+  rs (wind resource region), so keep rsas the region to map. If technology is
+  another tech, regions are in rb (BA region), so keep rb as the region to map.
+- Map ["Technology", "r" (region)] to ReEDS regional capital cost multipliers. Keep
+  (arbitrary) subclasses for renewables.
 - Apply inflation to scale 2018 dollars to present.
 - Final calculations.
 
 
 ### Mapping functions
+`sjoin_nearest`: joins a geo data frame of Points and Polygons/Multipolygons. Used in
+`points_to_polys`.
 
-`sjoin_nearest`: joins a geodataframe of Points and Polygons/Multipolygons. Used in `points_to_polys`.
+`points_to_polys`: joins a dataframe (which includes lat and lon columns) with a
+shapefile. Used in `bus_to_neem_reg` and `plant_to_reeds_reg`.
 
-`points_to_polys`: joins a dataframe (which includes lat and lon columns) with a shapefile. Used in `bus_to_neem_reg` and `plant_to_reeds_reg`.
 
 #### Functions used for AC regional multiplier mapping
+`bus_to_neem_reg`: maps bus locations to NEEM regions. Used in `write_bus_neem_map` and
+(if there are errors in the mapping file produced in `write_bus_neem_map`), this
+function is also used in `_calculate_ac_inv_costs`.
 
-`bus_to_neem_reg`: maps bus locations to NEEM regions. Used in `write_bus_neem_map` and (if there are errors in the mapping file produced in `write_bus_neem_map`), this function is also used in `_calculate_ac_inv_costs`.
-
-`write_bus_neem_map`: maps all base_grid bus locations to NEEM regions and produces a csv mapping file: regionsNEEM.shp. This csv is used in `_calculate_ac_inv_costs`.
+`write_bus_neem_map`: maps all base_grid bus locations to NEEM regions and produces a
+csv mapping file: regionsNEEM.shp. This csv is used in `_calculate_ac_inv_costs`.
 
 
 #### Functions used for generation/storage regional multiplier mapping
+`write_poly_shapefile`: using a csv with specified geometry, creates the shapefile for
+ReEDS wind resource regions: rs.shp. This shp is used in `plant_to_reeds_reg`.
 
-`write_poly_shapefile`: using a csv with specified geometry, creates the shapefile for ReEDS wind resource regions: rs.shp. This shp is used in `plant_to_reeds_reg`.
-
-`plant_to_reeds_reg`: maps plant locations to ReEDS regions. Used in `_calculate_gen_inv_costs`.
+`plant_to_reeds_reg`: maps plant locations to ReEDS regions. Used in
+`_calculate_gen_inv_costs`.
 
 
 ### Sources
-
 See [ATTRIBUTION.md](../../../ATTRIBUTION.md).
 
-#### Potential improvements:
 
-- If we want to have financial information other than the default ATB values, a separate financials module will be useful for CAPEX/other calculations.
+#### Potential improvements:
+- If we want to have financial information other than the default ATB values, a
+separate financials module will be useful for CAPEX/other calculations.
 
-- Find correct wind and solar classes (based on wind speed, irradiance) to map to ATB costs and ReEDS regional cost multipliers.
+- Find correct wind and solar classes (based on wind speed, irradiance) to map to ATB
+costs and ReEDS regional cost multipliers.

From 0dbc5d4e7b2e2a70ecf67ac1e4e0c36bcca0282a Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 30 Apr 2021 17:17:38 -0700
Subject: [PATCH 09/60] refactor: instantiate base grid using grid model

---
 powersimdata/design/investment/investment_costs.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/powersimdata/design/investment/investment_costs.py b/powersimdata/design/investment/investment_costs.py
index d673e7476..e9a793ff5 100644
--- a/powersimdata/design/investment/investment_costs.py
+++ b/powersimdata/design/investment/investment_costs.py
@@ -52,7 +52,9 @@ def calculate_ac_inv_costs(scenario, sum_results=True, exclude_branches=None):
         Whether summed or not, values are $USD, inflation-adjusted to today.
     """
 
-    base_grid = Grid(scenario.info["interconnect"].split("_"))
+    base_grid = Grid(
+        scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
+    )
     grid = scenario.state.get_grid()
 
     # find upgraded AC lines
@@ -265,7 +267,9 @@ def calculate_dc_inv_costs(scenario, sum_results=True):
         inflation-adjusted to today. If ``sum_results``, a float is returned, otherwise
         a Series.
     """
-    base_grid = Grid(scenario.info["interconnect"].split("_"))
+    base_grid = Grid(
+        scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
+    )
     grid = scenario.state.get_grid()
 
     grid_new = cp.deepcopy(grid)
@@ -346,7 +350,9 @@ def calculate_gen_inv_costs(scenario, year, cost_case, sum_results=True):
         curves.
     """
 
-    base_grid = Grid(scenario.info["interconnect"].split("_"))
+    base_grid = Grid(
+        scenario.info["interconnect"].split("_"), source=scenario.info["grid_model"]
+    )
     grid = scenario.state.get_grid()
 
     # Find change in generation capacity

From 6ec54f98416db39d45d30426720c4f4099f61a48 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 30 Apr 2021 17:22:47 -0700
Subject: [PATCH 10/60] refactor: pass grid object to functions

---
 .../design/investment/create_mapping_files.py | 22 ++++++++++++++-----
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/powersimdata/design/investment/create_mapping_files.py b/powersimdata/design/investment/create_mapping_files.py
index 9c0857ea4..7335e0e87 100644
--- a/powersimdata/design/investment/create_mapping_files.py
+++ b/powersimdata/design/investment/create_mapping_files.py
@@ -174,18 +174,28 @@ def bus_to_neem_reg(df):
     return pts_poly
 
 
-def write_bus_neem_map():
-    """Write bus location to NEEM region mapping to file"""
-    base_grid = Grid(["USA"])
+def write_bus_neem_map(base_grid):
+    """Write bus location to NEEM region mapping to file.
+
+    :param powersimdata.input.grid.Grid base_grid: a Grid instance.
+    :raises TypeError: if ``base_grid`` is not a Grid instance.
+    """
+    if not isinstance(base_grid, Grid):
+        raise TypeError("base_grid must be a Grid instance")
     df_pts_bus = bus_to_neem_reg(base_grid.bus)
     df_pts_bus.sort_index(inplace=True)
     os.makedirs(const.bus_neem_regions_path, exist_ok=True)
     df_pts_bus.to_csv(const.bus_neem_regions_path)
 
 
-def write_bus_reeds_map():
-    """Write bus location to ReEDS region mapping to file."""
-    base_grid = Grid(["USA"])
+def write_bus_reeds_map(base_grid):
+    """Write bus location to ReEDS region mapping to file.
+
+    :param powersimdata.input.grid.Grid base_grid: a Grid instance.
+    :raises TypeError: if ``base_grid`` is not a Grid instance.
+    """
+    if not isinstance(base_grid, Grid):
+        raise TypeError("base_grid must be a Grid instance")
     df_pts_bus = bus_to_reeds_reg(base_grid.bus)
     df_pts_bus.sort_index(inplace=True)
     os.makedirs(const.bus_reeds_regions_path, exist_ok=True)

From 8cde6aacc45d84f92e5f0a4b59ca57d954214f35 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Wed, 5 May 2021 14:45:43 -0700
Subject: [PATCH 11/60] chore: create feature request issue template (#453)

---
 .github/ISSUE_TEMPLATE/feature_request.md | 36 +++++++++++++++++++++++
 1 file changed, 36 insertions(+)
 create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md

diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md
new file mode 100644
index 000000000..eed038089
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/feature_request.md
@@ -0,0 +1,36 @@
+---
+name: Feature request
+about: Suggest an idea for this project
+title: Feature request
+labels: feature request
+assignees: ahurli, BainanXia, danielolsen, jon-hagg, rouille
+
+---
+
+ # :rocket:
+
+- [ ] Is your feature request essential for your project?
+
+
+### Describe the workflow you want to enable
+A clear and concise description of what can be enhanced, e.g., "I wish I could do [...]"
+
+### Describe your proposed implementation
+This should provide a description of the feature request, e.g.:
+* "The class `Foo` should have a new method `bar` that allows to [...]"
+* "Function `foo` needs a new arguments `bar` to set [...]"
+* "Create a new function `foo` to calculate [...]"
+
+If applicable, try to write a docstring for the desired feature. To illustrate, if you would like to add a new function in a module, provide:
+* the name of the function
+* a description of the task accomplished by the function
+* a list of the input and output parameters together with their types (e.g., `int`,
+  `str`, `pandas.DataFrame`, etc.) and a short description of its/their meaning
+
+### Describe alternatives you've considered, if relevant
+This should provide a description of any alternative solutions or features you've
+considered.
+
+### Additional context
+Add any other context or screenshots in this section, e.g., a plot from an article you
+believe would clearly communicate results.

From 9d2ae2122b90ebc19557bbda71405a381f8bf9cc Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 23 Apr 2021 15:08:29 -0700
Subject: [PATCH 12/60] feat: os agnostic copy functionality

---
 powersimdata/data_access/data_access.py       | 104 ++++++++++--------
 .../data_access/tests/test_data_access.py     |   2 +-
 powersimdata/scenario/execute.py              |  22 ++--
 3 files changed, 69 insertions(+), 59 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index 34b550e62..bbcff9075 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -4,7 +4,7 @@
 import posixpath
 import shutil
 import time
-from subprocess import PIPE, Popen
+from subprocess import Popen
 from tempfile import mkstemp
 
 import paramiko
@@ -43,9 +43,7 @@ def copy(self, src, dest, recursive=False, update=False):
         :param bool recursive: create directories recursively
         :param bool update: only copy if needed
         """
-        self.makedir(posixpath.dirname(dest))
-        command = CommandBuilder.copy(src, dest, recursive, update)
-        return self.execute_command(command)
+        raise NotImplementedError
 
     def remove(self, target, recursive=False, confirm=True):
         """Wrapper around rm command
@@ -86,10 +84,10 @@ def _check_filename(self, filename):
         if len(os.path.dirname(filename)) != 0:
             raise ValueError(f"Expecting file name but got path {filename}")
 
-    def makedir(self, relative_path):
-        """Create paths relative to the instance root
+    def makedir(self, full_path):
+        """Create path, including parents
 
-        :param str relative_path: the path, without filename, relative to root
+        :param str full_path: the path, excluding filename
         """
         raise NotImplementedError
 
@@ -180,21 +178,41 @@ def move_to(self, file_name, to_dir, change_name_to=None):
         :param str change_name_to: new name for file when copied to data store.
         """
         self._check_filename(file_name)
-        src = posixpath.join(server_setup.LOCAL_DIR, file_name)
+        src = os.path.join(server_setup.LOCAL_DIR, file_name)
         file_name = file_name if change_name_to is None else change_name_to
-        dest = posixpath.join(self.root, to_dir, file_name)
+        dest = os.path.join(self.root, to_dir, file_name)
         print(f"--> Moving file {src} to {dest}")
         self._check_file_exists(dest, should_exist=False)
-        self.copy(src, dest)
-        self.remove(src)
+        self.makedir(os.path.dirname(dest))
+        shutil.move(src, dest)
 
-    def makedir(self, relative_path):
-        """Create paths relative to the instance root
+    def makedir(self, full_path):
+        """Create path on local machine
 
-        :param str relative_path: the path, without filename, relative to root
+        :param str full_path: the path, excluding filename
         """
-        target = os.path.join(self.root, relative_path)
-        os.makedirs(target, exist_ok=True)
+        os.makedirs(full_path, exist_ok=True)
+
+    @staticmethod
+    def _fapply(func, pattern):
+        files = [f for f in glob.glob(pattern) if os.path.isfile(f)]
+        for f in files:
+            func(f)
+
+    def copy(self, src, dest, recursive=False, update=False):
+        """Wrapper around cp command which creates dest path if needed
+
+        :param str src: path to original
+        :param str dest: destination path
+        :param bool recursive: create directories recursively
+        :param bool update: ignored
+        """
+        self.makedir(dest)
+        if recursive:
+            shutil.copytree(src, dest)
+        else:
+            func = lambda s: shutil.copy(s, dest)  # noqa: E731
+            LocalDataAccess._fapply(func, src)
 
     def remove(self, target, recursive=False, confirm=True):
         """Remove target using rm semantics
@@ -211,32 +229,9 @@ def remove(self, target, recursive=False, confirm=True):
         if recursive:
             shutil.rmtree(target)
         else:
-            files = [f for f in glob.glob(target) if os.path.isfile(f)]
-            for f in files:
-                os.remove(f)
+            LocalDataAccess._fapply(os.remove, target)
         print("--> Done!")
 
-    def execute_command(self, command):
-        """Execute a command locally at the data access.
-
-        :param list command: list of str to be passed to command line.
-        """
-
-        def wrap(s):
-            if s is not None:
-                return s
-            return open(os.devnull)
-
-        proc = Popen(
-            command,
-            shell=True,
-            executable="/bin/bash",
-            stdout=PIPE,
-            stderr=PIPE,
-            text=True,
-        )
-        return wrap(None), wrap(proc.stdout), wrap(proc.stderr)
-
     def get_profile_version(self, grid_model, kind):
         """Returns available raw profile from blob storage or local disk
 
@@ -360,8 +355,8 @@ def move_to(self, file_name, to_dir=None, change_name_to=None):
             )
 
         file_name = file_name if change_name_to is None else change_name_to
-        to_dir = "" if to_dir is None else to_dir
-        to_path = posixpath.join(self.root, to_dir, file_name)
+        to_dir = posixpath.join(self.root, "" if to_dir is None else to_dir)
+        to_path = posixpath.join(to_dir, file_name)
         self.makedir(to_dir)
         self._check_file_exists(to_path, should_exist=False)
 
@@ -440,16 +435,31 @@ def push(self, file_name, checksum, change_name_to=None):
                 print(e)
             raise IOError("Failed to push file - most likely a conflict was detected.")
 
-    def makedir(self, relative_path):
-        """Create paths relative to the instance root
+    def makedir(self, full_path):
+        """Create path on server
 
-        :param str relative_path: the path, without filename, relative to root
+        :param str full_path: the path, excluding filename
         :raises IOError: if command generated stderr
         """
-        full_path = posixpath.join(self.root, relative_path)
         _, _, stderr = self.execute_command(f"mkdir -p {full_path}")
+        errors = stderr.readlines()
+        if len(errors) > 0:
+            raise IOError(f"Failed to create {full_path} on server")
+
+    def copy(self, src, dest, recursive=False, update=False):
+        """Wrapper around cp command which creates dest path if needed
+
+        :param str src: path to original
+        :param str dest: destination path
+        :param bool recursive: create directories recursively
+        :param bool update: only copy if needed
+        :raises IOError: if command generated stderr
+        """
+        self.makedir(dest)
+        command = CommandBuilder.copy(src, dest, recursive, update)
+        _, _, stderr = self.execute_command(command)
         if len(stderr.readlines()) != 0:
-            raise IOError("Failed to create %s on server" % full_path)
+            raise IOError(f"Failed to execute {command}")
 
     def remove(self, target, recursive=False, confirm=True):
         """Run rm command on server
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index b058c364b..de0998138 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -108,7 +108,7 @@ def test_move_to_multi_path(mock_data_access, make_temp):
     remote_path = mock_data_access.root / rel_path
     remote_path.mkdir(parents=True)
     fname = make_temp(remote=False)
-    mock_data_access.move_to(fname, rel_path)
+    mock_data_access.move_to(fname, str(rel_path))
     _check_content(os.path.join(remote_path, fname))
 
 
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index fffc1c381..ba216d555 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -332,21 +332,25 @@ def __init__(self, data_access, scenario_info, grid, ct):
         self.grid = grid
         self.ct = ct
         self.server_config = server_setup.PathConfig(server_setup.DATA_ROOT_DIR)
-        self.scenario_folder = "scenario_%s" % scenario_info["id"]
+        self.scenario_id = scenario_info["id"]
+        self.scenario_folder = "scenario_%s" % self.scenario_id
 
         self.REL_TMP_DIR = posixpath.join(
             server_setup.EXECUTE_DIR, self.scenario_folder
         )
+        self.TMP_DIR = posixpath.join(
+            self.server_config.execute_dir(), self.scenario_folder
+        )
 
     def create_folder(self):
         """Creates folder on server that will enclose simulation inputs."""
         print("--> Creating temporary folder on server for simulation inputs")
-        self._data_access.makedir(self.REL_TMP_DIR)
+        self._data_access.makedir(self.TMP_DIR)
 
     def prepare_mpc_file(self):
         """Creates MATPOWER case file."""
-        file_name = "%s_case.mat" % self._scenario_info["id"]
-        storage_file_name = "%s_case_storage.mat" % self._scenario_info["id"]
+        file_name = f"{self.scenario_id}_case.mat"
+        storage_file_name = f"{self.scenario_id}_case_storage.mat"
         file_path = os.path.join(server_setup.LOCAL_DIR, file_name)
         storage_file_path = os.path.join(server_setup.LOCAL_DIR, storage_file_name)
         print("Building MPC file")
@@ -371,7 +375,7 @@ def prepare_profile(self, kind, profile_as=None):
             print(
                 f"Writing scaled {kind} profile in {server_setup.LOCAL_DIR} on local machine"
             )
-            file_name = "%s_%s.csv" % (self._scenario_info["id"], kind)
+            file_name = "%s_%s.csv" % (self.scenario_id, kind)
             profile.to_csv(os.path.join(server_setup.LOCAL_DIR, file_name))
 
             self._data_access.move_to(
@@ -382,9 +386,5 @@ def prepare_profile(self, kind, profile_as=None):
                 self.server_config.execute_dir(),
                 f"scenario_{profile_as}",
             )
-            to_dir = posixpath.join(
-                self.server_config.execute_dir(), self.scenario_folder
-            )
-            _, _, stderr = self._data_access.copy(f"{from_dir}/{kind}.csv", to_dir)
-            if len(stderr.readlines()) != 0:
-                raise IOError(f"Failed to copy {kind}.csv on server")
+            to_dir = self.TMP_DIR
+            self._data_access.copy(f"{from_dir}/{kind}.csv", to_dir)

From 900e183661af0f5b90e626e4cde4848b6ae9c800 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 28 Apr 2021 15:36:20 -0700
Subject: [PATCH 13/60] chore: execute_command -> _execute_command

---
 powersimdata/data_access/data_access.py          | 16 ++++++++--------
 .../data_access/tests/test_data_access.py        |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index bbcff9075..b4cb73495 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -91,7 +91,7 @@ def makedir(self, full_path):
         """
         raise NotImplementedError
 
-    def execute_command(self, command):
+    def _execute_command(self, command):
         """Execute a command locally at the data access.
 
         :param list command: list of str to be passed to command line.
@@ -366,7 +366,7 @@ def move_to(self, file_name, to_dir=None, change_name_to=None):
 
         os.remove(from_path)
 
-    def execute_command(self, command):
+    def _execute_command(self, command):
         """Execute a command locally at the data access.
 
         :param list command: list of str to be passed to command line.
@@ -396,7 +396,7 @@ def checksum(self, relative_path):
         self._check_file_exists(full_path)
 
         command = f"sha1sum {full_path}"
-        _, stdout, _ = self.execute_command(command)
+        _, stdout, _ = self._execute_command(command)
         lines = stdout.readlines()
         return lines[0].strip()
 
@@ -427,7 +427,7 @@ def push(self, file_name, checksum, change_name_to=None):
                 200>{lockfile}"
 
         command = template.format(**values)
-        _, _, stderr = self.execute_command(command)
+        _, _, stderr = self._execute_command(command)
 
         errors = stderr.readlines()
         if len(errors) > 0:
@@ -441,7 +441,7 @@ def makedir(self, full_path):
         :param str full_path: the path, excluding filename
         :raises IOError: if command generated stderr
         """
-        _, _, stderr = self.execute_command(f"mkdir -p {full_path}")
+        _, _, stderr = self._execute_command(f"mkdir -p {full_path}")
         errors = stderr.readlines()
         if len(errors) > 0:
             raise IOError(f"Failed to create {full_path} on server")
@@ -457,7 +457,7 @@ def copy(self, src, dest, recursive=False, update=False):
         """
         self.makedir(dest)
         command = CommandBuilder.copy(src, dest, recursive, update)
-        _, _, stderr = self.execute_command(command)
+        _, _, stderr = self._execute_command(command)
         if len(stderr.readlines()) != 0:
             raise IOError(f"Failed to execute {command}")
 
@@ -475,7 +475,7 @@ def remove(self, target, recursive=False, confirm=True):
             if confirmed.lower() != "y":
                 print("Operation cancelled.")
                 return
-        _, _, stderr = self.execute_command(command)
+        _, _, stderr = self._execute_command(command)
         if len(stderr.readlines()) != 0:
             raise IOError(f"Failed to delete target={target} on server")
         print("--> Done!")
@@ -486,7 +486,7 @@ def _exists(self, filepath):
         :param str filepath: the path to the file
         :return: (*bool*) -- whether the file exists
         """
-        _, _, stderr = self.execute_command(f"ls {filepath}")
+        _, _, stderr = self._execute_command(f"ls {filepath}")
         return len(stderr.readlines()) == 0
 
     def close(self):
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index de0998138..2bc0aa4c4 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -70,7 +70,7 @@ def _check_content(filepath):
 @pytest.mark.integration
 @pytest.mark.ssh
 def test_setup_server_connection(data_access):
-    _, stdout, _ = data_access.execute_command("whoami")
+    _, stdout, _ = data_access._execute_command("whoami")
     assert stdout.read().decode("utf-8").strip() == get_server_user()
 
 

From c39d60b9acb7e7b62615e87842201306340853f1 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 28 Apr 2021 17:32:43 -0700
Subject: [PATCH 14/60] refactor: move path joins to data access subclasses

---
 powersimdata/data_access/data_access.py | 28 +++++++++++++++++++++++--
 powersimdata/scenario/delete.py         | 13 ++++--------
 powersimdata/scenario/execute.py        | 15 ++++---------
 powersimdata/scenario/move.py           | 26 ++++++-----------------
 powersimdata/scenario/state.py          |  2 --
 powersimdata/utility/server_setup.py    | 18 ----------------
 6 files changed, 40 insertions(+), 62 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index b4cb73495..45deb05ec 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -18,6 +18,12 @@
 class DataAccess:
     """Interface to a local or remote data store."""
 
+    def __init__(self, root=None):
+        """Constructor"""
+        self.root = server_setup.DATA_ROOT_DIR if root is None else root
+        self.backup_root = server_setup.BACKUP_DATA_ROOT_DIR
+        self.join = None
+
     def copy_from(self, file_name, from_dir):
         """Copy a file from data store to userspace.
 
@@ -35,6 +41,22 @@ def move_to(self, file_name, to_dir, change_name_to=None):
         """
         raise NotImplementedError
 
+    def get_base_dir(self, kind, backup=False):
+        _allowed = ("input", "output", "tmp")
+        if kind not in _allowed:
+            raise ValueError(f"Invalid 'kind', must be one of {_allowed}")
+
+        root = self.root if not backup else self.backup_root
+        if kind == "tmp":
+            return self.join(root, "tmp")
+        return self.join(root, "data", kind)
+
+    def match_scenario_files(self, scenario_id, kind, backup=False):
+        base_dir = self.get_base_dir(kind, backup)
+        if kind == "tmp":
+            return self.join(base_dir, f"scenario_{scenario_id}")
+        return self.join(base_dir, f"{scenario_id}_*")
+
     def copy(self, src, dest, recursive=False, update=False):
         """Wrapper around cp command which creates dest path if needed
 
@@ -141,8 +163,9 @@ class LocalDataAccess(DataAccess):
     """Interface to shared data volume"""
 
     def __init__(self, root=None):
-        self.root = root if root else server_setup.DATA_ROOT_DIR
+        super().__init__(root)
         self.description = "local machine"
+        self.join = os.path.join
 
     def copy_from(self, file_name, from_dir=None):
         """Copy a file from data store to userspace.
@@ -259,11 +282,12 @@ class SSHDataAccess(DataAccess):
 
     def __init__(self, root=None):
         """Constructor"""
+        super().__init__(root)
         self._ssh = None
         self._retry_after = 5
-        self.root = server_setup.DATA_ROOT_DIR if root is None else root
         self.local_root = server_setup.LOCAL_DIR
         self.description = "server"
+        self.join = posixpath.join
 
     @property
     def ssh(self):
diff --git a/powersimdata/scenario/delete.py b/powersimdata/scenario/delete.py
index 5be2c455c..f7ffe3098 100644
--- a/powersimdata/scenario/delete.py
+++ b/powersimdata/scenario/delete.py
@@ -1,5 +1,4 @@
 import os
-import posixpath
 
 from powersimdata.data_access.data_access import LocalDataAccess
 from powersimdata.scenario.state import State
@@ -41,25 +40,21 @@ def delete_scenario(self, confirm=True):
         self._scenario_list_manager.delete_entry(scenario_id)
         self._execute_list_manager.delete_entry(scenario_id)
 
-        wildcard = f"{scenario_id}_*"
-
         print("--> Deleting scenario input data on server")
-        target = posixpath.join(self.path_config.input_dir(), wildcard)
+        target = self._data_access.match_scenario_files(scenario_id, "input")
         self._data_access.remove(target, recursive=False, confirm=confirm)
 
         print("--> Deleting scenario output data on server")
-        target = posixpath.join(self.path_config.output_dir(), wildcard)
+        target = self._data_access.match_scenario_files(scenario_id, "output")
         self._data_access.remove(target, recursive=False, confirm=confirm)
 
         # Delete temporary folder enclosing simulation inputs
         print("--> Deleting temporary folder on server")
-        tmp_dir = posixpath.join(
-            self.path_config.execute_dir(), f"scenario_{scenario_id}"
-        )
+        tmp_dir = self._data_access.match_scenario_files(scenario_id, "tmp")
         self._data_access.remove(tmp_dir, recursive=True, confirm=confirm)
 
         print("--> Deleting input and output data on local machine")
-        target = os.path.join(server_setup.LOCAL_DIR, "data", "**", wildcard)
+        target = os.path.join(server_setup.LOCAL_DIR, "data", "**", f"{scenario_id}_*")
         LocalDataAccess().remove(target, recursive=False, confirm=confirm)
 
         # Delete attributes
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index ba216d555..2a69175bd 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -331,16 +331,12 @@ def __init__(self, data_access, scenario_info, grid, ct):
         self._scenario_info = scenario_info
         self.grid = grid
         self.ct = ct
-        self.server_config = server_setup.PathConfig(server_setup.DATA_ROOT_DIR)
         self.scenario_id = scenario_info["id"]
-        self.scenario_folder = "scenario_%s" % self.scenario_id
 
-        self.REL_TMP_DIR = posixpath.join(
-            server_setup.EXECUTE_DIR, self.scenario_folder
-        )
-        self.TMP_DIR = posixpath.join(
-            self.server_config.execute_dir(), self.scenario_folder
+        self.REL_TMP_DIR = self._data_access.join(
+            server_setup.EXECUTE_DIR, f"scenario_{self.scenario_id}"
         )
+        self.TMP_DIR = self._data_access.match_scenario_files(self.scenario_id, "tmp")
 
     def create_folder(self):
         """Creates folder on server that will enclose simulation inputs."""
@@ -382,9 +378,6 @@ def prepare_profile(self, kind, profile_as=None):
                 file_name, self.REL_TMP_DIR, change_name_to=f"{kind}.csv"
             )
         else:
-            from_dir = posixpath.join(
-                self.server_config.execute_dir(),
-                f"scenario_{profile_as}",
-            )
+            from_dir = self._data_access.match_scenario_files(profile_as, "tmp")
             to_dir = self.TMP_DIR
             self._data_access.copy(f"{from_dir}/{kind}.csv", to_dir)
diff --git a/powersimdata/scenario/move.py b/powersimdata/scenario/move.py
index ee5eb840d..d811842ba 100644
--- a/powersimdata/scenario/move.py
+++ b/powersimdata/scenario/move.py
@@ -1,7 +1,4 @@
-import posixpath
-
 from powersimdata.scenario.state import State
-from powersimdata.utility import server_setup
 
 
 class Move(State):
@@ -68,39 +65,28 @@ def __init__(self, data_access, scenario_info):
         """Constructor."""
         self._data_access = data_access
         self._scenario_info = scenario_info
-        self.backup_config = server_setup.PathConfig(server_setup.BACKUP_DATA_ROOT_DIR)
-        self.server_config = server_setup.PathConfig(server_setup.DATA_ROOT_DIR)
         self.scenario_id = self._scenario_info["id"]
-        self.wildcard = f"{self.scenario_id}_*"
 
     def move_input_data(self, confirm=True):
         """Moves input data."""
         print("--> Moving scenario input data to backup disk")
-        source = posixpath.join(
-            self.server_config.input_dir(),
-            self.wildcard,
-        )
-        target = self.backup_config.input_dir()
+        source = self._data_access.match_scenario_files(self.scenario_id, "input")
+        target = self._data_access.get_base_dir("input", backup=True)
         self._data_access.copy(source, target, update=True)
         self._data_access.remove(source, recursive=False, confirm=confirm)
 
     def move_output_data(self, confirm=True):
         """Moves output data"""
         print("--> Moving scenario output data to backup disk")
-        source = posixpath.join(
-            self.server_config.output_dir(),
-            self.wildcard,
-        )
-        target = self.backup_config.output_dir()
+        source = self._data_access.match_scenario_files(self.scenario_id, "output")
+        target = self._data_access.get_base_dir("output", backup=True)
         self._data_access.copy(source, target, update=True)
         self._data_access.remove(source, recursive=False, confirm=confirm)
 
     def move_temporary_folder(self, confirm=True):
         """Moves temporary folder."""
         print("--> Moving temporary folder to backup disk")
-        source = posixpath.join(
-            self.server_config.execute_dir(), "scenario_" + self.scenario_id
-        )
-        target = self.backup_config.execute_dir()
+        source = self._data_access.match_scenario_files(self.scenario_id, "tmp")
+        target = self._data_access.get_base_dir("tmp", backup=True)
         self._data_access.copy(source, target, recursive=True, update=True)
         self._data_access.remove(source, recursive=True, confirm=confirm)
diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py
index cc818f4b4..5bfcff997 100644
--- a/powersimdata/scenario/state.py
+++ b/powersimdata/scenario/state.py
@@ -1,6 +1,5 @@
 from powersimdata.data_access.execute_list import ExecuteListManager
 from powersimdata.data_access.scenario_list import ScenarioListManager
-from powersimdata.utility import server_setup
 
 
 class State(object):
@@ -22,7 +21,6 @@ def __init__(self, scenario):
         self._data_access = scenario.data_access
         self._scenario_list_manager = ScenarioListManager(self._data_access)
         self._execute_list_manager = ExecuteListManager(self._data_access)
-        self.path_config = server_setup.PathConfig(server_setup.DATA_ROOT_DIR)
 
     def switch(self, state):
         """Switches state.
diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py
index 07fdcb759..a65c96cda 100644
--- a/powersimdata/utility/server_setup.py
+++ b/powersimdata/utility/server_setup.py
@@ -1,5 +1,4 @@
 import os
-import posixpath
 from pathlib import Path
 
 SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
@@ -25,23 +24,6 @@ def get_deployment_mode():
     return DeploymentMode.Container
 
 
-class PathConfig:
-    def __init__(self, root=None):
-        self.root = root
-
-    def _join(self, rel_path):
-        return posixpath.join(self.root, rel_path)
-
-    def execute_dir(self):
-        return self._join(EXECUTE_DIR)
-
-    def input_dir(self):
-        return self._join(INPUT_DIR)
-
-    def output_dir(self):
-        return self._join(OUTPUT_DIR)
-
-
 def get_server_user():
     """Returns the first username found using the following sources:
 

From cf0759b5152f4bcf1656b671efd05523b89ec372 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 28 Apr 2021 17:56:56 -0700
Subject: [PATCH 15/60] test: add tests for path creation logic

---
 powersimdata/data_access/data_access.py       | 19 +++++++++--
 .../data_access/tests/test_data_access.py     | 33 +++++++++++++++++--
 2 files changed, 48 insertions(+), 4 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index 45deb05ec..0211ef54e 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -42,6 +42,13 @@ def move_to(self, file_name, to_dir, change_name_to=None):
         raise NotImplementedError
 
     def get_base_dir(self, kind, backup=False):
+        """Get path to given kind relative to instance root
+
+        :param str kind: one of {input, output, tmp}
+        :param bool backup: pass True if relative to backup root dir
+        :raises ValueError: if kind is invalid
+        :return: (*str*) -- the specified path
+        """
         _allowed = ("input", "output", "tmp")
         if kind not in _allowed:
             raise ValueError(f"Invalid 'kind', must be one of {_allowed}")
@@ -52,6 +59,13 @@ def get_base_dir(self, kind, backup=False):
         return self.join(root, "data", kind)
 
     def match_scenario_files(self, scenario_id, kind, backup=False):
+        """Get path matching the given kind of scenario data
+
+        :param int/str scenario_id: the scenario id
+        :param str kind: one of {input, output, tmp}
+        :param bool backup: pass True if relative to backup root dir
+        :return: (*str*) -- the specified path
+        """
         base_dir = self.get_base_dir(kind, backup)
         if kind == "tmp":
             return self.join(base_dir, f"scenario_{scenario_id}")
@@ -514,8 +528,9 @@ def _exists(self, filepath):
         return len(stderr.readlines()) == 0
 
     def close(self):
-        """Close the connection that was opened when the object was created."""
-        self.ssh.close()
+        """Close the connection if one is open"""
+        if self._ssh is not None:
+            self._ssh.close()
 
 
 def progress_bar(*args, **kwargs):
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index 2bc0aa4c4..29e52dd01 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -7,7 +7,7 @@
 
 from powersimdata.data_access.data_access import SSHDataAccess
 from powersimdata.tests.mock_ssh import MockConnection
-from powersimdata.utility.server_setup import get_server_user
+from powersimdata.utility import server_setup
 
 CONTENT = b"content"
 
@@ -67,11 +67,40 @@ def _check_content(filepath):
         assert CONTENT == f.read()
 
 
+root_dir = server_setup.DATA_ROOT_DIR
+backup_root = server_setup.BACKUP_DATA_ROOT_DIR
+
+
+def test_base_dir(data_access):
+    input_dir = data_access.get_base_dir("input")
+    assert f"{root_dir}/data/input" == input_dir
+
+    output_dir = data_access.get_base_dir("output", backup=True)
+    assert f"{backup_root}/data/output" == output_dir
+
+    tmp_dir = data_access.get_base_dir("tmp")
+    assert f"{root_dir}/tmp" == tmp_dir
+
+    with pytest.raises(ValueError):
+        data_access.get_base_dir("foo")
+
+
+def test_match_scenario_files(data_access):
+    output_files = data_access.match_scenario_files(99, "output")
+    assert f"{root_dir}/data/output/99_*" == output_files
+
+    tmp_files = data_access.match_scenario_files(42, "tmp", backup=True)
+    assert f"{backup_root}/tmp/scenario_42" == tmp_files
+
+    with pytest.raises(ValueError):
+        data_access.match_scenario_files(1, "foo")
+
+
 @pytest.mark.integration
 @pytest.mark.ssh
 def test_setup_server_connection(data_access):
     _, stdout, _ = data_access._execute_command("whoami")
-    assert stdout.read().decode("utf-8").strip() == get_server_user()
+    assert stdout.read().decode("utf-8").strip() == server_setup.get_server_user()
 
 
 def test_mocked_correctly(mock_data_access):

From 56650838521cd6569746d2d06997df2b14901657 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 30 Apr 2021 15:52:28 -0700
Subject: [PATCH 16/60] fix: implicit posixpath usage

---
 powersimdata/data_access/profile_helper.py            | 11 +++++------
 powersimdata/data_access/tests/test_profile_helper.py |  4 +---
 powersimdata/input/input_data.py                      | 10 +++++-----
 powersimdata/input/tests/test_input_data.py           |  2 +-
 powersimdata/output/output_data.py                    |  5 +++--
 powersimdata/scenario/create.py                       |  3 ++-
 powersimdata/utility/server_setup.py                  |  4 ++--
 7 files changed, 19 insertions(+), 20 deletions(-)

diff --git a/powersimdata/data_access/profile_helper.py b/powersimdata/data_access/profile_helper.py
index eb4a6fabc..5554a76df 100644
--- a/powersimdata/data_access/profile_helper.py
+++ b/powersimdata/data_access/profile_helper.py
@@ -17,26 +17,25 @@ def get_file_components(scenario_info, field_name):
 
         :param dict scenario_info: a ScenarioInfo instance
         :param str field_name: the kind of profile
-        :return: (*tuple*) -- file name and path
+        :return: (*tuple*) -- file name and list of path components
         """
         version = scenario_info["base_" + field_name]
         file_name = field_name + "_" + version + ".csv"
         grid_model = scenario_info["grid_model"]
-        from_dir = os.path.join("raw", grid_model)
-        return file_name, from_dir
+        return file_name, ("raw", grid_model)
 
     @staticmethod
     def download_file(file_name, from_dir):
         """Download the profile from blob storage at the given path
 
         :param str file_name: profile csv
-        :param str from_dir: the path relative to the blob container
+        :param tuple from_dir: tuple of path components
         :return: (*str*) -- path to downloaded file
         """
         print(f"--> Downloading {file_name} from blob storage.")
-        url_path = "/".join(os.path.split(from_dir))
+        url_path = "/".join(from_dir)
         url = f"{ProfileHelper.BASE_URL}/{url_path}/{file_name}"
-        dest = os.path.join(server_setup.LOCAL_DIR, from_dir, file_name)
+        dest = os.path.join(server_setup.LOCAL_DIR, *from_dir, file_name)
         os.makedirs(os.path.dirname(dest), exist_ok=True)
         resp = requests.get(url, stream=True)
         content_length = int(resp.headers.get("content-length", 0))
diff --git a/powersimdata/data_access/tests/test_profile_helper.py b/powersimdata/data_access/tests/test_profile_helper.py
index df90e2036..ce67b0afd 100644
--- a/powersimdata/data_access/tests/test_profile_helper.py
+++ b/powersimdata/data_access/tests/test_profile_helper.py
@@ -1,5 +1,3 @@
-import os
-
 from powersimdata.data_access.profile_helper import ProfileHelper
 
 
@@ -23,4 +21,4 @@ def test_get_file_components():
     s_info = {"base_wind": "v8", "grid_model": "europe"}
     file_name, from_dir = ProfileHelper.get_file_components(s_info, "wind")
     assert "wind_v8.csv" == file_name
-    assert os.path.join("raw", "europe") == from_dir
+    assert ("raw", "europe") == from_dir
diff --git a/powersimdata/input/input_data.py b/powersimdata/input/input_data.py
index c16d0876a..7fa64954d 100644
--- a/powersimdata/input/input_data.py
+++ b/powersimdata/input/input_data.py
@@ -28,19 +28,19 @@ def get_file_components(scenario_info, field_name):
 
         :param dict scenario_info: a ScenarioInfo instance
         :param str field_name: the input file type
-        :return: (*tuple*) -- file name and path
+        :return: (*tuple*) -- file name and list of path components
         """
         ext = _file_extension[field_name]
         file_name = scenario_info["id"] + "_" + field_name + "." + ext
-        from_dir = server_setup.INPUT_DIR
-        return file_name, from_dir
+        return file_name, server_setup.INPUT_DIR
 
     def download_file(self, file_name, from_dir):
         """Download the file if using server, otherwise no-op
 
         :param str file_name: either grid or ct file name
-        :param str from_dir: the path relative to the root dir
+        :param tuple from_dir: tuple of path components
         """
+        from_dir = self.data_access.join(*from_dir)
         self.data_access.copy_from(file_name, from_dir)
 
 
@@ -90,7 +90,7 @@ def get_data(self, scenario_info, field_name):
 
         file_name, from_dir = helper.get_file_components(scenario_info, field_name)
 
-        filepath = os.path.join(server_setup.LOCAL_DIR, from_dir, file_name)
+        filepath = os.path.join(server_setup.LOCAL_DIR, *from_dir, file_name)
         key = cache_key(filepath)
         cached = _cache.get(key)
         if cached is not None:
diff --git a/powersimdata/input/tests/test_input_data.py b/powersimdata/input/tests/test_input_data.py
index 0e37b3b20..fc90da6b8 100644
--- a/powersimdata/input/tests/test_input_data.py
+++ b/powersimdata/input/tests/test_input_data.py
@@ -9,7 +9,7 @@ def test_get_file_components():
     grid_file, from_dir = InputHelper.get_file_components(s_info, "grid")
     assert "123_ct.pkl" == ct_file
     assert "123_grid.mat" == grid_file
-    assert "data/input" == from_dir
+    assert ("data", "input") == from_dir
 
 
 def test_check_field():
diff --git a/powersimdata/output/output_data.py b/powersimdata/output/output_data.py
index 5da537b95..c835f8c10 100644
--- a/powersimdata/output/output_data.py
+++ b/powersimdata/output/output_data.py
@@ -36,7 +36,7 @@ def get_data(self, scenario_id, field_name):
         print("--> Loading %s" % field_name)
         file_name = scenario_id + "_" + field_name + ".pkl"
         from_dir = server_setup.OUTPUT_DIR
-        filepath = os.path.join(server_setup.LOCAL_DIR, from_dir, file_name)
+        filepath = os.path.join(server_setup.LOCAL_DIR, *from_dir, file_name)
 
         try:
             return pd.read_pickle(filepath)
@@ -46,7 +46,8 @@ def get_data(self, scenario_id, field_name):
         except FileNotFoundError:
             print(f"{filepath} not found on local machine")
 
-        self._data_access.copy_from(file_name, from_dir)
+        remote_dir = self._data_access.join(*from_dir)
+        self._data_access.copy_from(file_name, remote_dir)
         return pd.read_pickle(filepath)
 
 
diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py
index bb11e27b5..bc0b677b4 100644
--- a/powersimdata/scenario/create.py
+++ b/powersimdata/scenario/create.py
@@ -80,7 +80,8 @@ def _upload_change_table(self):
         print("--> Writing change table on local machine")
         self.builder.change_table.write(self._scenario_info["id"])
         file_name = self._scenario_info["id"] + "_ct.pkl"
-        self._data_access.move_to(file_name, server_setup.INPUT_DIR)
+        input_dir = self._data_access.join(*server_setup.INPUT_DIR)
+        self._data_access.move_to(file_name, input_dir)
 
     def get_bus_demand(self):
         """Returns demand profiles, by bus.
diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py
index a65c96cda..e65a74e45 100644
--- a/powersimdata/utility/server_setup.py
+++ b/powersimdata/utility/server_setup.py
@@ -6,8 +6,8 @@
 BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
 DATA_ROOT_DIR = "/mnt/bes/pcm"
 EXECUTE_DIR = "tmp"
-INPUT_DIR = "data/input"
-OUTPUT_DIR = "data/output"
+INPUT_DIR = ("data", "input")
+OUTPUT_DIR = ("data", "output")
 LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
 MODEL_DIR = "/home/bes/pcm"
 

From dec82c101cf7656e785b3a7fac91339677608d68 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 3 May 2021 12:51:59 -0700
Subject: [PATCH 17/60] refactor: avoid hard coded strings

---
 powersimdata/data_access/data_access.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index 0211ef54e..867f2d040 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -14,6 +14,12 @@
 from powersimdata.utility import server_setup
 from powersimdata.utility.helpers import CommandBuilder
 
+_dirs = {
+    "tmp": (server_setup.EXECUTE_DIR,),
+    "input": server_setup.INPUT_DIR,
+    "output": server_setup.OUTPUT_DIR,
+}
+
 
 class DataAccess:
     """Interface to a local or remote data store."""
@@ -49,14 +55,12 @@ def get_base_dir(self, kind, backup=False):
         :raises ValueError: if kind is invalid
         :return: (*str*) -- the specified path
         """
-        _allowed = ("input", "output", "tmp")
+        _allowed = list(_dirs.keys())
         if kind not in _allowed:
             raise ValueError(f"Invalid 'kind', must be one of {_allowed}")
 
         root = self.root if not backup else self.backup_root
-        if kind == "tmp":
-            return self.join(root, "tmp")
-        return self.join(root, "data", kind)
+        return self.join(root, *_dirs[kind])
 
     def match_scenario_files(self, scenario_id, kind, backup=False):
         """Get path matching the given kind of scenario data

From 3a53e65e2be625996d9cfca65205a6294ac4a6f2 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 3 May 2021 16:48:21 -0700
Subject: [PATCH 18/60] chore: use consistent self.join

---
 powersimdata/data_access/data_access.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index 867f2d040..bb751ec15 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -219,9 +219,9 @@ def move_to(self, file_name, to_dir, change_name_to=None):
         :param str change_name_to: new name for file when copied to data store.
         """
         self._check_filename(file_name)
-        src = os.path.join(server_setup.LOCAL_DIR, file_name)
+        src = self.join(server_setup.LOCAL_DIR, file_name)
         file_name = file_name if change_name_to is None else change_name_to
-        dest = os.path.join(self.root, to_dir, file_name)
+        dest = self.join(self.root, to_dir, file_name)
         print(f"--> Moving file {src} to {dest}")
         self._check_file_exists(dest, should_exist=False)
         self.makedir(os.path.dirname(dest))
@@ -248,10 +248,10 @@ def copy(self, src, dest, recursive=False, update=False):
         :param bool recursive: create directories recursively
         :param bool update: ignored
         """
-        self.makedir(dest)
         if recursive:
             shutil.copytree(src, dest)
         else:
+            self.makedir(dest)
             func = lambda s: shutil.copy(s, dest)  # noqa: E731
             LocalDataAccess._fapply(func, src)
 
@@ -366,7 +366,7 @@ def copy_from(self, file_name, from_dir=None):
         to_dir = os.path.join(self.local_root, from_dir)
         os.makedirs(to_dir, exist_ok=True)
 
-        from_path = posixpath.join(self.root, from_dir, file_name)
+        from_path = self.join(self.root, from_dir, file_name)
         to_path = os.path.join(to_dir, file_name)
         self._check_file_exists(from_path, should_exist=True)
 
@@ -397,8 +397,8 @@ def move_to(self, file_name, to_dir=None, change_name_to=None):
             )
 
         file_name = file_name if change_name_to is None else change_name_to
-        to_dir = posixpath.join(self.root, "" if to_dir is None else to_dir)
-        to_path = posixpath.join(to_dir, file_name)
+        to_dir = self.join(self.root, "" if to_dir is None else to_dir)
+        to_path = self.join(to_dir, file_name)
         self.makedir(to_dir)
         self._check_file_exists(to_path, should_exist=False)
 
@@ -434,7 +434,7 @@ def checksum(self, relative_path):
         :param str relative_path: path relative to root
         :return: (*str*) -- the checksum of the file
         """
-        full_path = posixpath.join(self.root, relative_path)
+        full_path = self.join(self.root, relative_path)
         self._check_file_exists(full_path)
 
         command = f"sha1sum {full_path}"
@@ -455,9 +455,9 @@ def push(self, file_name, checksum, change_name_to=None):
         self.move_to(file_name, change_name_to=backup)
 
         values = {
-            "original": posixpath.join(self.root, new_name),
-            "updated": posixpath.join(self.root, backup),
-            "lockfile": posixpath.join(self.root, "scenario.lockfile"),
+            "original": self.join(self.root, new_name),
+            "updated": self.join(self.root, backup),
+            "lockfile": self.join(self.root, "scenario.lockfile"),
             "checksum": checksum,
         }
 

From 8f62277a869cb22240ccda6c5c845de8230befae Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 4 May 2021 11:33:42 -0700
Subject: [PATCH 19/60] chore: remove _execute_command

---
 powersimdata/data_access/data_access.py       | 27 +++++--------------
 .../data_access/tests/test_data_access.py     |  2 +-
 2 files changed, 7 insertions(+), 22 deletions(-)

diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index bb751ec15..0d900b7ca 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -131,13 +131,6 @@ def makedir(self, full_path):
         """
         raise NotImplementedError
 
-    def _execute_command(self, command):
-        """Execute a command locally at the data access.
-
-        :param list command: list of str to be passed to command line.
-        """
-        raise NotImplementedError
-
     def execute_command_async(self, command):
         """Execute a command locally at the DataAccess, without waiting for completion.
 
@@ -408,14 +401,6 @@ def move_to(self, file_name, to_dir=None, change_name_to=None):
 
         os.remove(from_path)
 
-    def _execute_command(self, command):
-        """Execute a command locally at the data access.
-
-        :param list command: list of str to be passed to command line.
-        :return: (*tuple*) -- stdin, stdout, stderr of executed command.
-        """
-        return self.ssh.exec_command(command)
-
     def execute_command_async(self, command):
         """Execute a command via ssh, without waiting for completion.
 
@@ -438,7 +423,7 @@ def checksum(self, relative_path):
         self._check_file_exists(full_path)
 
         command = f"sha1sum {full_path}"
-        _, stdout, _ = self._execute_command(command)
+        _, stdout, _ = self.ssh.exec_command(command)
         lines = stdout.readlines()
         return lines[0].strip()
 
@@ -469,7 +454,7 @@ def push(self, file_name, checksum, change_name_to=None):
                 200>{lockfile}"
 
         command = template.format(**values)
-        _, _, stderr = self._execute_command(command)
+        _, _, stderr = self.ssh.exec_command(command)
 
         errors = stderr.readlines()
         if len(errors) > 0:
@@ -483,7 +468,7 @@ def makedir(self, full_path):
         :param str full_path: the path, excluding filename
         :raises IOError: if command generated stderr
         """
-        _, _, stderr = self._execute_command(f"mkdir -p {full_path}")
+        _, _, stderr = self.ssh.exec_command(f"mkdir -p {full_path}")
         errors = stderr.readlines()
         if len(errors) > 0:
             raise IOError(f"Failed to create {full_path} on server")
@@ -499,7 +484,7 @@ def copy(self, src, dest, recursive=False, update=False):
         """
         self.makedir(dest)
         command = CommandBuilder.copy(src, dest, recursive, update)
-        _, _, stderr = self._execute_command(command)
+        _, _, stderr = self.ssh.exec_command(command)
         if len(stderr.readlines()) != 0:
             raise IOError(f"Failed to execute {command}")
 
@@ -517,7 +502,7 @@ def remove(self, target, recursive=False, confirm=True):
             if confirmed.lower() != "y":
                 print("Operation cancelled.")
                 return
-        _, _, stderr = self._execute_command(command)
+        _, _, stderr = self.ssh.exec_command(command)
         if len(stderr.readlines()) != 0:
             raise IOError(f"Failed to delete target={target} on server")
         print("--> Done!")
@@ -528,7 +513,7 @@ def _exists(self, filepath):
         :param str filepath: the path to the file
         :return: (*bool*) -- whether the file exists
         """
-        _, _, stderr = self._execute_command(f"ls {filepath}")
+        _, _, stderr = self.ssh.exec_command(f"ls {filepath}")
         return len(stderr.readlines()) == 0
 
     def close(self):
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index 29e52dd01..f83aa7f3b 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -99,7 +99,7 @@ def test_match_scenario_files(data_access):
 @pytest.mark.integration
 @pytest.mark.ssh
 def test_setup_server_connection(data_access):
-    _, stdout, _ = data_access._execute_command("whoami")
+    _, stdout, _ = data_access.ssh.exec_command("whoami")
     assert stdout.read().decode("utf-8").strip() == server_setup.get_server_user()
 
 

From a2cab2be8556f4b2d140624b1c7910b7a268d1f0 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 5 May 2021 13:46:35 -0700
Subject: [PATCH 20/60] chore: cleanup print statements and reuse join method

---
 powersimdata/scenario/delete.py  | 15 ++++++++-------
 powersimdata/scenario/execute.py |  3 ++-
 2 files changed, 10 insertions(+), 8 deletions(-)

diff --git a/powersimdata/scenario/delete.py b/powersimdata/scenario/delete.py
index f7ffe3098..7a9587a0e 100644
--- a/powersimdata/scenario/delete.py
+++ b/powersimdata/scenario/delete.py
@@ -1,5 +1,3 @@
-import os
-
 from powersimdata.data_access.data_access import LocalDataAccess
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
@@ -40,22 +38,25 @@ def delete_scenario(self, confirm=True):
         self._scenario_list_manager.delete_entry(scenario_id)
         self._execute_list_manager.delete_entry(scenario_id)
 
-        print("--> Deleting scenario input data on server")
+        print("--> Deleting scenario input data")
         target = self._data_access.match_scenario_files(scenario_id, "input")
         self._data_access.remove(target, recursive=False, confirm=confirm)
 
-        print("--> Deleting scenario output data on server")
+        print("--> Deleting scenario output data")
         target = self._data_access.match_scenario_files(scenario_id, "output")
         self._data_access.remove(target, recursive=False, confirm=confirm)
 
         # Delete temporary folder enclosing simulation inputs
-        print("--> Deleting temporary folder on server")
+        print("--> Deleting temporary folder")
         tmp_dir = self._data_access.match_scenario_files(scenario_id, "tmp")
         self._data_access.remove(tmp_dir, recursive=True, confirm=confirm)
 
         print("--> Deleting input and output data on local machine")
-        target = os.path.join(server_setup.LOCAL_DIR, "data", "**", f"{scenario_id}_*")
-        LocalDataAccess().remove(target, recursive=False, confirm=confirm)
+        local_data_access = LocalDataAccess()
+        target = local_data_access.join(
+            server_setup.LOCAL_DIR, "data", "**", f"{scenario_id}_*"
+        )
+        local_data_access.remove(target, recursive=False, confirm=confirm)
 
         # Delete attributes
         self._clean()
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 2a69175bd..9a92e30bc 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -340,7 +340,8 @@ def __init__(self, data_access, scenario_info, grid, ct):
 
     def create_folder(self):
         """Creates folder on server that will enclose simulation inputs."""
-        print("--> Creating temporary folder on server for simulation inputs")
+        description = self._data_access.description
+        print(f"--> Creating temporary folder on {description} for simulation inputs")
         self._data_access.makedir(self.TMP_DIR)
 
     def prepare_mpc_file(self):

From f01e4dc0727e896c81f6a59e1b34927ac08b5930 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Thu, 13 May 2021 08:12:12 -0700
Subject: [PATCH 21/60] style: add badges to README (#478)

---
 README.md | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/README.md b/README.md
index 5c62be1b9..f12274190 100644
--- a/README.md
+++ b/README.md
@@ -1,11 +1,16 @@
+[![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 ![Tests](https://github.com/Breakthrough-Energy/PowerSimData/workflows/Pytest/badge.svg)
-[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Documentation](https://github.com/Breakthrough-Energy/docs/actions/workflows/publish.yml/badge.svg)](https://breakthrough-energy.github.io/docs/)
+![GitHub contributors](https://img.shields.io/github/contributors/Breakthrough-Energy/PowerSimData?logo=GitHub)
+![GitHub commit activity](https://img.shields.io/github/commit-activity/m/Breakthrough-Energy/PowerSimData?logo=GitHub)
+![GitHub last commit (branch)](https://img.shields.io/github/last-commit/Breakthrough-Energy/PowerSimData/develop?logo=GitHub)
+![GitHub pull requests](https://img.shields.io/github/issues-pr/Breakthrough-Energy/PowerSimData?logo=GitHub)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
+[![Code of Conduct](https://img.shields.io/badge/code%20of-conduct-ff69b4.svg?style=flat)](https://breakthrough-energy.github.io/docs/communication/code_of_conduct.html)
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4538590.svg)](https://doi.org/10.5281/zenodo.4538590)
 
 
-# PowerSimData
 **PowerSimData** is part of a Python software ecosystem developed by [Breakthrough
 Energy Sciences](https://science.breakthroughenergy.org/) to carry out power flow study
 in the U.S. electrical grid.

From 7efaaac7552db56a046eea12f0d33a5f39ae3ad0 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Thu, 13 May 2021 14:35:55 -0700
Subject: [PATCH 22/60] style: add logo to README (#479)

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index f12274190..b4b7c54fe 100644
--- a/README.md
+++ b/README.md
@@ -1,3 +1,6 @@
+
+![logo](https://raw.githubusercontent.com/Breakthrough-Energy/docs/master/source/_static/img/BE_Sciences_RGB_Horizontal_Color.svg)
+
 [![made-with-python](https://img.shields.io/badge/Made%20with-Python-1f425f.svg)](https://www.python.org/)
 [![Code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black)
 ![Tests](https://github.com/Breakthrough-Energy/PowerSimData/workflows/Pytest/badge.svg)
@@ -11,6 +14,7 @@
 [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.4538590.svg)](https://doi.org/10.5281/zenodo.4538590)
 
 
+# PowerSimData
 **PowerSimData** is part of a Python software ecosystem developed by [Breakthrough
 Energy Sciences](https://science.breakthroughenergy.org/) to carry out power flow study
 in the U.S. electrical grid.

From d60d7078a0ea6f3dbb787ffa71314c41c734f1a6 Mon Sep 17 00:00:00 2001
From: jon-hagg <66005238+jon-hagg@users.noreply.github.com>
Date: Fri, 14 May 2021 10:55:31 -0700
Subject: [PATCH 23/60] chore: make psycopg2 optional (#480)

---
 optional-requirements.txt                     |  1 +
 powersimdata/data_access/execute_list.py      | 66 -------------------
 powersimdata/data_access/execute_table.py     | 66 +++++++++++++++++++
 powersimdata/data_access/scenario_list.py     | 66 -------------------
 powersimdata/data_access/scenario_table.py    | 66 +++++++++++++++++++
 .../data_access/tests/test_data_access.py     |  2 +-
 .../data_access/tests/test_execute_table.py   |  2 +-
 .../data_access/tests/test_scenario_table.py  |  2 +-
 requirements.txt                              |  1 -
 setup.py                                      |  1 -
 10 files changed, 136 insertions(+), 137 deletions(-)
 create mode 100644 powersimdata/data_access/execute_table.py
 create mode 100644 powersimdata/data_access/scenario_table.py

diff --git a/optional-requirements.txt b/optional-requirements.txt
index e221b9773..e344bd12f 100644
--- a/optional-requirements.txt
+++ b/optional-requirements.txt
@@ -4,3 +4,4 @@ fiona~=1.8.14
 matplotlib==3.2.1
 rtree~=0.9.4
 shapely==1.7.1
+psycopg2~=2.8.5
diff --git a/powersimdata/data_access/execute_list.py b/powersimdata/data_access/execute_list.py
index cdf42b44e..1e70fbd88 100644
--- a/powersimdata/data_access/execute_list.py
+++ b/powersimdata/data_access/execute_list.py
@@ -1,70 +1,4 @@
 from powersimdata.data_access.csv_store import CsvStore, verify_hash
-from powersimdata.data_access.sql_store import SqlStore, to_data_frame
-
-
-class ExecuteTable(SqlStore):
-    """Storage abstraction for execute list using sql database."""
-
-    table = "execute_list"
-    columns = ["id", "status"]
-
-    def get_status(self, scenario_id):
-        """Get status of scenario by scenario_id
-
-        :param str scenario_id: the scenario id
-        :return: (*pandas.DataFrame*) -- results as a data frame.
-        """
-        query = self.select_where("id")
-        self.cur.execute(query, (scenario_id,))
-        result = self.cur.fetchmany()
-        return to_data_frame(result)
-
-    def get_execute_table(self, limit=None):
-        """Return the execute table as a data frame
-
-        :return: (*pandas.DataFrame*) -- execute list as a data frame.
-        """
-        query = self.select_all()
-        self.cur.execute(query)
-        if limit is None:
-            result = self.cur.fetchall()
-        else:
-            result = self.cur.fetchmany(limit)
-        return to_data_frame(result)
-
-    def add_entry(self, scenario_info):
-        """Add entry to execute list
-
-        :param collections.OrderedDict scenario_info: entry to add
-        """
-        scenario_id, status = scenario_info["id"], "created"
-        sql = self.insert()
-        self.cur.execute(
-            sql,
-            (
-                scenario_id,
-                status,
-            ),
-        )
-
-    def set_status(self, scenario_id, status):
-        """Updates status of scenario in execute list
-
-        :param int scenario_id: the scenario id
-        :param str status: execution status.
-        """
-        self.cur.execute(
-            "UPDATE execute_list SET status = %s WHERE id = %s",
-            (status, scenario_id),
-        )
-
-    def delete_entry(self, scenario_id):
-        """Deletes entry from execute list.
-
-        :param int/str scenario_id: the id of the scenario
-        """
-        sql = self.delete("id")
-        self.cur.execute(sql, (scenario_id,))
 
 
 class ExecuteListManager(CsvStore):
diff --git a/powersimdata/data_access/execute_table.py b/powersimdata/data_access/execute_table.py
new file mode 100644
index 000000000..1a0cfda33
--- /dev/null
+++ b/powersimdata/data_access/execute_table.py
@@ -0,0 +1,66 @@
+from powersimdata.data_access.sql_store import SqlStore, to_data_frame
+
+
+class ExecuteTable(SqlStore):
+    """Storage abstraction for execute list using sql database."""
+
+    table = "execute_list"
+    columns = ["id", "status"]
+
+    def get_status(self, scenario_id):
+        """Get status of scenario by scenario_id
+
+        :param str scenario_id: the scenario id
+        :return: (*pandas.DataFrame*) -- results as a data frame.
+        """
+        query = self.select_where("id")
+        self.cur.execute(query, (scenario_id,))
+        result = self.cur.fetchmany()
+        return to_data_frame(result)
+
+    def get_execute_table(self, limit=None):
+        """Return the execute table as a data frame
+
+        :return: (*pandas.DataFrame*) -- execute list as a data frame.
+        """
+        query = self.select_all()
+        self.cur.execute(query)
+        if limit is None:
+            result = self.cur.fetchall()
+        else:
+            result = self.cur.fetchmany(limit)
+        return to_data_frame(result)
+
+    def add_entry(self, scenario_info):
+        """Add entry to execute list
+
+        :param collections.OrderedDict scenario_info: entry to add
+        """
+        scenario_id, status = scenario_info["id"], "created"
+        sql = self.insert()
+        self.cur.execute(
+            sql,
+            (
+                scenario_id,
+                status,
+            ),
+        )
+
+    def set_status(self, scenario_id, status):
+        """Updates status of scenario in execute list
+
+        :param int scenario_id: the scenario id
+        :param str status: execution status.
+        """
+        self.cur.execute(
+            "UPDATE execute_list SET status = %s WHERE id = %s",
+            (status, scenario_id),
+        )
+
+    def delete_entry(self, scenario_id):
+        """Deletes entry from execute list.
+
+        :param int/str scenario_id: the id of the scenario
+        """
+        sql = self.delete("id")
+        self.cur.execute(sql, (scenario_id,))
diff --git a/powersimdata/data_access/scenario_list.py b/powersimdata/data_access/scenario_list.py
index da65f8515..084817712 100644
--- a/powersimdata/data_access/scenario_list.py
+++ b/powersimdata/data_access/scenario_list.py
@@ -3,72 +3,6 @@
 import pandas as pd
 
 from powersimdata.data_access.csv_store import CsvStore, verify_hash
-from powersimdata.data_access.sql_store import SqlStore, to_data_frame
-
-
-class ScenarioTable(SqlStore):
-    """Storage abstraction for scenario list using sql database."""
-
-    table = "scenario_list"
-    columns = [
-        "id",
-        "plan",
-        "name",
-        "state",
-        "grid_model",
-        "interconnect",
-        "base_demand",
-        "base_hydro",
-        "base_solar",
-        "base_wind",
-        "change_table",
-        "start_date",
-        "end_date",
-        "interval",
-        "engine",
-        "runtime",
-        "infeasibilities",
-    ]
-
-    def get_scenario_by_id(self, scenario_id):
-        """Get entry from scenario list by id
-
-        :param str scenario_id: scenario id
-        :return: (*pandas.DataFrame*) -- results as a data frame.
-        """
-        query = self.select_where("id")
-        self.cur.execute(query, (scenario_id,))
-        result = self.cur.fetchmany()
-        return to_data_frame(result)
-
-    def get_scenario_table(self, limit=None):
-        """Returns scenario table from database
-
-        :return: (*pandas.DataFrame*) -- scenario list as a data frame.
-        """
-        query = self.select_all()
-        self.cur.execute(query)
-        if limit is None:
-            result = self.cur.fetchall()
-        else:
-            result = self.cur.fetchmany(limit)
-        return to_data_frame(result)
-
-    def add_entry(self, scenario_info):
-        """Adds scenario to the scenario list.
-
-        :param collections.OrderedDict scenario_info: entry to add to scenario list.
-        """
-        sql = self.insert(subset=scenario_info.keys())
-        self.cur.execute(sql, tuple(scenario_info.values()))
-
-    def delete_entry(self, scenario_id):
-        """Deletes entry in scenario list.
-
-        :param int/str scenario_id: the id of the scenario
-        """
-        sql = self.delete("id")
-        self.cur.execute(sql, (scenario_id,))
 
 
 class ScenarioListManager(CsvStore):
diff --git a/powersimdata/data_access/scenario_table.py b/powersimdata/data_access/scenario_table.py
new file mode 100644
index 000000000..ad4383a85
--- /dev/null
+++ b/powersimdata/data_access/scenario_table.py
@@ -0,0 +1,66 @@
+from powersimdata.data_access.sql_store import SqlStore, to_data_frame
+
+
+class ScenarioTable(SqlStore):
+    """Storage abstraction for scenario list using sql database."""
+
+    table = "scenario_list"
+    columns = [
+        "id",
+        "plan",
+        "name",
+        "state",
+        "grid_model",
+        "interconnect",
+        "base_demand",
+        "base_hydro",
+        "base_solar",
+        "base_wind",
+        "change_table",
+        "start_date",
+        "end_date",
+        "interval",
+        "engine",
+        "runtime",
+        "infeasibilities",
+    ]
+
+    def get_scenario_by_id(self, scenario_id):
+        """Get entry from scenario list by id
+
+        :param str scenario_id: scenario id
+        :return: (*pandas.DataFrame*) -- results as a data frame.
+        """
+        query = self.select_where("id")
+        self.cur.execute(query, (scenario_id,))
+        result = self.cur.fetchmany()
+        return to_data_frame(result)
+
+    def get_scenario_table(self, limit=None):
+        """Returns scenario table from database
+
+        :return: (*pandas.DataFrame*) -- scenario list as a data frame.
+        """
+        query = self.select_all()
+        self.cur.execute(query)
+        if limit is None:
+            result = self.cur.fetchall()
+        else:
+            result = self.cur.fetchmany(limit)
+        return to_data_frame(result)
+
+    def add_entry(self, scenario_info):
+        """Adds scenario to the scenario list.
+
+        :param collections.OrderedDict scenario_info: entry to add to scenario list.
+        """
+        sql = self.insert(subset=scenario_info.keys())
+        self.cur.execute(sql, tuple(scenario_info.values()))
+
+    def delete_entry(self, scenario_id):
+        """Deletes entry in scenario list.
+
+        :param int/str scenario_id: the id of the scenario
+        """
+        sql = self.delete("id")
+        self.cur.execute(sql, (scenario_id,))
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index f83aa7f3b..7a2027c32 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -67,7 +67,7 @@ def _check_content(filepath):
         assert CONTENT == f.read()
 
 
-root_dir = server_setup.DATA_ROOT_DIR
+root_dir = server_setup.DATA_ROOT_DIR.rstrip("/")
 backup_root = server_setup.BACKUP_DATA_ROOT_DIR
 
 
diff --git a/powersimdata/data_access/tests/test_execute_table.py b/powersimdata/data_access/tests/test_execute_table.py
index b1969694c..abf90eb46 100644
--- a/powersimdata/data_access/tests/test_execute_table.py
+++ b/powersimdata/data_access/tests/test_execute_table.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from powersimdata.data_access.execute_list import ExecuteTable
+from powersimdata.data_access.execute_table import ExecuteTable
 from powersimdata.data_access.sql_store import SqlException
 
 row_id = 9000
diff --git a/powersimdata/data_access/tests/test_scenario_table.py b/powersimdata/data_access/tests/test_scenario_table.py
index 3b3ad0167..2093c2719 100644
--- a/powersimdata/data_access/tests/test_scenario_table.py
+++ b/powersimdata/data_access/tests/test_scenario_table.py
@@ -2,7 +2,7 @@
 
 import pytest
 
-from powersimdata.data_access.scenario_list import ScenarioTable
+from powersimdata.data_access.scenario_table import ScenarioTable
 from powersimdata.data_access.sql_store import SqlException
 
 # uncomment to enable logging queries to stdout
diff --git a/requirements.txt b/requirements.txt
index 4f7d00a3b..47f813e0f 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,7 +1,6 @@
 numpy~=1.20
 pandas~=1.2
 paramiko==2.7.2
-psycopg2~=2.8.5
 scipy~=1.5
 tqdm==4.29.1
 requests~=2.25
diff --git a/setup.py b/setup.py
index bf384c476..e42ff3082 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,6 @@
     "paramiko",
     "scipy",
     "tqdm",
-    "psycopg2",
     "requests",
 ]
 

From 88699a917e1e6b40c193fceda479cf830949cba2 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 3 May 2021 16:33:45 -0700
Subject: [PATCH 24/60] feat: modular config pattern

---
 powersimdata/data_access/context.py  |  2 +-
 powersimdata/scenario/execute.py     |  2 +-
 powersimdata/utility/config.py       | 54 ++++++++++++++++++++++++++++
 powersimdata/utility/server_setup.py | 34 +++++++-----------
 4 files changed, 68 insertions(+), 24 deletions(-)
 create mode 100644 powersimdata/utility/config.py

diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py
index e14dcfb67..d84abbda9 100644
--- a/powersimdata/data_access/context.py
+++ b/powersimdata/data_access/context.py
@@ -1,6 +1,6 @@
 from powersimdata.data_access.data_access import LocalDataAccess, SSHDataAccess
 from powersimdata.utility import server_setup
-from powersimdata.utility.server_setup import DeploymentMode, get_deployment_mode
+from powersimdata.utility.config import DeploymentMode, get_deployment_mode
 
 
 class Context:
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 9a92e30bc..fdb86ce54 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -11,7 +11,7 @@
 from powersimdata.input.transform_profile import TransformProfile
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
-from powersimdata.utility.server_setup import DeploymentMode, get_deployment_mode
+from powersimdata.utility.config import DeploymentMode, get_deployment_mode
 
 
 class Execute(State):
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
new file mode 100644
index 000000000..498c3df54
--- /dev/null
+++ b/powersimdata/utility/config.py
@@ -0,0 +1,54 @@
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Config:
+    DATA_ROOT_DIR = "/mnt/bes/pcm"
+    EXECUTE_DIR = "tmp"
+    INPUT_DIR = ("data", "input")
+    OUTPUT_DIR = ("data", "output")
+    LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
+    MODEL_DIR = "/home/bes/pcm"
+
+
+@dataclass(frozen=True)
+class ServerConfig(Config):
+    SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
+    SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
+    BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
+
+
+@dataclass(frozen=True)
+class ContainerConfig(Config):
+    SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "reisejl")
+
+
+@dataclass(frozen=True)
+class LocalConfig(Config):
+    DATA_ROOT_DIR = Config.LOCAL_DIR
+
+
+class DeploymentMode:
+    Server = "SERVER"
+    Container = "CONTAINER"
+    Local = "LOCAL"
+
+    CONFIG_MAP = {Server: ServerConfig, Container: ContainerConfig, Local: LocalConfig}
+
+
+def get_deployment_mode():
+    # TODO: consider auto detection
+    mode = os.getenv("DEPLOYMENT_MODE")
+    if mode is None:
+        return DeploymentMode.Server
+    if mode == "1" or mode.lower() == "container":
+        return DeploymentMode.Container
+    if mode == "2" or mode.lower() == "local":
+        return DeploymentMode.Local
+
+
+def get_config():
+    mode = get_deployment_mode()
+    return DeploymentMode.CONFIG_MAP[mode]()
diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py
index e65a74e45..55d7f84ad 100644
--- a/powersimdata/utility/server_setup.py
+++ b/powersimdata/utility/server_setup.py
@@ -1,27 +1,17 @@
 import os
-from pathlib import Path
 
-SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
-SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
-BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
-DATA_ROOT_DIR = "/mnt/bes/pcm"
-EXECUTE_DIR = "tmp"
-INPUT_DIR = ("data", "input")
-OUTPUT_DIR = ("data", "output")
-LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
-MODEL_DIR = "/home/bes/pcm"
-
-
-class DeploymentMode:
-    Server = "SERVER"
-    Container = "CONTAINER"
-
-
-def get_deployment_mode():
-    mode = os.getenv("DEPLOYMENT_MODE")
-    if mode is None:
-        return DeploymentMode.Server
-    return DeploymentMode.Container
+from powersimdata.utility.config import get_config
+
+config = get_config()
+SERVER_ADDRESS = config.SERVER_ADDRESS
+SERVER_SSH_PORT = config.SERVER_SSH_PORT
+BACKUP_DATA_ROOT_DIR = config.BACKUP_DATA_ROOT_DIR
+DATA_ROOT_DIR = config.DATA_ROOT_DIR
+EXECUTE_DIR = config.EXECUTE_DIR
+INPUT_DIR = config.INPUT_DIR
+OUTPUT_DIR = config.OUTPUT_DIR
+LOCAL_DIR = config.LOCAL_DIR
+MODEL_DIR = config.MODEL_DIR
 
 
 def get_server_user():

From e3a1fac5f05d7f82f301316fdd95010592ae2e30 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 4 May 2021 12:30:36 -0700
Subject: [PATCH 25/60] refactor: split launch_simulation into subclasses

---
 powersimdata/data_access/context.py  |  14 +++
 powersimdata/data_access/launcher.py | 139 +++++++++++++++++++++++++++
 powersimdata/scenario/execute.py     | 128 ++----------------------
 3 files changed, 163 insertions(+), 118 deletions(-)
 create mode 100644 powersimdata/data_access/launcher.py

diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py
index d84abbda9..afb7caa38 100644
--- a/powersimdata/data_access/context.py
+++ b/powersimdata/data_access/context.py
@@ -1,4 +1,5 @@
 from powersimdata.data_access.data_access import LocalDataAccess, SSHDataAccess
+from powersimdata.data_access.launcher import HttpLauncher, NativeLauncher, SSHLauncher
 from powersimdata.utility import server_setup
 from powersimdata.utility.config import DeploymentMode, get_deployment_mode
 
@@ -23,3 +24,16 @@ def get_data_access(data_loc=None):
         if mode == DeploymentMode.Server:
             return SSHDataAccess(root)
         return LocalDataAccess(root)
+
+    @staticmethod
+    def get_launcher(scenario):
+        """Return instance for interaction with simulation engine
+
+        :param powersimdata.Scenario scenario: a scenario object
+        """
+        mode = get_deployment_mode()
+        if mode == DeploymentMode.Server:
+            return SSHLauncher(scenario)
+        elif mode == DeploymentMode.Container:
+            return HttpLauncher(scenario)
+        return NativeLauncher(scenario)
diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
new file mode 100644
index 000000000..bc53acc31
--- /dev/null
+++ b/powersimdata/data_access/launcher.py
@@ -0,0 +1,139 @@
+import posixpath
+
+import requests
+
+from powersimdata.utility import server_setup
+
+
+def _check_threads(threads):
+    """Validate threads argument
+
+    :param int threads: the number of threads to be used
+    :raises TypeError: if threads is not an int
+    :raises ValueError: if threads is not a positive value
+    """
+    if threads:
+        if not isinstance(threads, int):
+            raise TypeError("threads must be an int")
+        if threads < 1:
+            raise ValueError("threads must be a positive value")
+
+
+def _check_solver(solver):
+    """Validate solver argument
+
+    :param str solver: the solver used for the optimization
+    :raises ValueError: if invalid solver provided
+    """
+    solvers = ("gurobi", "glpk")
+    if solver is not None and solver.lower() not in solvers:
+        raise ValueError(f"Invalid solver: options are {solvers}")
+
+
+# TODO - check_progress (just print a message for SSHLauncher)
+# TODO - extract_data
+class Launcher:
+    def __init__(self, scenario):
+        self.scenario = scenario
+
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        raise NotImplementedError
+
+    def launch_simulation(self, threads=None, solver=None, extract_data=True):
+        _check_threads(threads)
+        _check_solver(solver)
+        self._launch(threads, solver, extract_data)
+
+
+class SSHLauncher(Launcher):
+    def _run_script(self, script, extra_args=None):
+        """Returns running process
+
+        :param str script: script to be used.
+        :param list extra_args: list of strings to be passed after scenario id.
+        :return: (*subprocess.Popen*) -- process used to run script
+        """
+        if extra_args is None:
+            extra_args = []
+
+        engine = self.scenario._scenario_info["engine"]
+        path_to_package = posixpath.join(server_setup.MODEL_DIR, engine)
+        folder = "pyreise" if engine == "REISE" else "pyreisejl"
+
+        path_to_script = posixpath.join(path_to_package, folder, "utility", script)
+        cmd_pythonpath = [f'export PYTHONPATH="{path_to_package}:$PYTHONPATH";']
+        cmd_pythoncall = [
+            "nohup",
+            "python3",
+            "-u",
+            path_to_script,
+            self.scenario.scenario_id,
+        ]
+        cmd_io_redirect = ["</dev/null >/dev/null 2>&1 &"]
+        cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect
+        process = self.scenario._data_access.execute_command_async(cmd)
+        print("PID: %s" % process.pid)
+        return process
+
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launch simulation on server, via ssh.
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
+        :raises TypeError: if extract_data is not a boolean
+        :return: (*subprocess.Popen*) -- new process used to launch simulation.
+        """
+        extra_args = []
+
+        if threads:
+            # Use the -t flag as defined in call.py in REISE.jl
+            extra_args.append("--threads " + str(threads))
+
+        if solver:
+            extra_args.append("--solver " + solver)
+
+        if not isinstance(extract_data, bool):
+            raise TypeError("extract_data must be a boolean: 'True' or 'False'")
+        if extract_data:
+            extra_args.append("--extract-data")
+
+        return self._run_script("call.py", extra_args=extra_args)
+
+
+class HttpLauncher(Launcher):
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation in container via http call
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: always True
+        :return: (*requests.Response*) -- the http response object
+        """
+        scenario_id = self.scenario.scenario_id
+        url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
+        resp = requests.post(url, params={"threads": threads, "solver": solver})
+        if resp.status_code != 200:
+            print(
+                f"Failed to launch simulation: status={resp.status_code}. See response for details"
+            )
+        return resp
+
+
+class NativeLauncher(Launcher):
+    def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation by importing from REISE.jl
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: always True
+        :return: (*dict*) -- json response
+        """
+        pass
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index fdb86ce54..7248786d0 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -4,6 +4,7 @@
 
 import requests
 
+from powersimdata.data_access.context import Context
 from powersimdata.input.case_mat import export_case_mat
 from powersimdata.input.grid import Grid
 from powersimdata.input.input_data import InputData
@@ -31,6 +32,7 @@ class Execute(State):
         "prepare_simulation_input",
         "print_scenario_info",
         "print_scenario_status",
+        "scenario_id",
     }
 
     def __init__(self, scenario):
@@ -47,8 +49,10 @@ def __init__(self, scenario):
         print("--> Status\n%s" % self._scenario_status)
 
         self._set_ct_and_grid()
+        self._launcher = Context.get_launcher(scenario)
 
-    def _scenario_id(self):
+    @property
+    def scenario_id(self):
         return self._scenario_info["id"]
 
     def _set_ct_and_grid(self):
@@ -81,50 +85,11 @@ def get_grid(self):
 
     def _update_scenario_status(self):
         """Updates scenario status."""
-        self._scenario_status = self._execute_list_manager.get_status(
-            self._scenario_id()
-        )
+        self._scenario_status = self._execute_list_manager.get_status(self.scenario_id)
 
     def _update_scenario_info(self):
         """Updates scenario information."""
-        self._scenario_info = self._scenario_list_manager.get_scenario(
-            self._scenario_id()
-        )
-
-    def _run_script(self, script, extra_args=None):
-        """Returns running process
-
-        :param str script: script to be used.
-        :param list extra_args: list of strings to be passed after scenario id.
-        :return: (*subprocess.Popen*) -- process used to run script
-        """
-
-        if not extra_args:
-            extra_args = []
-
-        path_to_package = posixpath.join(
-            server_setup.MODEL_DIR, self._scenario_info["engine"]
-        )
-
-        if self._scenario_info["engine"] == "REISE":
-            folder = "pyreise"
-        else:
-            folder = "pyreisejl"
-
-        path_to_script = posixpath.join(path_to_package, folder, "utility", script)
-        cmd_pythonpath = [f'export PYTHONPATH="{path_to_package}:$PYTHONPATH";']
-        cmd_pythoncall = [
-            "nohup",
-            "python3",
-            "-u",
-            path_to_script,
-            self._scenario_info["id"],
-        ]
-        cmd_io_redirect = ["</dev/null >/dev/null 2>&1 &"]
-        cmd = cmd_pythonpath + cmd_pythoncall + extra_args + cmd_io_redirect
-        process = self._data_access.execute_command_async(cmd)
-        print("PID: %s" % process.pid)
-        return process
+        self._scenario_info = self._scenario_list_manager.get_scenario(self.scenario_id)
 
     def print_scenario_info(self):
         """Prints scenario information."""
@@ -167,7 +132,7 @@ def prepare_simulation_input(self, profiles_as=None):
 
             si.prepare_mpc_file()
 
-            self._execute_list_manager.set_status(self._scenario_id(), "prepared")
+            self._execute_list_manager.set_status(self.scenario_id, "prepared")
         else:
             print("---------------------------")
             print("SCENARIO CANNOT BE PREPARED")
@@ -187,75 +152,6 @@ def _check_if_ready(self):
                 f"Status must be one of {valid_status}, but got status={self._scenario_status}"
             )
 
-    def _launch_on_server(self, threads=None, solver=None, extract_data=True):
-        """Launch simulation on server, via ssh.
-
-        :param int/None threads: the number of threads to be used. This defaults to None,
-            where None means auto.
-        :param str solver: the solver used for optimization. This defaults to
-            None, which translates to gurobi
-        :param bool extract_data: whether the results of the simulation engine should
-            automatically extracted after the simulation has run. This defaults to True.
-        :raises TypeError: if extract_data is not a boolean
-        :return: (*subprocess.Popen*) -- new process used to launch simulation.
-        """
-        extra_args = []
-
-        if threads:
-            # Use the -t flag as defined in call.py in REISE.jl
-            extra_args.append("--threads " + str(threads))
-
-        if solver:
-            extra_args.append("--solver " + solver)
-
-        if not isinstance(extract_data, bool):
-            raise TypeError("extract_data must be a boolean: 'True' or 'False'")
-        if extract_data:
-            extra_args.append("--extract-data")
-
-        return self._run_script("call.py", extra_args=extra_args)
-
-    def _launch_in_container(self, threads, solver):
-        """Launches simulation in container via http call
-
-        :param int/None threads: the number of threads to be used. This defaults to None,
-            where None means auto.
-        :param str solver: the solver used for optimization. This defaults to
-            None, which translates to gurobi
-        :return: (*requests.Response*) -- the http response object
-        """
-        scenario_id = self._scenario_id()
-        url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
-        resp = requests.post(url, params={"threads": threads, "solver": solver})
-        if resp.status_code != 200:
-            print(
-                f"Failed to launch simulation: status={resp.status_code}. See response for details"
-            )
-        return resp
-
-    def _check_threads(self, threads):
-        """Validate threads argument
-
-        :param int threads: the number of threads to be used
-        :raises TypeError: if threads is not an int
-        :raises ValueError: if threads is not a positive value
-        """
-        if threads:
-            if not isinstance(threads, int):
-                raise TypeError("threads must be an int")
-            if threads < 1:
-                raise ValueError("threads must be a positive value")
-
-    def _check_solver(self, solver):
-        """Validate solver argument
-
-        :param str solver: the solver used for the optimization
-        :raises ValueError: if invalid solver provided
-        """
-        solvers = ("gurobi", "glpk")
-        if solver is not None and solver.lower() not in solvers:
-            raise ValueError(f"Invalid solver: options are {solvers}")
-
     def launch_simulation(self, threads=None, extract_data=True, solver=None):
         """Launches simulation on target environment (server or container)
 
@@ -269,14 +165,10 @@ def launch_simulation(self, threads=None, extract_data=True, solver=None):
             process (if using ssh to server) or http response (if run in container)
         """
         self._check_if_ready()
-        self._check_threads(threads)
-        self._check_solver(solver)
 
         mode = get_deployment_mode()
         print(f"--> Launching simulation on {mode.lower()}")
-        if mode == DeploymentMode.Server:
-            return self._launch_on_server(threads, solver, extract_data)
-        return self._launch_in_container(threads, solver)
+        self._launcher.launch_simulation(threads, extract_data, solver)
 
     def check_progress(self):
         """Get the lastest information from the server container
@@ -287,7 +179,7 @@ def check_progress(self):
         if mode != DeploymentMode.Container:
             raise NotImplementedError("Operation only supported for container mode")
 
-        scenario_id = self._scenario_id()
+        scenario_id = self.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
         resp = requests.get(url)
         return resp.json()

From c5bf5e63f7ff74cf9c6a52072dbeb9315a7b629d Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 4 May 2021 13:57:15 -0700
Subject: [PATCH 26/60] refactor: split up other launcher methods

---
 powersimdata/data_access/launcher.py | 40 ++++++++++++++++++++++++----
 powersimdata/scenario/execute.py     | 25 ++++-------------
 2 files changed, 40 insertions(+), 25 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index bc53acc31..a6e77a137 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -30,8 +30,6 @@ def _check_solver(solver):
         raise ValueError(f"Invalid solver: options are {solvers}")
 
 
-# TODO - check_progress (just print a message for SSHLauncher)
-# TODO - extract_data
 class Launcher:
     def __init__(self, scenario):
         self.scenario = scenario
@@ -39,6 +37,10 @@ def __init__(self, scenario):
     def _launch(self, threads=None, solver=None, extract_data=True):
         raise NotImplementedError
 
+    def extract_simulation_output(self):
+        """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server."""
+        pass
+
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
         _check_threads(threads)
         _check_solver(solver)
@@ -88,12 +90,11 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :return: (*subprocess.Popen*) -- new process used to launch simulation.
         """
         extra_args = []
-
-        if threads:
+        if threads is not None:
             # Use the -t flag as defined in call.py in REISE.jl
             extra_args.append("--threads " + str(threads))
 
-        if solver:
+        if solver is not None:
             extra_args.append("--solver " + solver)
 
         if not isinstance(extract_data, bool):
@@ -103,6 +104,18 @@ def _launch(self, threads=None, solver=None, extract_data=True):
 
         return self._run_script("call.py", extra_args=extra_args)
 
+    def extract_simulation_output(self):
+        """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server.
+
+        :return: (*subprocess.Popen*) -- new process used to extract output
+            data.
+        """
+        print("--> Extracting output data on server")
+        return self._run_script("extract_data.py")
+
+    def check_progress(self):
+        print("Information is available on the server.")
+
 
 class HttpLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
@@ -124,6 +137,16 @@ def _launch(self, threads=None, solver=None, extract_data=True):
             )
         return resp
 
+    def check_progress(self):
+        """Get the status of an ongoing simulation, if possible
+
+        :return: (*dict*) -- json response
+        """
+        scenario_id = self.scenario.scenario_id
+        url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
+        resp = requests.get(url)
+        return resp.json()
+
 
 class NativeLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
@@ -137,3 +160,10 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :return: (*dict*) -- json response
         """
         pass
+
+    def check_progress(self):
+        """Get the status of an ongoing simulation, if possible
+
+        :return: (*dict*) -- json response
+        """
+        pass
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 7248786d0..e9fe4c41a 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -2,8 +2,6 @@
 import os
 import posixpath
 
-import requests
-
 from powersimdata.data_access.context import Context
 from powersimdata.input.case_mat import export_case_mat
 from powersimdata.input.grid import Grid
@@ -12,7 +10,7 @@
 from powersimdata.input.transform_profile import TransformProfile
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
-from powersimdata.utility.config import DeploymentMode, get_deployment_mode
+from powersimdata.utility.config import get_deployment_mode
 
 
 class Execute(State):
@@ -171,18 +169,11 @@ def launch_simulation(self, threads=None, extract_data=True, solver=None):
         self._launcher.launch_simulation(threads, extract_data, solver)
 
     def check_progress(self):
-        """Get the lastest information from the server container
+        """Get the status of an ongoing simulation, if possible
 
-        :raises NotImplementedError: if not running in container mode
+        :return: (*dict*) -- progress information, or None
         """
-        mode = get_deployment_mode()
-        if mode != DeploymentMode.Container:
-            raise NotImplementedError("Operation only supported for container mode")
-
-        scenario_id = self.scenario_id
-        url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
-        resp = requests.get(url)
-        return resp.json()
+        return self._launcher.check_progress()
 
     def extract_simulation_output(self):
         """Extracts simulation outputs {PG, PF, LMP, CONGU, CONGL} on server.
@@ -192,13 +183,7 @@ def extract_simulation_output(self):
         """
         self._update_scenario_status()
         if self._scenario_status == "finished":
-            mode = get_deployment_mode()
-            if mode == DeploymentMode.Container:
-                print("WARNING: extraction not yet supported, please extract manually")
-                return
-
-            print("--> Extracting output data on server")
-            return self._run_script("extract_data.py")
+            return self._launcher.extract_simulation_output()
         else:
             print("---------------------------")
             print("OUTPUTS CANNOT BE EXTRACTED")

From 5a6e4971035e90edcc1b70a3e59f2b417a216ebe Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 5 May 2021 15:12:15 -0700
Subject: [PATCH 27/60] feat: launch simulation natively

---
 powersimdata/data_access/launcher.py | 11 +++++++++--
 powersimdata/scenario/execute.py     |  1 -
 powersimdata/utility/config.py       |  5 +++++
 powersimdata/utility/server_setup.py |  1 +
 4 files changed, 15 insertions(+), 3 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index a6e77a137..68deb080b 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -1,4 +1,5 @@
 import posixpath
+import sys
 
 import requests
 
@@ -159,11 +160,17 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :param bool extract_data: always True
         :return: (*dict*) -- json response
         """
-        pass
+        sys.path.append(server_setup.ENGINE_DIR)
+        from pyreisejl.utility import app
+
+        return app.launch_simulation(self.scenario.scenario_id, threads, solver)
 
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
         :return: (*dict*) -- json response
         """
-        pass
+        sys.path.append(server_setup.ENGINE_DIR)
+        from pyreisejl.utility import app
+
+        return app.check_progress()
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index e9fe4c41a..4cbb50da5 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -1,6 +1,5 @@
 import copy
 import os
-import posixpath
 
 from powersimdata.data_access.context import Context
 from powersimdata.input.case_mat import export_case_mat
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index 498c3df54..b13a96187 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -5,6 +5,10 @@
 
 @dataclass(frozen=True)
 class Config:
+    SERVER_ADDRESS = None
+    SERVER_SSH_PORT = None
+    BACKUP_DATA_ROOT_DIR = None
+    ENGINE_DIR = None
     DATA_ROOT_DIR = "/mnt/bes/pcm"
     EXECUTE_DIR = "tmp"
     INPUT_DIR = ("data", "input")
@@ -28,6 +32,7 @@ class ContainerConfig(Config):
 @dataclass(frozen=True)
 class LocalConfig(Config):
     DATA_ROOT_DIR = Config.LOCAL_DIR
+    ENGINE_DIR = os.getenv("ENGINE_DIR")
 
 
 class DeploymentMode:
diff --git a/powersimdata/utility/server_setup.py b/powersimdata/utility/server_setup.py
index 55d7f84ad..e528e5c2d 100644
--- a/powersimdata/utility/server_setup.py
+++ b/powersimdata/utility/server_setup.py
@@ -12,6 +12,7 @@
 OUTPUT_DIR = config.OUTPUT_DIR
 LOCAL_DIR = config.LOCAL_DIR
 MODEL_DIR = config.MODEL_DIR
+ENGINE_DIR = config.ENGINE_DIR
 
 
 def get_server_user():

From 3db6dff3542c3147b93d1c32cefb25fd9bfffe10 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 6 May 2021 12:15:56 -0700
Subject: [PATCH 28/60] fix: method signature, docstrings

---
 powersimdata/data_access/launcher.py | 14 +++++++++-----
 powersimdata/scenario/execute.py     |  8 ++++----
 powersimdata/utility/config.py       |  4 ++--
 3 files changed, 15 insertions(+), 11 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index 68deb080b..b821957ba 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -45,7 +45,7 @@ def extract_simulation_output(self):
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
         _check_threads(threads)
         _check_solver(solver)
-        self._launch(threads, solver, extract_data)
+        return self._launch(threads, solver, extract_data)
 
 
 class SSHLauncher(Launcher):
@@ -127,7 +127,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: always True
-        :return: (*requests.Response*) -- the http response object
+        :return: (*requests.Response*) -- http response from the engine, with a json
+            body as is returned by check_progress
         """
         scenario_id = self.scenario.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
@@ -141,7 +142,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
-        :return: (*dict*) -- json response
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         scenario_id = self.scenario.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/status/{scenario_id}"
@@ -158,7 +160,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: always True
-        :return: (*dict*) -- json response
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         sys.path.append(server_setup.ENGINE_DIR)
         from pyreisejl.utility import app
@@ -168,7 +171,8 @@ def _launch(self, threads=None, solver=None, extract_data=True):
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
-        :return: (*dict*) -- json response
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         sys.path.append(server_setup.ENGINE_DIR)
         from pyreisejl.utility import app
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 4cbb50da5..6df3b2f04 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -149,15 +149,15 @@ def _check_if_ready(self):
                 f"Status must be one of {valid_status}, but got status={self._scenario_status}"
             )
 
-    def launch_simulation(self, threads=None, extract_data=True, solver=None):
+    def launch_simulation(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment (server or container)
 
         :param int/None threads: the number of threads to be used. This defaults to None,
             where None means auto.
-        :param bool extract_data: whether the results of the simulation engine should
-            automatically extracted after the simulation has run. This defaults to True.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
         :return: (*subprocess.Popen*) or (*requests.Response*) - either the
             process (if using ssh to server) or http response (if run in container)
         """
@@ -165,7 +165,7 @@ def launch_simulation(self, threads=None, extract_data=True, solver=None):
 
         mode = get_deployment_mode()
         print(f"--> Launching simulation on {mode.lower()}")
-        self._launcher.launch_simulation(threads, extract_data, solver)
+        return self._launcher.launch_simulation(threads, solver, extract_data)
 
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index b13a96187..aa014453e 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -8,13 +8,13 @@ class Config:
     SERVER_ADDRESS = None
     SERVER_SSH_PORT = None
     BACKUP_DATA_ROOT_DIR = None
+    MODEL_DIR = None
     ENGINE_DIR = None
     DATA_ROOT_DIR = "/mnt/bes/pcm"
     EXECUTE_DIR = "tmp"
     INPUT_DIR = ("data", "input")
     OUTPUT_DIR = ("data", "output")
     LOCAL_DIR = os.path.join(Path.home(), "ScenarioData", "")
-    MODEL_DIR = "/home/bes/pcm"
 
 
 @dataclass(frozen=True)
@@ -22,6 +22,7 @@ class ServerConfig(Config):
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
     SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
     BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
+    MODEL_DIR = "/home/bes/pcm"
 
 
 @dataclass(frozen=True)
@@ -44,7 +45,6 @@ class DeploymentMode:
 
 
 def get_deployment_mode():
-    # TODO: consider auto detection
     mode = os.getenv("DEPLOYMENT_MODE")
     if mode is None:
         return DeploymentMode.Server

From 1abe2b5aa43da0b35286e00b2da5bd5625a24e36 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 6 May 2021 15:36:57 -0700
Subject: [PATCH 29/60] fix: refresh attributes on state change

---
 powersimdata/scenario/analyze.py  | 10 +++++++---
 powersimdata/scenario/create.py   |  6 +++---
 powersimdata/scenario/delete.py   |  1 -
 powersimdata/scenario/execute.py  | 17 +++++++++--------
 powersimdata/scenario/move.py     |  1 -
 powersimdata/scenario/scenario.py |  3 ++-
 powersimdata/scenario/state.py    | 21 ++++++++++-----------
 7 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/powersimdata/scenario/analyze.py b/powersimdata/scenario/analyze.py
index 950896bf8..e299a80db 100644
--- a/powersimdata/scenario/analyze.py
+++ b/powersimdata/scenario/analyze.py
@@ -39,17 +39,16 @@ class Analyze(State):
         "get_storage_pg",
         "get_wind",
         "print_infeasibilities",
-        "print_scenario_info",
     }
 
     def __init__(self, scenario):
         """Constructor."""
-        self._scenario_info = scenario.info
-        self._scenario_status = scenario.status
         super().__init__(scenario)
 
         self.data_loc = "disk" if scenario.status == "moved" else None
+        self.refresh(scenario)
 
+    def refresh(self, scenario):
         print(
             "SCENARIO: %s | %s\n"
             % (self._scenario_info["plan"], self._scenario_info["name"])
@@ -321,3 +320,8 @@ def get_wind(self):
         """
         profile = TransformProfile(self._scenario_info, self.get_grid(), self.get_ct())
         return profile.get_profile("wind")
+
+    def _leave(self):
+        """Cleans when leaving state."""
+        del self.grid
+        del self.ct
diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py
index bc0b677b4..67bac22a9 100644
--- a/powersimdata/scenario/create.py
+++ b/powersimdata/scenario/create.py
@@ -27,7 +27,6 @@ class Create(State):
     default_exported_methods = (
         "create_scenario",
         "get_bus_demand",
-        "print_scenario_info",
         "set_builder",
         "set_grid",
     )
@@ -37,8 +36,6 @@ def __init__(self, scenario):
         self.builder = None
         self.grid = None
         self.ct = None
-        self._scenario_status = None
-        self._scenario_info = scenario.info
         self.exported_methods = set(self.default_exported_methods)
         super().__init__(scenario)
 
@@ -176,6 +173,9 @@ def set_grid(self, grid_model="usa_tamu", interconnect="USA"):
         self._scenario_info["grid_model"] = self.builder.grid_model
         self._scenario_info["interconnect"] = self.builder.interconnect
 
+    def _leave(self):
+        del self.builder
+
 
 class _Builder(object):
     """Scenario Builder.
diff --git a/powersimdata/scenario/delete.py b/powersimdata/scenario/delete.py
index 7a9587a0e..db80e6dab 100644
--- a/powersimdata/scenario/delete.py
+++ b/powersimdata/scenario/delete.py
@@ -10,7 +10,6 @@ class Delete(State):
     allowed = []
     exported_methods = {
         "delete_scenario",
-        "print_scenario_info",
     }
 
     def print_scenario_info(self):
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 6df3b2f04..b9bf0b46a 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -27,17 +27,20 @@ class Execute(State):
         "get_grid",
         "launch_simulation",
         "prepare_simulation_input",
-        "print_scenario_info",
         "print_scenario_status",
         "scenario_id",
     }
 
     def __init__(self, scenario):
         """Constructor."""
-        self._scenario_info = scenario.info
-        self._scenario_status = scenario.status
         super().__init__(scenario)
+        self.refresh(scenario)
 
+    @property
+    def scenario_id(self):
+        return self._scenario_info["id"]
+
+    def refresh(self, scenario):
         print(
             "SCENARIO: %s | %s\n"
             % (self._scenario_info["plan"], self._scenario_info["name"])
@@ -48,10 +51,6 @@ def __init__(self, scenario):
         self._set_ct_and_grid()
         self._launcher = Context.get_launcher(scenario)
 
-    @property
-    def scenario_id(self):
-        return self._scenario_info["id"]
-
     def _set_ct_and_grid(self):
         """Sets change table and grid."""
         base_grid = Grid(
@@ -129,7 +128,9 @@ def prepare_simulation_input(self, profiles_as=None):
 
             si.prepare_mpc_file()
 
-            self._execute_list_manager.set_status(self.scenario_id, "prepared")
+            prepared = "prepared"
+            self._execute_list_manager.set_status(self.scenario_id, prepared)
+            self._scenario_status = prepared
         else:
             print("---------------------------")
             print("SCENARIO CANNOT BE PREPARED")
diff --git a/powersimdata/scenario/move.py b/powersimdata/scenario/move.py
index d811842ba..374ec1542 100644
--- a/powersimdata/scenario/move.py
+++ b/powersimdata/scenario/move.py
@@ -11,7 +11,6 @@ class Move(State):
     allowed = []
     exported_methods = {
         "move_scenario",
-        "print_scenario_info",
     }
 
     def print_scenario_info(self):
diff --git a/powersimdata/scenario/scenario.py b/powersimdata/scenario/scenario.py
index 63e804b5e..fdc27b27b 100644
--- a/powersimdata/scenario/scenario.py
+++ b/powersimdata/scenario/scenario.py
@@ -57,6 +57,7 @@ def __init__(self, descriptor=None):
 
         if not descriptor:
             self.info = OrderedDict(self._default_info)
+            self.status = None
             self.state = Create(self)
         else:
             self._set_info(descriptor)
@@ -68,7 +69,7 @@ def __init__(self, descriptor=None):
                 elif state == "analyze":
                     self.state = Analyze(self)
             except AttributeError:
-                return
+                pass
 
     def __getattr__(self, name):
         if name in self.state.exported_methods:
diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py
index 5bfcff997..944103f16 100644
--- a/powersimdata/scenario/state.py
+++ b/powersimdata/scenario/state.py
@@ -1,7 +1,3 @@
-from powersimdata.data_access.execute_list import ExecuteListManager
-from powersimdata.data_access.scenario_list import ScenarioListManager
-
-
 class State(object):
     """Defines an interface for encapsulating the behavior associated with a
         particular state of the Scenario object.
@@ -18,9 +14,15 @@ def __init__(self, scenario):
         if type(self) == State:
             raise TypeError("Only subclasses of 'State' can be instantiated directly")
 
+        self._scenario = scenario
+        self._scenario_info = scenario.info
+        self._scenario_status = scenario.status
         self._data_access = scenario.data_access
-        self._scenario_list_manager = ScenarioListManager(self._data_access)
-        self._execute_list_manager = ExecuteListManager(self._data_access)
+        self._scenario_list_manager = scenario._scenario_list_manager
+        self._execute_list_manager = scenario._execute_list_manager
+
+    def refresh(self, scenario):
+        pass
 
     def switch(self, state):
         """Switches state.
@@ -33,6 +35,7 @@ def switch(self, state):
             self._leave()
             self.__class__ = state
             self._enter(state)
+            self.refresh(self._scenario)
         else:
             raise Exception(
                 "State switching: %s --> %s not permitted" % (self, state.name)
@@ -47,11 +50,7 @@ def __str__(self):
 
     def _leave(self):
         """Cleans when leaving state."""
-        if self.name == "create":
-            del self.builder
-        elif self.name == "analyze":
-            del self.grid
-            del self.ct
+        pass
 
     def _enter(self, state):
         """Initializes when entering state."""

From 179e9ca68658318d82e5e9f6b9bf8c6f09234cae Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 7 May 2021 15:28:23 -0700
Subject: [PATCH 30/60] chore: address PR comments

---
 powersimdata/data_access/context.py           |  5 +++-
 powersimdata/data_access/data_access.py       | 10 ++++----
 powersimdata/data_access/launcher.py          | 24 ++++++++++++++++++-
 .../data_access/tests/test_data_access.py     |  4 ++--
 powersimdata/scenario/analyze.py              |  4 +++-
 powersimdata/scenario/execute.py              |  3 ++-
 powersimdata/utility/config.py                |  1 -
 7 files changed, 40 insertions(+), 11 deletions(-)

diff --git a/powersimdata/data_access/context.py b/powersimdata/data_access/context.py
index afb7caa38..88ad74d47 100644
--- a/powersimdata/data_access/context.py
+++ b/powersimdata/data_access/context.py
@@ -14,6 +14,8 @@ def get_data_access(data_loc=None):
 
         :param str data_loc: pass "disk" if using data from backup disk,
             otherwise leave the default.
+        :return: (:class:`powersimdata.data_access.data_access.DataAccess`) -- a data access
+            instance
         """
         if data_loc == "disk":
             root = server_setup.BACKUP_DATA_ROOT_DIR
@@ -29,7 +31,8 @@ def get_data_access(data_loc=None):
     def get_launcher(scenario):
         """Return instance for interaction with simulation engine
 
-        :param powersimdata.Scenario scenario: a scenario object
+        :param powersimdata.scenario.scenario.Scenario scenario: a scenario object
+        :return: (:class:`powersimdata.data_access.launcher.Launcher`) -- a launcher instance
         """
         mode = get_deployment_mode()
         if mode == DeploymentMode.Server:
diff --git a/powersimdata/data_access/data_access.py b/powersimdata/data_access/data_access.py
index 0d900b7ca..be854f14d 100644
--- a/powersimdata/data_access/data_access.py
+++ b/powersimdata/data_access/data_access.py
@@ -24,10 +24,12 @@
 class DataAccess:
     """Interface to a local or remote data store."""
 
-    def __init__(self, root=None):
+    def __init__(self, root=None, backup_root=None):
         """Constructor"""
         self.root = server_setup.DATA_ROOT_DIR if root is None else root
-        self.backup_root = server_setup.BACKUP_DATA_ROOT_DIR
+        self.backup_root = (
+            server_setup.BACKUP_DATA_ROOT_DIR if backup_root is None else backup_root
+        )
         self.join = None
 
     def copy_from(self, file_name, from_dir):
@@ -291,9 +293,9 @@ class SSHDataAccess(DataAccess):
 
     _last_attempt = 0
 
-    def __init__(self, root=None):
+    def __init__(self, root=None, backup_root=None):
         """Constructor"""
-        super().__init__(root)
+        super().__init__(root, backup_root)
         self._ssh = None
         self._retry_after = 5
         self.local_root = server_setup.LOCAL_DIR
diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index b821957ba..e2645afb2 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -36,6 +36,16 @@ def __init__(self, scenario):
         self.scenario = scenario
 
     def _launch(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation on target environment
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
+        :raises NotImplementedError: always - this must be implemented in a subclass
+        """
         raise NotImplementedError
 
     def extract_simulation_output(self):
@@ -43,6 +53,18 @@ def extract_simulation_output(self):
         pass
 
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
+        """Launches simulation on target environment
+
+        :param int/None threads: the number of threads to be used. This defaults to None,
+            where None means auto.
+        :param str solver: the solver used for optimization. This defaults to
+            None, which translates to gurobi
+        :param bool extract_data: whether the results of the simulation engine should
+            automatically extracted after the simulation has run. This defaults to True.
+        :return: (*subprocess.Popen*) or (*requests.Response*) - either the
+            process (if using ssh to server) or http response (if run in container)
+            or (*dict*) (if run locally)
+        """
         _check_threads(threads)
         _check_solver(solver)
         return self._launch(threads, solver, extract_data)
@@ -99,7 +121,7 @@ def _launch(self, threads=None, solver=None, extract_data=True):
             extra_args.append("--solver " + solver)
 
         if not isinstance(extract_data, bool):
-            raise TypeError("extract_data must be a boolean: 'True' or 'False'")
+            raise TypeError("extract_data must be a bool")
         if extract_data:
             extra_args.append("--extract-data")
 
diff --git a/powersimdata/data_access/tests/test_data_access.py b/powersimdata/data_access/tests/test_data_access.py
index 7a2027c32..f29bcbe02 100644
--- a/powersimdata/data_access/tests/test_data_access.py
+++ b/powersimdata/data_access/tests/test_data_access.py
@@ -10,11 +10,12 @@
 from powersimdata.utility import server_setup
 
 CONTENT = b"content"
+backup_root = "/mnt/backup_dir"
 
 
 @pytest.fixture
 def data_access():
-    data_access = SSHDataAccess()
+    data_access = SSHDataAccess(backup_root=backup_root)
     yield data_access
     data_access.close()
 
@@ -68,7 +69,6 @@ def _check_content(filepath):
 
 
 root_dir = server_setup.DATA_ROOT_DIR.rstrip("/")
-backup_root = server_setup.BACKUP_DATA_ROOT_DIR
 
 
 def test_base_dir(data_access):
diff --git a/powersimdata/scenario/analyze.py b/powersimdata/scenario/analyze.py
index e299a80db..c29268d0e 100644
--- a/powersimdata/scenario/analyze.py
+++ b/powersimdata/scenario/analyze.py
@@ -61,7 +61,9 @@ def refresh(self, scenario):
     def _set_allowed_state(self):
         """Sets allowed state."""
         if self._scenario_status == "extracted":
-            self.allowed = ["delete", "move"]
+            self.allowed = ["delete"]
+            if self._data_access.backup_root is not None:
+                self.allowed.append("move")
 
     def _set_ct_and_grid(self):
         """Sets change table and grid."""
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index b9bf0b46a..a56add4f7 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -151,7 +151,7 @@ def _check_if_ready(self):
             )
 
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
-        """Launches simulation on target environment (server or container)
+        """Launches simulation on target environment
 
         :param int/None threads: the number of threads to be used. This defaults to None,
             where None means auto.
@@ -161,6 +161,7 @@ def launch_simulation(self, threads=None, solver=None, extract_data=True):
             automatically extracted after the simulation has run. This defaults to True.
         :return: (*subprocess.Popen*) or (*requests.Response*) - either the
             process (if using ssh to server) or http response (if run in container)
+            or (*dict*) (if run locally)
         """
         self._check_if_ready()
 
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index aa014453e..20de76fb0 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -21,7 +21,6 @@ class Config:
 class ServerConfig(Config):
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
     SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
-    BACKUP_DATA_ROOT_DIR = "/mnt/RE-Storage/v2"
     MODEL_DIR = "/home/bes/pcm"
 
 

From 3f04222105351c81d88ec23e5c0a4b3388168484 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Wed, 12 May 2021 18:17:22 -0700
Subject: [PATCH 31/60] feat, docs: provision local directory and fix
 docstrings

---
 powersimdata/data_access/launcher.py |  5 ++++
 powersimdata/scenario/create.py      |  1 +
 powersimdata/scenario/execute.py     |  8 ++++++
 powersimdata/scenario/state.py       |  6 ++++-
 powersimdata/utility/config.py       | 38 ++++++++++++++++++++++++++++
 5 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index e2645afb2..59b2e8a2c 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -32,6 +32,11 @@ def _check_solver(solver):
 
 
 class Launcher:
+    """Base class for interaction with simulation engine.
+
+    :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
+    """
+
     def __init__(self, scenario):
         self.scenario = scenario
 
diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py
index 67bac22a9..a730638b6 100644
--- a/powersimdata/scenario/create.py
+++ b/powersimdata/scenario/create.py
@@ -174,6 +174,7 @@ def set_grid(self, grid_model="usa_tamu", interconnect="USA"):
         self._scenario_info["interconnect"] = self.builder.interconnect
 
     def _leave(self):
+        """Cleans when leaving state."""
         del self.builder
 
 
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index a56add4f7..1019b677f 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -38,9 +38,17 @@ def __init__(self, scenario):
 
     @property
     def scenario_id(self):
+        """Get the current scenario id
+
+        :return: (*str*) -- scenario id
+        """
         return self._scenario_info["id"]
 
     def refresh(self, scenario):
+        """Called during state changes to ensure instance is properly initialized
+
+        :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
+        """
         print(
             "SCENARIO: %s | %s\n"
             % (self._scenario_info["plan"], self._scenario_info["name"])
diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py
index 944103f16..cfd3d8e72 100644
--- a/powersimdata/scenario/state.py
+++ b/powersimdata/scenario/state.py
@@ -1,6 +1,6 @@
 class State(object):
     """Defines an interface for encapsulating the behavior associated with a
-        particular state of the Scenario object.
+    particular state of the Scenario object.
 
     :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
     :raise TypeError: if not instantiated through a derived class
@@ -22,6 +22,10 @@ def __init__(self, scenario):
         self._execute_list_manager = scenario._execute_list_manager
 
     def refresh(self, scenario):
+        """Called during state changes to ensure instance is properly initialized
+
+        :param powrsimdata.scenario.scenario.Scenario scenario: scenario instance
+        """
         pass
 
     def switch(self, state):
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index 20de76fb0..2f7a0f6e7 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -1,10 +1,17 @@
 import os
+import shutil
 from dataclasses import dataclass
 from pathlib import Path
 
+from powersimdata.utility import templates
+
 
 @dataclass(frozen=True)
 class Config:
+    """Base class for configuration data. It should contain all expected keys,
+    defaulting to None when not universally applicable.
+    """
+
     SERVER_ADDRESS = None
     SERVER_SSH_PORT = None
     BACKUP_DATA_ROOT_DIR = None
@@ -19,6 +26,8 @@ class Config:
 
 @dataclass(frozen=True)
 class ServerConfig(Config):
+    """Values specific to internal client/server usage"""
+
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "becompute01.gatesventures.com")
     SERVER_SSH_PORT = os.getenv("BE_SERVER_SSH_PORT", 22)
     MODEL_DIR = "/home/bes/pcm"
@@ -26,16 +35,37 @@ class ServerConfig(Config):
 
 @dataclass(frozen=True)
 class ContainerConfig(Config):
+    """Values specific to containerized environment"""
+
     SERVER_ADDRESS = os.getenv("BE_SERVER_ADDRESS", "reisejl")
 
 
 @dataclass(frozen=True)
 class LocalConfig(Config):
+    """Values specific to native installation"""
+
     DATA_ROOT_DIR = Config.LOCAL_DIR
     ENGINE_DIR = os.getenv("ENGINE_DIR")
 
+    def initialize(self):
+        """Create data directory with blank templates"""
+        confirmed = input(
+            f"Provision directory {self.LOCAL_DIR}? [y/n] (default is 'n')"
+        )
+        if confirmed.lower() != "y":
+            print("Operation cancelled.")
+            return
+        os.makedirs(self.LOCAL_DIR, exist_ok=True)
+        for fname in ("ScenarioList.csv", "ExecuteList.csv"):
+            orig = os.path.join(templates.__path__[0], fname)
+            dest = os.path.join(self.LOCAL_DIR, fname)
+            shutil.copy(orig, dest)
+        print("--> Done!")
+
 
 class DeploymentMode:
+    """Constants representing the type of installation being used"""
+
     Server = "SERVER"
     Container = "CONTAINER"
     Local = "LOCAL"
@@ -44,6 +74,10 @@ class DeploymentMode:
 
 
 def get_deployment_mode():
+    """Get the deployment mode used to determine various configuration values
+
+    :return: (*str*) -- the deployment mode
+    """
     mode = os.getenv("DEPLOYMENT_MODE")
     if mode is None:
         return DeploymentMode.Server
@@ -54,5 +88,9 @@ def get_deployment_mode():
 
 
 def get_config():
+    """Get a config instance based on the DEPLOYMENT_MODE environment variable
+
+    :return: (*powersimdata.utility.config.Config*) -- a config instance
+    """
     mode = get_deployment_mode()
     return DeploymentMode.CONFIG_MAP[mode]()

From 086022783d7aeb79d3f8fa0ec4426746058118ed Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 13 May 2021 11:59:14 -0700
Subject: [PATCH 32/60] feat: configure root dir using config file

---
 .dockerignore                  | 4 ++++
 .gitignore                     | 1 +
 powersimdata/utility/config.py | 8 ++++++++
 3 files changed, 13 insertions(+)

diff --git a/.dockerignore b/.dockerignore
index bf5e8d691..192d301d8 100644
--- a/.dockerignore
+++ b/.dockerignore
@@ -7,3 +7,7 @@ build
 **/__pycache__
 .ipynb_checkpoints
 **/.ropeproject
+.env
+.venv
+.dockerignore
+config.ini
diff --git a/.gitignore b/.gitignore
index cbe3bcd07..01ba1505f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 # This is specific to this package
 powersimdata/utility/.server_user
+config.ini
 
 # The remainder of this file taken from github/gitignore
 # https://github.com/github/gitignore/blob/master/Python.gitignore
diff --git a/powersimdata/utility/config.py b/powersimdata/utility/config.py
index 2f7a0f6e7..9912d30b6 100644
--- a/powersimdata/utility/config.py
+++ b/powersimdata/utility/config.py
@@ -1,3 +1,4 @@
+import configparser
 import os
 import shutil
 from dataclasses import dataclass
@@ -5,6 +6,13 @@
 
 from powersimdata.utility import templates
 
+INI_FILE = "config.ini"
+if Path(INI_FILE).exists():
+    conf = configparser.ConfigParser()
+    conf.read(INI_FILE)
+    for k, v in conf["PowerSimData"].items():
+        os.environ[k.upper()] = v
+
 
 @dataclass(frozen=True)
 class Config:

From c84cacbdd8451c16b4952ee87e8a2262664ce06e Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 14 May 2021 10:54:15 -0700
Subject: [PATCH 33/60] fix: print statements mentioning the server

---
 powersimdata/data_access/execute_list.py  | 8 ++++----
 powersimdata/data_access/scenario_list.py | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/powersimdata/data_access/execute_list.py b/powersimdata/data_access/execute_list.py
index 1e70fbd88..5fa5c9b5d 100644
--- a/powersimdata/data_access/execute_list.py
+++ b/powersimdata/data_access/execute_list.py
@@ -18,7 +18,7 @@ def get_status(self, scenario_id):
         """Return the status for the scenario
 
         :param str/int scenario_id: the scenario id
-        :raises Exception: if scenario not found in execute list on server.
+        :raises Exception: if scenario not found in execute list.
         :return: (*str*) -- scenario status
         """
         table = self.get_execute_table()
@@ -46,12 +46,12 @@ def set_status(self, scenario_id, status):
         table = self.get_execute_table()
         table.loc[int(scenario_id), "status"] = status
 
-        print(f"-->  Setting status={status} in execute table on server")
+        print(f"--> Setting status={status} in execute list")
         return table
 
     @verify_hash
     def delete_entry(self, scenario_id):
-        """Deletes entry from execute list on server.
+        """Deletes entry from execute list.
 
         :param int/str scenario_id: the id of the scenario
         :return: (*pandas.DataFrame*) -- the updated data frame
@@ -59,5 +59,5 @@ def delete_entry(self, scenario_id):
         table = self.get_execute_table()
         table.drop(int(scenario_id), inplace=True)
 
-        print("--> Deleting entry in execute table on server")
+        print("--> Deleting entry in %s" % self._FILE_NAME)
         return table
diff --git a/powersimdata/data_access/scenario_list.py b/powersimdata/data_access/scenario_list.py
index 084817712..d67bcef07 100644
--- a/powersimdata/data_access/scenario_list.py
+++ b/powersimdata/data_access/scenario_list.py
@@ -64,7 +64,7 @@ def err_message(text):
 
     @verify_hash
     def add_entry(self, scenario_info):
-        """Adds scenario to the scenario list file on server.
+        """Adds scenario to the scenario list file.
 
         :param collections.OrderedDict scenario_info: entry to add to scenario list.
         :return: (*pandas.DataFrame*) -- the updated data frame
@@ -78,7 +78,7 @@ def add_entry(self, scenario_info):
         table = table.append(entry)
         table.set_index("id", inplace=True)
 
-        print("--> Adding entry in %s on server" % self._FILE_NAME)
+        print("--> Adding entry in %s" % self._FILE_NAME)
         return table
 
     @verify_hash
@@ -91,5 +91,5 @@ def delete_entry(self, scenario_id):
         table = self.get_scenario_table()
         table.drop(int(scenario_id), inplace=True)
 
-        print("--> Deleting entry in %s on server" % self._FILE_NAME)
+        print("--> Deleting entry in %s" % self._FILE_NAME)
         return table

From 27c0c560f81a3ee41533fd3dbd63d7a1e009c1d7 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Tue, 18 May 2021 15:44:22 -0700
Subject: [PATCH 34/60] chore: improve docstrings and consolidate return type
 for non ssh launchers

---
 powersimdata/data_access/launcher.py | 30 +++++++++++++++-------------
 powersimdata/scenario/execute.py     |  9 +++++----
 2 files changed, 21 insertions(+), 18 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index 59b2e8a2c..10a423d26 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -13,7 +13,7 @@ def _check_threads(threads):
     :raises TypeError: if threads is not an int
     :raises ValueError: if threads is not a positive value
     """
-    if threads:
+    if threads is not None:
         if not isinstance(threads, int):
             raise TypeError("threads must be an int")
         if threads < 1:
@@ -24,8 +24,11 @@ def _check_solver(solver):
     """Validate solver argument
 
     :param str solver: the solver used for the optimization
+    :raises TypeError: if solver is not a str
     :raises ValueError: if invalid solver provided
     """
+    if not isinstance(solver, str):
+        raise TypeError("solver must be a str")
     solvers = ("gurobi", "glpk")
     if solver is not None and solver.lower() not in solvers:
         raise ValueError(f"Invalid solver: options are {solvers}")
@@ -43,7 +46,7 @@ def __init__(self, scenario):
     def _launch(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
@@ -60,15 +63,14 @@ def extract_simulation_output(self):
     def launch_simulation(self, threads=None, solver=None, extract_data=True):
         """Launches simulation on target environment
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: whether the results of the simulation engine should
             automatically extracted after the simulation has run. This defaults to True.
-        :return: (*subprocess.Popen*) or (*requests.Response*) - either the
-            process (if using ssh to server) or http response (if run in container)
-            or (*dict*) (if run locally)
+        :return: (*subprocess.Popen*) or (*dict*) - the process, if using ssh to server,
+            otherwise a dict containing status information.
         """
         _check_threads(threads)
         _check_solver(solver)
@@ -108,7 +110,7 @@ def _run_script(self, script, extra_args=None):
     def _launch(self, threads=None, solver=None, extract_data=True):
         """Launch simulation on server, via ssh.
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
@@ -147,15 +149,15 @@ def check_progress(self):
 
 class HttpLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
-        """Launches simulation in container via http call
+        """Launch simulation in container via http call
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
         :param bool extract_data: always True
-        :return: (*requests.Response*) -- http response from the engine, with a json
-            body as is returned by check_progress
+        :return: (*dict*) -- contains "output", "errors", "scenario_id", and "status"
+            keys which map to stdout, stderr, and the respective scenario attributes
         """
         scenario_id = self.scenario.scenario_id
         url = f"http://{server_setup.SERVER_ADDRESS}:5000/launch/{scenario_id}"
@@ -164,7 +166,7 @@ def _launch(self, threads=None, solver=None, extract_data=True):
             print(
                 f"Failed to launch simulation: status={resp.status_code}. See response for details"
             )
-        return resp
+        return resp.json()
 
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
@@ -180,9 +182,9 @@ def check_progress(self):
 
 class NativeLauncher(Launcher):
     def _launch(self, threads=None, solver=None, extract_data=True):
-        """Launches simulation by importing from REISE.jl
+        """Launch simulation by importing from REISE.jl
 
-        :param int/None threads: the number of threads to be used. This defaults to None,
+        :param int threads: the number of threads to be used. This defaults to None,
             where None means auto.
         :param str solver: the solver used for optimization. This defaults to
             None, which translates to gurobi
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 1019b677f..ae3d68280 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -167,9 +167,8 @@ def launch_simulation(self, threads=None, solver=None, extract_data=True):
             None, which translates to gurobi
         :param bool extract_data: whether the results of the simulation engine should
             automatically extracted after the simulation has run. This defaults to True.
-        :return: (*subprocess.Popen*) or (*requests.Response*) - either the
-            process (if using ssh to server) or http response (if run in container)
-            or (*dict*) (if run locally)
+        :return: (*subprocess.Popen*) or (*dict*) - the process, if using ssh to server,
+            otherwise a dict containing status information.
         """
         self._check_if_ready()
 
@@ -180,7 +179,9 @@ def launch_simulation(self, threads=None, solver=None, extract_data=True):
     def check_progress(self):
         """Get the status of an ongoing simulation, if possible
 
-        :return: (*dict*) -- progress information, or None
+        :return: (*dict*) -- either None if using ssh, or a dict which contains
+            "output", "errors", "scenario_id", and "status" keys which map to
+            stdout, stderr, and the respective scenario attributes
         """
         return self._launcher.check_progress()
 

From ca09a925733e9c5285b08c3720e99ed558042f32 Mon Sep 17 00:00:00 2001
From: jon-hagg <66005238+jon-hagg@users.noreply.github.com>
Date: Wed, 19 May 2021 16:58:27 -0700
Subject: [PATCH 35/60] fix: path join for profiles_as (#483)

---
 powersimdata/scenario/execute.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index ae3d68280..9aa7f3965 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -251,7 +251,7 @@ def prepare_profile(self, kind, profile_as=None):
         """Prepares profile for simulation.
 
         :param kind: one of *demand*, *'hydro'*, *'solar'* or *'wind'*.
-        :param int/str/None profile_as: if given, copy profile from this scenario.
+        :param int/str profile_as: if given, copy profile from this scenario.
         """
         if profile_as is None:
             tp = TransformProfile(self._scenario_info, self.grid, self.ct)
@@ -267,5 +267,5 @@ def prepare_profile(self, kind, profile_as=None):
             )
         else:
             from_dir = self._data_access.match_scenario_files(profile_as, "tmp")
-            to_dir = self.TMP_DIR
-            self._data_access.copy(f"{from_dir}/{kind}.csv", to_dir)
+            src = self._data_access.join(from_dir, f"{kind}.csv")
+            self._data_access.copy(src, self.TMP_DIR)

From d656b60a0e0c973ce3becef9807014aa0e55ecad Mon Sep 17 00:00:00 2001
From: danielolsen <danielolsen@users.noreply.github.com>
Date: Thu, 20 May 2021 15:24:02 -0700
Subject: [PATCH 36/60] fix: create copy of bus table before modifying (#484)

---
 powersimdata/input/input_data.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/powersimdata/input/input_data.py b/powersimdata/input/input_data.py
index 7fa64954d..5f6bbbc36 100644
--- a/powersimdata/input/input_data.py
+++ b/powersimdata/input/input_data.py
@@ -149,7 +149,7 @@ def get_bus_demand(scenario_info, grid):
     :param powersimdata.input.grid.Grid grid: grid to construct bus demand for.
     :return: (*pandas.DataFrame*) -- data frame of demand.
     """
-    bus = grid.bus
+    bus = grid.bus.copy()
     demand = InputData().get_data(scenario_info, "demand")[bus.zone_id.unique()]
     bus["zone_Pd"] = bus.groupby("zone_id")["Pd"].transform("sum")
     bus["zone_share"] = bus["Pd"] / bus["zone_Pd"]

From 934b5367d3244d90a9110d8d15109d76e62beacb Mon Sep 17 00:00:00 2001
From: danielolsen <danielolsen@users.noreply.github.com>
Date: Fri, 21 May 2021 10:27:23 -0700
Subject: [PATCH 37/60] doc: update docstrings for input_data and
 profile_helper (#487)

* doc: fix docstring for scenario_info in InputHelper

* doc: fix docstring for scenario_info in ProfileHelper

* doc: add periods to docstrings within input_data

* doc: add periods to docstrings within profile_helper
---
 powersimdata/data_access/profile_helper.py | 22 +++++++++++-----------
 powersimdata/input/input_data.py           | 18 +++++++++---------
 2 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/powersimdata/data_access/profile_helper.py b/powersimdata/data_access/profile_helper.py
index 5554a76df..15abfd7be 100644
--- a/powersimdata/data_access/profile_helper.py
+++ b/powersimdata/data_access/profile_helper.py
@@ -15,9 +15,9 @@ def get_file_components(scenario_info, field_name):
         """Get the file name and relative path for the given profile and
         scenario.
 
-        :param dict scenario_info: a ScenarioInfo instance
-        :param str field_name: the kind of profile
-        :return: (*tuple*) -- file name and list of path components
+        :param dict scenario_info: metadata for a scenario.
+        :param str field_name: the kind of profile.
+        :return: (*tuple*) -- file name and list of path components.
         """
         version = scenario_info["base_" + field_name]
         file_name = field_name + "_" + version + ".csv"
@@ -26,11 +26,11 @@ def get_file_components(scenario_info, field_name):
 
     @staticmethod
     def download_file(file_name, from_dir):
-        """Download the profile from blob storage at the given path
+        """Download the profile from blob storage at the given path.
 
-        :param str file_name: profile csv
-        :param tuple from_dir: tuple of path components
-        :return: (*str*) -- path to downloaded file
+        :param str file_name: profile csv.
+        :param tuple from_dir: tuple of path components.
+        :return: (*str*) -- path to downloaded file.
         """
         print(f"--> Downloading {file_name} from blob storage.")
         url_path = "/".join(from_dir)
@@ -55,11 +55,11 @@ def download_file(file_name, from_dir):
 
     @staticmethod
     def parse_version(grid_model, kind, version):
-        """Parse available versions from the given spec
+        """Parse available versions from the given spec.
 
         :param str grid_model: grid model.
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
-        :param dict version: version information per grid model
+        :param dict version: version information per grid model.
         :return: (*list*) -- available profile version.
         """
         if grid_model in version and kind in version[grid_model]:
@@ -69,7 +69,7 @@ def parse_version(grid_model, kind, version):
 
     @staticmethod
     def get_profile_version_cloud(grid_model, kind):
-        """Returns available raw profile from blob storage
+        """Returns available raw profile from blob storage.
 
         :param str grid_model: grid model.
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
@@ -81,7 +81,7 @@ def get_profile_version_cloud(grid_model, kind):
 
     @staticmethod
     def get_profile_version_local(grid_model, kind):
-        """Returns available raw profile from local file
+        """Returns available raw profile from local file.
 
         :param str grid_model: grid model.
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.
diff --git a/powersimdata/input/input_data.py b/powersimdata/input/input_data.py
index 5f6bbbc36..070723da9 100644
--- a/powersimdata/input/input_data.py
+++ b/powersimdata/input/input_data.py
@@ -24,21 +24,21 @@ def __init__(self, data_access):
 
     @staticmethod
     def get_file_components(scenario_info, field_name):
-        """Get the file name and relative path for either ct or grid
+        """Get the file name and relative path for either ct or grid.
 
-        :param dict scenario_info: a ScenarioInfo instance
-        :param str field_name: the input file type
-        :return: (*tuple*) -- file name and list of path components
+        :param dict scenario_info: metadata for a scenario.
+        :param str field_name: the input file type.
+        :return: (*tuple*) -- file name and list of path components.
         """
         ext = _file_extension[field_name]
         file_name = scenario_info["id"] + "_" + field_name + "." + ext
         return file_name, server_setup.INPUT_DIR
 
     def download_file(self, file_name, from_dir):
-        """Download the file if using server, otherwise no-op
+        """Download the file if using server, otherwise no-op.
 
-        :param str file_name: either grid or ct file name
-        :param tuple from_dir: tuple of path components
+        :param str file_name: either grid or ct file name.
+        :param tuple from_dir: tuple of path components.
         """
         from_dir = self.data_access.join(*from_dir)
         self.data_access.copy_from(file_name, from_dir)
@@ -50,7 +50,7 @@ def _check_field(field_name):
     :param str field_name: *'demand'*, *'hydro'*, *'solar'*, *'wind'*,
         *'ct'* or *'grid'*.
     :raises ValueError: if not *'demand'*, *'hydro'*, *'solar'*, *'wind'*
-        *'ct'* or *'grid'*
+        *'ct'* or *'grid'*.
     """
     possible = list(_file_extension.keys())
     if field_name not in possible:
@@ -108,7 +108,7 @@ def get_data(self, scenario_info, field_name):
         return data
 
     def get_profile_version(self, grid_model, kind):
-        """Returns available raw profile from blob storage or local disk
+        """Returns available raw profile from blob storage or local disk.
 
         :param str grid_model: grid model.
         :param str kind: *'demand'*, *'hydro'*, *'solar'* or *'wind'*.

From 26d0929aedf72997a9c590e23c980681c5938ae8 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 20 May 2021 17:13:34 -0700
Subject: [PATCH 38/60] refactor: separate logic to save profiles locally

---
 powersimdata/input/transform_profile.py | 16 ++++++++++++++++
 powersimdata/scenario/execute.py        | 12 +++++-------
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/powersimdata/input/transform_profile.py b/powersimdata/input/transform_profile.py
index 1e7826d89..38a9532c4 100644
--- a/powersimdata/input/transform_profile.py
+++ b/powersimdata/input/transform_profile.py
@@ -3,6 +3,22 @@
 from powersimdata.input.input_data import InputData
 
 
+def export_scaled_profile(kind, scenario_info, grid, ct, filepath):
+    """Apply transformation to the given kind of profile and save the result locally.
+
+    :param dict scenario_info: a dict containing the profile version, with
+        key in the form base_{kind}
+    :param powersimdata.input.grid.Grid grid: a Grid object previously
+        transformed.
+    :param dict ct: change table.
+    :param str filepath: path to save the result, including the filename
+    """
+    tp = TransformProfile(scenario_info, grid, ct)
+    profile = tp.get_profile(kind)
+    print(f"Writing scaled {kind} profile to {filepath} on local machine")
+    profile.to_csv(filepath)
+
+
 class TransformProfile(object):
     """Transform profile according to operations listed in change table."""
 
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 9aa7f3965..115ce9c00 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -6,7 +6,7 @@
 from powersimdata.input.grid import Grid
 from powersimdata.input.input_data import InputData
 from powersimdata.input.transform_grid import TransformGrid
-from powersimdata.input.transform_profile import TransformProfile
+from powersimdata.input.transform_profile import export_scaled_profile
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
 from powersimdata.utility.config import get_deployment_mode
@@ -254,13 +254,11 @@ def prepare_profile(self, kind, profile_as=None):
         :param int/str profile_as: if given, copy profile from this scenario.
         """
         if profile_as is None:
-            tp = TransformProfile(self._scenario_info, self.grid, self.ct)
-            profile = tp.get_profile(kind)
-            print(
-                f"Writing scaled {kind} profile in {server_setup.LOCAL_DIR} on local machine"
-            )
             file_name = "%s_%s.csv" % (self.scenario_id, kind)
-            profile.to_csv(os.path.join(server_setup.LOCAL_DIR, file_name))
+            filepath = os.path.join(server_setup.LOCAL_DIR, file_name)
+            export_scaled_profile(
+                kind, self._scenario_info, self.grid, self.ct, filepath
+            )
 
             self._data_access.move_to(
                 file_name, self.REL_TMP_DIR, change_name_to=f"{kind}.csv"

From 13b4357a900b0ee33a58d0821c571397aeed473d Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Thu, 20 May 2021 17:25:57 -0700
Subject: [PATCH 39/60] chore: no need to inherit from object

---
 powersimdata/input/abstract_grid.py     | 2 +-
 powersimdata/input/change_table.py      | 2 +-
 powersimdata/input/grid.py              | 2 +-
 powersimdata/input/input_data.py        | 2 +-
 powersimdata/input/transform_grid.py    | 2 +-
 powersimdata/input/transform_profile.py | 2 +-
 powersimdata/network/csv_reader.py      | 2 +-
 powersimdata/output/output_data.py      | 2 +-
 powersimdata/scenario/create.py         | 2 +-
 powersimdata/scenario/execute.py        | 2 +-
 powersimdata/scenario/move.py           | 2 +-
 powersimdata/scenario/scenario.py       | 2 +-
 powersimdata/scenario/state.py          | 2 +-
 powersimdata/tests/mock_grid.py         | 2 +-
 powersimdata/utility/helpers.py         | 2 +-
 15 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/powersimdata/input/abstract_grid.py b/powersimdata/input/abstract_grid.py
index 76197ac5b..4ca10998b 100644
--- a/powersimdata/input/abstract_grid.py
+++ b/powersimdata/input/abstract_grid.py
@@ -1,7 +1,7 @@
 import pandas as pd
 
 
-class AbstractGrid(object):
+class AbstractGrid:
     """Grid Builder."""
 
     def __init__(self):
diff --git a/powersimdata/input/change_table.py b/powersimdata/input/change_table.py
index 69d0b4e3b..eb7dddb4c 100644
--- a/powersimdata/input/change_table.py
+++ b/powersimdata/input/change_table.py
@@ -37,7 +37,7 @@ def ordinal(n):
     return str(n + 1) + ord_dict.get((n + 1) if (n + 1) < 20 else (n + 1) % 10, "th")
 
 
-class ChangeTable(object):
+class ChangeTable:
     """Create change table for changes that need to be applied to the original
     grid as well as to the original demand, hydro, solar and wind profiles.
     A pickle file enclosing the change table in form of a dictionary can be
diff --git a/powersimdata/input/grid.py b/powersimdata/input/grid.py
index 008d3b373..61cb69c59 100644
--- a/powersimdata/input/grid.py
+++ b/powersimdata/input/grid.py
@@ -11,7 +11,7 @@
 _cache = MemoryCache()
 
 
-class Grid(object):
+class Grid:
     """Grid
 
     :param str/list interconnect: geographical region covered. Either *'USA'*, one of
diff --git a/powersimdata/input/input_data.py b/powersimdata/input/input_data.py
index 070723da9..9d74e8757 100644
--- a/powersimdata/input/input_data.py
+++ b/powersimdata/input/input_data.py
@@ -57,7 +57,7 @@ def _check_field(field_name):
         raise ValueError("Only %s data can be loaded" % " | ".join(possible))
 
 
-class InputData(object):
+class InputData:
     """Load input data.
 
     :param str data_loc: data location.
diff --git a/powersimdata/input/transform_grid.py b/powersimdata/input/transform_grid.py
index cb3bd6151..004886dd7 100644
--- a/powersimdata/input/transform_grid.py
+++ b/powersimdata/input/transform_grid.py
@@ -6,7 +6,7 @@
 from powersimdata.utility.distance import haversine
 
 
-class TransformGrid(object):
+class TransformGrid:
     """Transforms grid according to operations listed in change table."""
 
     def __init__(self, grid, ct):
diff --git a/powersimdata/input/transform_profile.py b/powersimdata/input/transform_profile.py
index 38a9532c4..e67cc917b 100644
--- a/powersimdata/input/transform_profile.py
+++ b/powersimdata/input/transform_profile.py
@@ -19,7 +19,7 @@ def export_scaled_profile(kind, scenario_info, grid, ct, filepath):
     profile.to_csv(filepath)
 
 
-class TransformProfile(object):
+class TransformProfile:
     """Transform profile according to operations listed in change table."""
 
     def __init__(self, scenario_info, grid, ct):
diff --git a/powersimdata/network/csv_reader.py b/powersimdata/network/csv_reader.py
index 9fc5408cf..bc4bd81ed 100644
--- a/powersimdata/network/csv_reader.py
+++ b/powersimdata/network/csv_reader.py
@@ -1,7 +1,7 @@
 from powersimdata.input.helpers import csv_to_data_frame
 
 
-class CSVReader(object):
+class CSVReader:
     """MPC files reader.
 
     :param str data_loc: path to data.
diff --git a/powersimdata/output/output_data.py b/powersimdata/output/output_data.py
index c835f8c10..43bcaa154 100644
--- a/powersimdata/output/output_data.py
+++ b/powersimdata/output/output_data.py
@@ -10,7 +10,7 @@
 from powersimdata.utility import server_setup
 
 
-class OutputData(object):
+class OutputData:
     """Load output data.
 
     :param str data_loc: data location.
diff --git a/powersimdata/scenario/create.py b/powersimdata/scenario/create.py
index a730638b6..9da58b433 100644
--- a/powersimdata/scenario/create.py
+++ b/powersimdata/scenario/create.py
@@ -178,7 +178,7 @@ def _leave(self):
         del self.builder
 
 
-class _Builder(object):
+class _Builder:
     """Scenario Builder.
 
     :param str grid_model: grid model.
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 115ce9c00..833180445 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -202,7 +202,7 @@ def extract_simulation_output(self):
             return
 
 
-class SimulationInput(object):
+class SimulationInput:
     """Prepares scenario for execution.
 
     :param powersimdata.data_access.data_access.DataAccess data_access:
diff --git a/powersimdata/scenario/move.py b/powersimdata/scenario/move.py
index 374ec1542..870d25b6d 100644
--- a/powersimdata/scenario/move.py
+++ b/powersimdata/scenario/move.py
@@ -52,7 +52,7 @@ def _clean(self):
         self._data_access.close()
 
 
-class BackUpDisk(object):
+class BackUpDisk:
     """Back up scenario data to backup disk mounted on server.
 
     :param powersimdata.data_access.data_access.DataAccess data_access:
diff --git a/powersimdata/scenario/scenario.py b/powersimdata/scenario/scenario.py
index fdc27b27b..d0837f1ac 100644
--- a/powersimdata/scenario/scenario.py
+++ b/powersimdata/scenario/scenario.py
@@ -12,7 +12,7 @@
 pd.set_option("display.max_colwidth", None)
 
 
-class Scenario(object):
+class Scenario:
     """Handles scenario.
 
     :param int/str descriptor: scenario name or index. If None, default to a Scenario
diff --git a/powersimdata/scenario/state.py b/powersimdata/scenario/state.py
index cfd3d8e72..edf28cf6e 100644
--- a/powersimdata/scenario/state.py
+++ b/powersimdata/scenario/state.py
@@ -1,4 +1,4 @@
-class State(object):
+class State:
     """Defines an interface for encapsulating the behavior associated with a
     particular state of the Scenario object.
 
diff --git a/powersimdata/tests/mock_grid.py b/powersimdata/tests/mock_grid.py
index 1d56a7066..713cffccb 100644
--- a/powersimdata/tests/mock_grid.py
+++ b/powersimdata/tests/mock_grid.py
@@ -170,7 +170,7 @@
 }
 
 
-class MockGrid(object):
+class MockGrid:
     def __init__(self, grid_attrs=None, model="usa_tamu"):
         """Constructor.
 
diff --git a/powersimdata/utility/helpers.py b/powersimdata/utility/helpers.py
index 13c450277..33d0a830c 100644
--- a/powersimdata/utility/helpers.py
+++ b/powersimdata/utility/helpers.py
@@ -111,7 +111,7 @@ def _build(self, arg):
         raise ValueError(f"unsupported type for cache key = {type(arg)}")
 
 
-class PrintManager(object):
+class PrintManager:
     """Manages print messages."""
 
     def __init__(self):

From 8dd72ea79889a26197bb370dd6b3739eba62c6d8 Mon Sep 17 00:00:00 2001
From: jon-hagg <66005238+jon-hagg@users.noreply.github.com>
Date: Mon, 24 May 2021 16:41:27 -0700
Subject: [PATCH 40/60] chore: update blob storage account for profiles (#488)

---
 powersimdata/data_access/profile_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/powersimdata/data_access/profile_helper.py b/powersimdata/data_access/profile_helper.py
index 15abfd7be..d077f3c9f 100644
--- a/powersimdata/data_access/profile_helper.py
+++ b/powersimdata/data_access/profile_helper.py
@@ -8,7 +8,7 @@
 
 
 class ProfileHelper:
-    BASE_URL = "https://bescienceswebsite.blob.core.windows.net/profiles"
+    BASE_URL = "https://besciences.blob.core.windows.net/profiles"
 
     @staticmethod
     def get_file_components(scenario_info, field_name):

From 7267d0f2c11c69405dce57b121f8cd2057b8617d Mon Sep 17 00:00:00 2001
From: jon-hagg <66005238+jon-hagg@users.noreply.github.com>
Date: Wed, 26 May 2021 11:22:29 -0700
Subject: [PATCH 41/60] test: include transform profile tests by default (#485)

---
 .../input/tests/test_transform_profile.py     | 84 +++++--------------
 powersimdata/scenario/tests/test_create.py    |  1 +
 powersimdata/scenario/tests/test_scenario.py  |  1 +
 3 files changed, 21 insertions(+), 65 deletions(-)

diff --git a/powersimdata/input/tests/test_transform_profile.py b/powersimdata/input/tests/test_transform_profile.py
index 7e6b6c65f..954a356de 100644
--- a/powersimdata/input/tests/test_transform_profile.py
+++ b/powersimdata/input/tests/test_transform_profile.py
@@ -197,49 +197,39 @@ def base_grid():
     return grid
 
 
-@pytest.fixture(scope="module")
-def raw_hydro():
+def raw_profile(kind):
     input_data = InputData()
+    grid_model = "test_usa_tamu"
     profile_info = {
-        "grid_model": "usa_tamu",
-        "base_hydro": param["hydro"],
+        "grid_model": grid_model,
+        f"base_{kind}": param[kind],
     }
-    profile = input_data.get_data(profile_info, "hydro")
+    profile = input_data.get_data(profile_info, kind)
     return profile_info, profile
 
 
+@pytest.fixture(scope="module")
+def raw_hydro():
+    return raw_profile("hydro")
+
+
 @pytest.fixture(scope="module")
 def raw_wind():
-    input_data = InputData()
-    profile_info = {
-        "grid_model": "usa_tamu",
-        "base_wind": param["wind"],
-    }
-    profile = input_data.get_data(profile_info, "wind")
-    return profile_info, profile
+    return raw_profile("wind")
 
 
 @pytest.fixture(scope="module")
 def raw_solar():
-    input_data = InputData()
-    profile_info = {
-        "grid_model": "usa_tamu",
-        "base_solar": param["solar"],
-    }
-    profile = input_data.get_data(profile_info, "solar")
-    return profile_info, profile
+    return raw_profile("solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
-def test_demand_is_scaled(base_grid):
-    input_data = InputData()
-    demand_info = {
-        "interconnect": "_".join(interconnect),
-        "grid_model": "usa_tamu",
-        "base_demand": param["demand"],
-    }
-    raw_demand = input_data.get_data(demand_info, "demand")
+@pytest.fixture(scope="module")
+def raw_demand():
+    return raw_profile("demand")
+
+
+def test_demand_is_scaled(base_grid, raw_demand):
+    demand_info, raw_demand = raw_demand
     base_demand = raw_demand[base_grid.id2zone.keys()]
 
     n_zone = param["n_zone_to_scale"]
@@ -273,22 +263,16 @@ def test_demand_is_scaled(base_grid):
         assert transformed_profile[unscaled_zone].equals(base_demand[unscaled_zone])
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_solar_is_scaled_by_zone(base_grid, raw_solar):
     ct = get_change_table_for_zone_scaling(base_grid, "solar")
     _check_plants_are_scaled(ct, base_grid, *raw_solar, "solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_solar_is_scaled_by_id(base_grid, raw_solar):
     ct = get_change_table_for_id_scaling(base_grid, "solar")
     _check_plants_are_scaled(ct, base_grid, *raw_solar, "solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_solar_is_scaled_by_zone_and_id(base_grid, raw_solar):
     ct_zone = get_change_table_for_zone_scaling(base_grid, "solar")
     ct_id = get_change_table_for_id_scaling(base_grid, "solar")
@@ -296,22 +280,16 @@ def test_solar_is_scaled_by_zone_and_id(base_grid, raw_solar):
     _check_plants_are_scaled(ct, base_grid, *raw_solar, "solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_wind_is_scaled_by_zone(base_grid, raw_wind):
     ct = get_change_table_for_zone_scaling(base_grid, "wind")
     _check_plants_are_scaled(ct, base_grid, *raw_wind, "wind")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_wind_is_scaled_by_id(base_grid, raw_wind):
     ct = get_change_table_for_id_scaling(base_grid, "wind")
     _check_plants_are_scaled(ct, base_grid, *raw_wind, "wind")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_wind_is_scaled_by_zone_and_id(base_grid, raw_wind):
     ct_zone = get_change_table_for_zone_scaling(base_grid, "wind")
     ct_id = get_change_table_for_id_scaling(base_grid, "wind")
@@ -319,22 +297,16 @@ def test_wind_is_scaled_by_zone_and_id(base_grid, raw_wind):
     _check_plants_are_scaled(ct, base_grid, *raw_wind, "wind")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_hydro_is_scaled_by_zone(base_grid, raw_hydro):
     ct = get_change_table_for_zone_scaling(base_grid, "hydro")
     _check_plants_are_scaled(ct, base_grid, *raw_hydro, "hydro")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_hydro_is_scaled_by_id(base_grid, raw_hydro):
     ct = get_change_table_for_id_scaling(base_grid, "hydro")
     _check_plants_are_scaled(ct, base_grid, *raw_hydro, "hydro")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_hydro_is_scaled_by_zone_and_id(base_grid, raw_hydro):
     ct_zone = get_change_table_for_zone_scaling(base_grid, "hydro")
     ct_id = get_change_table_for_id_scaling(base_grid, "hydro")
@@ -342,58 +314,40 @@ def test_hydro_is_scaled_by_zone_and_id(base_grid, raw_hydro):
     _check_plants_are_scaled(ct, base_grid, *raw_hydro, "hydro")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_solar_are_added(base_grid, raw_solar):
     ct = get_change_table_for_new_plant_addition(base_grid, "solar")
     _ = _check_new_plants_are_added(ct, base_grid, *raw_solar, "solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_wind_are_added(base_grid, raw_wind):
     ct = get_change_table_for_new_plant_addition(base_grid, "wind")
     _ = _check_new_plants_are_added(ct, base_grid, *raw_wind, "wind")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_hydro_added(base_grid, raw_hydro):
     ct = get_change_table_for_new_plant_addition(base_grid, "hydro")
     _ = _check_new_plants_are_added(ct, base_grid, *raw_hydro, "hydro")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_solar_are_not_scaled(base_grid, raw_solar):
     _check_new_plants_are_not_scaled(base_grid, *raw_solar, "solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_wind_are_not_scaled(base_grid, raw_wind):
     _check_new_plants_are_not_scaled(base_grid, *raw_wind, "wind")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_hydro_are_not_scaled(base_grid, raw_hydro):
     _check_new_plants_are_not_scaled(base_grid, *raw_hydro, "hydro")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_solar_profile(base_grid, raw_solar):
     _check_profile_of_new_plants_are_produced_correctly(base_grid, *raw_solar, "solar")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_wind_profile(base_grid, raw_wind):
     _check_profile_of_new_plants_are_produced_correctly(base_grid, *raw_wind, "wind")
 
 
-@pytest.mark.integration
-@pytest.mark.ssh
 def test_new_hydro_profile(base_grid, raw_hydro):
     _check_profile_of_new_plants_are_produced_correctly(base_grid, *raw_hydro, "hydro")
diff --git a/powersimdata/scenario/tests/test_create.py b/powersimdata/scenario/tests/test_create.py
index fbb4a8bf7..37cd274e3 100644
--- a/powersimdata/scenario/tests/test_create.py
+++ b/powersimdata/scenario/tests/test_create.py
@@ -3,6 +3,7 @@
 from powersimdata.scenario.scenario import Scenario
 
 
+@pytest.mark.integration
 @pytest.mark.ssh
 def test_get_bus_demand():
     scenario = Scenario("")
diff --git a/powersimdata/scenario/tests/test_scenario.py b/powersimdata/scenario/tests/test_scenario.py
index 393b590ee..1949f926c 100644
--- a/powersimdata/scenario/tests/test_scenario.py
+++ b/powersimdata/scenario/tests/test_scenario.py
@@ -3,6 +3,7 @@
 from powersimdata.scenario.scenario import Scenario
 
 
+@pytest.mark.integration
 @pytest.mark.ssh
 def test_bad_scenario_name():
     # This test will fail if we do add a scenario with this name

From 67f1cd9692297e8ce0e69f13104f6dabf73fd524 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 28 May 2021 11:19:12 -0700
Subject: [PATCH 42/60] fix: skip type checks if None

---
 powersimdata/data_access/launcher.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/powersimdata/data_access/launcher.py b/powersimdata/data_access/launcher.py
index 10a423d26..e86e003e2 100644
--- a/powersimdata/data_access/launcher.py
+++ b/powersimdata/data_access/launcher.py
@@ -13,11 +13,12 @@ def _check_threads(threads):
     :raises TypeError: if threads is not an int
     :raises ValueError: if threads is not a positive value
     """
-    if threads is not None:
-        if not isinstance(threads, int):
-            raise TypeError("threads must be an int")
-        if threads < 1:
-            raise ValueError("threads must be a positive value")
+    if threads is None:
+        return
+    if not isinstance(threads, int):
+        raise TypeError("threads must be an int")
+    if threads < 1:
+        raise ValueError("threads must be a positive value")
 
 
 def _check_solver(solver):
@@ -27,10 +28,12 @@ def _check_solver(solver):
     :raises TypeError: if solver is not a str
     :raises ValueError: if invalid solver provided
     """
+    if solver is None:
+        return
     if not isinstance(solver, str):
         raise TypeError("solver must be a str")
     solvers = ("gurobi", "glpk")
-    if solver is not None and solver.lower() not in solvers:
+    if solver.lower() not in solvers:
         raise ValueError(f"Invalid solver: options are {solvers}")
 
 

From d16d1eb635fd0d2565a4b5b4dc33cd703bd345a2 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Fri, 28 May 2021 11:33:20 -0700
Subject: [PATCH 43/60] test: add tests for launcher validation

---
 .../data_access/tests/test_launcher.py        | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 powersimdata/data_access/tests/test_launcher.py

diff --git a/powersimdata/data_access/tests/test_launcher.py b/powersimdata/data_access/tests/test_launcher.py
new file mode 100644
index 000000000..3694efaf1
--- /dev/null
+++ b/powersimdata/data_access/tests/test_launcher.py
@@ -0,0 +1,23 @@
+import pytest
+
+from powersimdata.data_access.launcher import _check_solver, _check_threads
+
+
+def test_check_solver():
+    _check_solver(None)
+    _check_solver("gurobi")
+    _check_solver("GLPK")
+    with pytest.raises(TypeError):
+        _check_solver(123)
+    with pytest.raises(ValueError):
+        _check_solver("not-a-real-solver")
+
+
+def test_check_threads():
+    _check_threads(None)
+    _check_threads(1)
+    _check_threads(8)
+    with pytest.raises(TypeError):
+        _check_threads("4")
+    with pytest.raises(ValueError):
+        _check_threads(0)

From 6c4bb05bedef6e107b5352d6ad2b9e032d0eed43 Mon Sep 17 00:00:00 2001
From: Ben RdO <ben.rouilledorfeuil@breakthroughenergy.org>
Date: Fri, 28 May 2021 14:35:00 -0700
Subject: [PATCH 44/60] chore: rename module and move function (#490)

---
 .../input/{case_mat.py => export_data.py}      | 18 ++++++++++++++++++
 powersimdata/input/transform_profile.py        | 16 ----------------
 powersimdata/scenario/execute.py               |  5 ++---
 3 files changed, 20 insertions(+), 19 deletions(-)
 rename powersimdata/input/{case_mat.py => export_data.py} (83%)

diff --git a/powersimdata/input/case_mat.py b/powersimdata/input/export_data.py
similarity index 83%
rename from powersimdata/input/case_mat.py
rename to powersimdata/input/export_data.py
index 0d264d728..c2d59b84e 100644
--- a/powersimdata/input/case_mat.py
+++ b/powersimdata/input/export_data.py
@@ -3,6 +3,8 @@
 import numpy as np
 from scipy.io import savemat
 
+from powersimdata.input.transform_profile import TransformProfile
+
 
 def export_case_mat(grid, filepath, storage_filepath=None):
     """Export a grid to a format suitable for loading into simulation engine.
@@ -120,3 +122,19 @@ def export_case_mat(grid, filepath, storage_filepath=None):
         savemat(storage_filepath, mpc_storage, appendmat=False)
 
     savemat(filepath, mpc, appendmat=False)
+
+
+def export_transformed_profile(kind, scenario_info, grid, ct, filepath):
+    """Apply transformation to the given kind of profile and save the result locally.
+
+    :param dict scenario_info: a dict containing the profile version, with
+        key in the form base_{kind}
+    :param powersimdata.input.grid.Grid grid: a Grid object previously
+        transformed.
+    :param dict ct: change table.
+    :param str filepath: path to save the result, including the filename
+    """
+    tp = TransformProfile(scenario_info, grid, ct)
+    profile = tp.get_profile(kind)
+    print(f"Writing scaled {kind} profile to {filepath} on local machine")
+    profile.to_csv(filepath)
diff --git a/powersimdata/input/transform_profile.py b/powersimdata/input/transform_profile.py
index e67cc917b..fbc09a752 100644
--- a/powersimdata/input/transform_profile.py
+++ b/powersimdata/input/transform_profile.py
@@ -3,22 +3,6 @@
 from powersimdata.input.input_data import InputData
 
 
-def export_scaled_profile(kind, scenario_info, grid, ct, filepath):
-    """Apply transformation to the given kind of profile and save the result locally.
-
-    :param dict scenario_info: a dict containing the profile version, with
-        key in the form base_{kind}
-    :param powersimdata.input.grid.Grid grid: a Grid object previously
-        transformed.
-    :param dict ct: change table.
-    :param str filepath: path to save the result, including the filename
-    """
-    tp = TransformProfile(scenario_info, grid, ct)
-    profile = tp.get_profile(kind)
-    print(f"Writing scaled {kind} profile to {filepath} on local machine")
-    profile.to_csv(filepath)
-
-
 class TransformProfile:
     """Transform profile according to operations listed in change table."""
 
diff --git a/powersimdata/scenario/execute.py b/powersimdata/scenario/execute.py
index 833180445..47d530079 100644
--- a/powersimdata/scenario/execute.py
+++ b/powersimdata/scenario/execute.py
@@ -2,11 +2,10 @@
 import os
 
 from powersimdata.data_access.context import Context
-from powersimdata.input.case_mat import export_case_mat
+from powersimdata.input.export_data import export_case_mat, export_transformed_profile
 from powersimdata.input.grid import Grid
 from powersimdata.input.input_data import InputData
 from powersimdata.input.transform_grid import TransformGrid
-from powersimdata.input.transform_profile import export_scaled_profile
 from powersimdata.scenario.state import State
 from powersimdata.utility import server_setup
 from powersimdata.utility.config import get_deployment_mode
@@ -256,7 +255,7 @@ def prepare_profile(self, kind, profile_as=None):
         if profile_as is None:
             file_name = "%s_%s.csv" % (self.scenario_id, kind)
             filepath = os.path.join(server_setup.LOCAL_DIR, file_name)
-            export_scaled_profile(
+            export_transformed_profile(
                 kind, self._scenario_info, self.grid, self.ct, filepath
             )
 

From 0c6896a3fd3f4ef9287008e8fde7eaca2274de6f Mon Sep 17 00:00:00 2001
From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com>
Date: Wed, 2 Jun 2021 08:37:14 -0700
Subject: [PATCH 45/60] chore(deps): bump urllib3 from 1.26.4 to 1.26.5 (#493)

Bumps [urllib3](https://github.com/urllib3/urllib3) from 1.26.4 to 1.26.5.
- [Release notes](https://github.com/urllib3/urllib3/releases)
- [Changelog](https://github.com/urllib3/urllib3/blob/main/CHANGES.rst)
- [Commits](https://github.com/urllib3/urllib3/compare/1.26.4...1.26.5)

---
updated-dependencies:
- dependency-name: urllib3
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
---
 Pipfile.lock | 59 +++++++++++++++++++++++++---------------------------
 1 file changed, 28 insertions(+), 31 deletions(-)

diff --git a/Pipfile.lock b/Pipfile.lock
index f590f9fa6..c5a34e335 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -24,27 +24,35 @@
                 "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1",
                 "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"
             ],
-            "markers": "python_version >= '3.6'",
             "version": "==3.2.0"
         },
         "certifi": {
             "hashes": [
-                "sha256:1a4995114262bffbc2413b159f2a1a480c969de6e6eb13ee966d470af86af59c",
-                "sha256:719a74fb9e33b9bd44cc7f3a8d94bc35e4049deebe19ba7d8e108280cfd59830"
+                "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
+                "sha256:50b1e4f8446b06f41be7dd6338db18e0990601dce795c2b1686458aa7e8fa7d8"
             ],
-            "version": "==2020.12.5"
+            "version": "==2021.5.30"
         },
         "cffi": {
             "hashes": [
                 "sha256:005a36f41773e148deac64b08f233873a4d0c18b053d37da83f6af4d9087b813",
+                "sha256:04c468b622ed31d408fea2346bec5bbffba2cc44226302a0de1ade9f5ea3d373",
+                "sha256:06d7cd1abac2ffd92e65c0609661866709b4b2d82dd15f611e602b9b188b0b69",
+                "sha256:06db6321b7a68b2bd6df96d08a5adadc1fa0e8f419226e25b2a5fbf6ccc7350f",
                 "sha256:0857f0ae312d855239a55c81ef453ee8fd24136eaba8e87a2eceba644c0d4c06",
+                "sha256:0f861a89e0043afec2a51fd177a567005847973be86f709bbb044d7f42fc4e05",
                 "sha256:1071534bbbf8cbb31b498d5d9db0f274f2f7a865adca4ae429e147ba40f73dea",
                 "sha256:158d0d15119b4b7ff6b926536763dc0714313aa59e320ddf787502c70c4d4bee",
+                "sha256:1bf1ac1984eaa7675ca8d5745a8cb87ef7abecb5592178406e55858d411eadc0",
                 "sha256:1f436816fc868b098b0d63b8920de7d208c90a67212546d02f84fe78a9c26396",
+                "sha256:24a570cd11895b60829e941f2613a4f79df1a27344cbbb82164ef2e0116f09c7",
+                "sha256:24ec4ff2c5c0c8f9c6b87d5bb53555bf267e1e6f70e52e5a9740d32861d36b6f",
                 "sha256:2894f2df484ff56d717bead0a5c2abb6b9d2bf26d6960c4604d5c48bbc30ee73",
                 "sha256:29314480e958fd8aab22e4a58b355b629c59bf5f2ac2492b61e3dc06d8c7a315",
+                "sha256:293e7ea41280cb28c6fcaaa0b1aa1f533b8ce060b9e701d78511e1e6c4a1de76",
                 "sha256:34eff4b97f3d982fb93e2831e6750127d1355a923ebaeeb565407b3d2f8d41a1",
                 "sha256:35f27e6eb43380fa080dccf676dece30bef72e4a67617ffda586641cd4508d49",
+                "sha256:3c3f39fa737542161d8b0d680df2ec249334cd70a8f420f71c9304bd83c3cbed",
                 "sha256:3d3dd4c9e559eb172ecf00a2a7517e97d1e96de2a5e610bd9b68cea3925b4892",
                 "sha256:43e0b9d9e2c9e5d152946b9c5fe062c151614b262fda2e7b201204de0b99e482",
                 "sha256:48e1c69bbacfc3d932221851b39d49e81567a4d4aac3b21258d9c24578280058",
@@ -52,6 +60,7 @@
                 "sha256:58e3f59d583d413809d60779492342801d6e82fefb89c86a38e040c16883be53",
                 "sha256:5de7970188bb46b7bf9858eb6890aad302577a5f6f75091fd7cdd3ef13ef3045",
                 "sha256:65fa59693c62cf06e45ddbb822165394a288edce9e276647f0046e1ec26920f3",
+                "sha256:681d07b0d1e3c462dd15585ef5e33cb021321588bebd910124ef4f4fb71aef55",
                 "sha256:69e395c24fc60aad6bb4fa7e583698ea6cc684648e1ffb7fe85e3c1ca131a7d5",
                 "sha256:6c97d7350133666fbb5cf4abdc1178c812cb205dc6f41d174a7b0f18fb93337e",
                 "sha256:6e4714cc64f474e4d6e37cfff31a814b509a35cb17de4fb1999907575684479c",
@@ -69,8 +78,10 @@
                 "sha256:b85eb46a81787c50650f2392b9b4ef23e1f126313b9e0e9013b35c15e4288e2e",
                 "sha256:bb89f306e5da99f4d922728ddcd6f7fcebb3241fc40edebcb7284d7514741991",
                 "sha256:cbde590d4faaa07c72bf979734738f328d239913ba3e043b1e98fe9a39f8b2b6",
+                "sha256:cc5a8e069b9ebfa22e26d0e6b97d6f9781302fe7f4f2b8776c3e1daea35f1adc",
                 "sha256:cd2868886d547469123fadc46eac7ea5253ea7fcb139f12e1dfc2bbd406427d1",
                 "sha256:d42b11d692e11b6634f7613ad8df5d6d5f8875f5d48939520d351007b3c13406",
+                "sha256:df5052c5d867c1ea0b311fb7c3cd28b19df469c056f7fdcfe88c7473aa63e333",
                 "sha256:f2d45f97ab6bb54753eab54fffe75aaf3de4ff2341c9daee1987ee1837636f1d",
                 "sha256:fd78e5fee591709f32ef6edb9a015b4aa1a5022598e36227500c8f4e02328d9c"
             ],
@@ -81,7 +92,6 @@
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
                 "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.0.0"
         },
         "cryptography": {
@@ -99,7 +109,6 @@
                 "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
                 "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
             ],
-            "markers": "python_version >= '3.6'",
             "version": "==3.4.7"
         },
         "idna": {
@@ -107,7 +116,6 @@
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
         "numpy": {
@@ -196,7 +204,6 @@
                 "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
                 "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.20"
         },
         "pynacl": {
@@ -220,7 +227,6 @@
                 "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff",
                 "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.4.0"
         },
         "python-dateutil": {
@@ -228,7 +234,6 @@
                 "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
                 "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.8.1"
         },
         "pytz": {
@@ -273,11 +278,10 @@
         },
         "six": {
             "hashes": [
-                "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259",
-                "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced"
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==1.15.0"
+            "version": "==1.16.0"
         },
         "tqdm": {
             "hashes": [
@@ -289,11 +293,11 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:2f4da4594db7e1e110a944bb1b551fdf4e6c136ad42e4234131391e21eb5b0df",
-                "sha256:e7b021f7241115872f92f43c6508082facffbd1c048e3c6e2bb9c2a157e28937"
+                "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c",
+                "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
-            "version": "==1.26.4"
+            "index": "pypi",
+            "version": "==1.26.5"
         }
     },
     "develop": {
@@ -306,11 +310,10 @@
         },
         "attrs": {
             "hashes": [
-                "sha256:31b2eced602aa8423c2aea9c76a724617ed67cf9513173fd3a4f03e3a929c7e6",
-                "sha256:832aa3cde19744e49938b91fea06d69ecb9e649c93ba974535d08ad92164f700"
+                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
+                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==20.3.0"
+            "version": "==21.2.0"
         },
         "black": {
             "hashes": [
@@ -322,11 +325,10 @@
         },
         "click": {
             "hashes": [
-                "sha256:d2b5255c7c6349bc1bd1e59e08cd12acbbd63ce649f2588755783aa94dfb6b1a",
-                "sha256:dacca89f4bfadd5de3d7489b7c8a566eee0d3676333fbb50030263894c38c0dc"
+                "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a",
+                "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
-            "version": "==7.1.2"
+            "version": "==8.0.1"
         },
         "coverage": {
             "hashes": [
@@ -405,7 +407,6 @@
                 "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
                 "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==20.9"
         },
         "pathspec": {
@@ -420,7 +421,6 @@
                 "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
                 "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.13.1"
         },
         "py": {
@@ -428,7 +428,6 @@
                 "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
                 "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.10.0"
         },
         "pyparsing": {
@@ -436,7 +435,6 @@
                 "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
                 "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.4.7"
         },
         "pytest": {
@@ -506,7 +504,6 @@
                 "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                 "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
             ],
-            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==0.10.2"
         }
     }

From 4413c1283c6d3af3bcb72ecb4cf60a70f314b874 Mon Sep 17 00:00:00 2001
From: jon-hagg <66005238+jon-hagg@users.noreply.github.com>
Date: Thu, 3 Jun 2021 13:52:45 -0700
Subject: [PATCH 46/60] refactor: set grid_model property on grid (#494)

* refactor: set grid_model property on grid

* fix: update more references to get_grid_model

* refactor: reuse plant_id list comprehension
---
 .../generation/clean_capacity_scaling.py      |  2 +-
 powersimdata/design/generation/cost_curves.py |  6 +--
 .../generation/tests/test_cost_curves.py      |  1 -
 powersimdata/design/scenario_info.py          |  2 +-
 .../design/tests/test_scenario_info.py        | 46 ++++++-------------
 .../design/transmission/tests/test_upgrade.py |  1 -
 powersimdata/design/transmission/upgrade.py   |  3 +-
 powersimdata/input/grid.py                    | 12 ++---
 powersimdata/tests/mock_grid.py               |  1 +
 powersimdata/tests/mock_scenario_info.py      |  1 -
 10 files changed, 26 insertions(+), 49 deletions(-)

diff --git a/powersimdata/design/generation/clean_capacity_scaling.py b/powersimdata/design/generation/clean_capacity_scaling.py
index 7aed7031f..a65142fd7 100644
--- a/powersimdata/design/generation/clean_capacity_scaling.py
+++ b/powersimdata/design/generation/clean_capacity_scaling.py
@@ -101,7 +101,7 @@ def _make_zonename2target(grid, targets):
     :raises ValueError: if a zone is not present in any target areas, or
         if a zone is present in more than one target area.
     """
-    grid_model = grid.get_grid_model()
+    grid_model = grid.grid_model
     target_zones = {
         target_name: area_to_loadzone(grid_model, target_name)
         if pd.isnull(targets.loc[target_name, "area_type"])
diff --git a/powersimdata/design/generation/cost_curves.py b/powersimdata/design/generation/cost_curves.py
index 9ff257017..e4b35d3a6 100644
--- a/powersimdata/design/generation/cost_curves.py
+++ b/powersimdata/design/generation/cost_curves.py
@@ -228,7 +228,7 @@ def build_supply_curve(grid, num_segments, area, gen_type, area_type=None, plot=
         raise ValueError(f"{gen_type} is not a valid generation type.")
 
     # Identify the load zones that correspond to the specified area and area_type
-    returned_zones = area_to_loadzone(grid.get_grid_model(), area, area_type)
+    returned_zones = area_to_loadzone(grid.grid_model, area, area_type)
 
     # Trim the DataFrame to only be of the desired area and generation type
     supply_data = supply_data.loc[supply_data.zone_name.isin(returned_zones)]
@@ -467,7 +467,7 @@ def plot_linear_vs_quadratic_terms(
         raise ValueError(f"{gen_type} is not a valid generation type.")
 
     # Identify the load zones that correspond to the specified area and area_type
-    returned_zones = area_to_loadzone(grid.get_grid_model(), area, area_type)
+    returned_zones = area_to_loadzone(grid.grid_model, area, area_type)
 
     # Trim the DataFrame to only be of the desired area and generation type
     supply_data = supply_data.loc[supply_data.zone_name.isin(returned_zones)]
@@ -574,7 +574,7 @@ def plot_capacity_vs_price(
         raise ValueError(f"{gen_type} is not a valid generation type.")
 
     # Identify the load zones that correspond to the specified area and area_type
-    returned_zones = area_to_loadzone(grid.get_grid_model(), area, area_type)
+    returned_zones = area_to_loadzone(grid.grid_model, area, area_type)
 
     # Trim the DataFrame to only be of the desired area and generation type
     supply_data = supply_data.loc[supply_data.zone_name.isin(returned_zones)]
diff --git a/powersimdata/design/generation/tests/test_cost_curves.py b/powersimdata/design/generation/tests/test_cost_curves.py
index 5ae2347bd..effa79174 100644
--- a/powersimdata/design/generation/tests/test_cost_curves.py
+++ b/powersimdata/design/generation/tests/test_cost_curves.py
@@ -140,7 +140,6 @@
 grid = MockGrid(grid_attrs)
 grid.interconnect = "Western"
 grid.zone2id = {"Utah": 210, "Colorado": 212, "Washington": 201}
-grid.get_grid_model = lambda: "usa_tamu"
 
 
 def test_get_supply_data():
diff --git a/powersimdata/design/scenario_info.py b/powersimdata/design/scenario_info.py
index bbe8c67f3..f8c13c010 100644
--- a/powersimdata/design/scenario_info.py
+++ b/powersimdata/design/scenario_info.py
@@ -27,7 +27,7 @@ def __init__(self, scenario):
         self.pg = scenario.state.get_pg()
         self.grid = scenario.state.get_grid()
         self.demand = scenario.state.get_demand()
-        self.grid_model = self.grid.get_grid_model()
+        self.grid_model = self.grid.grid_model
         solar = scenario.state.get_solar()
         wind = scenario.state.get_wind()
         hydro = scenario.state.get_hydro()
diff --git a/powersimdata/design/tests/test_scenario_info.py b/powersimdata/design/tests/test_scenario_info.py
index f25e8e35d..eb9c55c20 100644
--- a/powersimdata/design/tests/test_scenario_info.py
+++ b/powersimdata/design/tests/test_scenario_info.py
@@ -79,36 +79,21 @@
     if mock_plant["zone_name"][i] == "Arizona"
 ]
 
-solar_plant_id = [
-    plant_id
-    for i, plant_id in enumerate(mock_plant["plant_id"])
-    if mock_plant["type"][i] == "solar"
-]
-wind_plant_id = [
-    plant_id
-    for i, plant_id in enumerate(mock_plant["plant_id"])
-    if mock_plant["type"][i] == "wind"
-]
-ng_plant_id = [
-    plant_id
-    for i, plant_id in enumerate(mock_plant["plant_id"])
-    if mock_plant["type"][i] == "ng"
-]
-coal_plant_id = [
-    plant_id
-    for i, plant_id in enumerate(mock_plant["plant_id"])
-    if mock_plant["type"][i] == "coal"
-]
-dfo_plant_id = [
-    plant_id
-    for i, plant_id in enumerate(mock_plant["plant_id"])
-    if mock_plant["type"][i] == "dfo"
-]
-hydro_plant_id = [
-    plant_id
-    for i, plant_id in enumerate(mock_plant["plant_id"])
-    if mock_plant["type"][i] == "hydro"
-]
+
+def _select_plant_id(type):
+    return [
+        plant_id
+        for i, plant_id in enumerate(mock_plant["plant_id"])
+        if mock_plant["type"][i] == type
+    ]
+
+
+solar_plant_id = _select_plant_id("solar")
+wind_plant_id = _select_plant_id("wind")
+ng_plant_id = _select_plant_id("ng")
+coal_plant_id = _select_plant_id("coal")
+dfo_plant_id = _select_plant_id("dfo")
+hydro_plant_id = _select_plant_id("hydro")
 
 mock_solar = mock_pg[solar_plant_id] * 2
 mock_wind = mock_pg[wind_plant_id] * 4
@@ -128,7 +113,6 @@ def setUp(self):
             wind=mock_wind,
             hydro=mock_hydro,
         )
-        scenario.state.grid.get_grid_model = lambda: "usa_tamu"
         scenario.state.grid.zone2id = {"Oregon": 202, "Arizona": 209}
         self.scenario_info = ScenarioInfo(scenario)
 
diff --git a/powersimdata/design/transmission/tests/test_upgrade.py b/powersimdata/design/transmission/tests/test_upgrade.py
index 1b587e955..676e36a4f 100644
--- a/powersimdata/design/transmission/tests/test_upgrade.py
+++ b/powersimdata/design/transmission/tests/test_upgrade.py
@@ -61,7 +61,6 @@
 mock_grid = MockGrid(
     grid_attrs={"branch": mock_branch, "bus": mock_bus, "plant": mock_plant}
 )
-mock_grid.get_grid_model = lambda: "usa_tamu"
 
 
 class TestStubTopologyHelpers(unittest.TestCase):
diff --git a/powersimdata/design/transmission/upgrade.py b/powersimdata/design/transmission/upgrade.py
index f816af7a6..57eddde2a 100644
--- a/powersimdata/design/transmission/upgrade.py
+++ b/powersimdata/design/transmission/upgrade.py
@@ -176,9 +176,8 @@ def get_branches_by_area(grid, area_names, method="either"):
 
     branch = grid.branch
     selected_branches = set()
-    grid_model = grid.get_grid_model()
     for a in area_names:
-        load_zone_names = area_to_loadzone(grid_model, a)
+        load_zone_names = area_to_loadzone(grid.grid_model, a)
         to_bus_in_area = branch.to_zone_name.isin(load_zone_names)
         from_bus_in_area = branch.from_zone_name.isin(load_zone_names)
         if method in ("internal", "either"):
diff --git a/powersimdata/input/grid.py b/powersimdata/input/grid.py
index 61cb69c59..cb2e73911 100644
--- a/powersimdata/input/grid.py
+++ b/powersimdata/input/grid.py
@@ -31,13 +31,6 @@ def __init__(self, interconnect, source="usa_tamu", engine="REISE"):
         if engine not in supported_engines:
             raise ValueError(f"Engine must be one of {','.join(supported_engines)}")
 
-        try:
-            self.model_immutables = ModelImmutables(source)
-        except ValueError:
-            self.model_immutables = ModelImmutables(
-                _get_grid_model_from_scenario_list(source)
-            )
-
         key = cache_key(interconnect, source)
         cached = _cache.get(key)
         if cached is not None:
@@ -65,7 +58,10 @@ def __init__(self, interconnect, source="usa_tamu", engine="REISE"):
 
         _cache.put(key, self)
 
-    def get_grid_model(self):
+        self.grid_model = self._get_grid_model()
+        self.model_immutables = ModelImmutables(self.grid_model)
+
+    def _get_grid_model(self):
         """Get the grid model.
 
         :return: (*str*).
diff --git a/powersimdata/tests/mock_grid.py b/powersimdata/tests/mock_grid.py
index 713cffccb..ed90248c9 100644
--- a/powersimdata/tests/mock_grid.py
+++ b/powersimdata/tests/mock_grid.py
@@ -196,6 +196,7 @@ def __init__(self, grid_attrs=None, model="usa_tamu"):
         if len(extra_keys) > 0:
             raise ValueError("Got unknown key(s):" + str(extra_keys))
 
+        self.grid_model = model
         self.model_immutables = ModelImmutables(model)
 
         cols = {
diff --git a/powersimdata/tests/mock_scenario_info.py b/powersimdata/tests/mock_scenario_info.py
index a5a627f92..ac7e45df1 100644
--- a/powersimdata/tests/mock_scenario_info.py
+++ b/powersimdata/tests/mock_scenario_info.py
@@ -6,7 +6,6 @@ class MockScenarioInfo(ScenarioInfo):
     def __init__(self, scenario=None):
         self._DEFAULT_FLOAT = 42
         scenario = MockScenario() if scenario is None else scenario
-        scenario.state.grid.get_grid_model = lambda: "mock_grid"
         super().__init__(scenario)
 
     def area_to_loadzone(self, area, area_type=None):

From 121d93a6d8e970e6563143f2fe21f60cf986f4ad Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Fri, 4 Jun 2021 10:44:29 -0700
Subject: [PATCH 47/60] chore: add networkx dependency

---
 Pipfile          |   3 +-
 Pipfile.lock     | 517 ++++++++++++++++++++++++++---------------------
 requirements.txt |   1 +
 setup.py         |   1 +
 4 files changed, 286 insertions(+), 236 deletions(-)

diff --git a/Pipfile b/Pipfile
index f784c092d..cc22aeff1 100644
--- a/Pipfile
+++ b/Pipfile
@@ -10,10 +10,11 @@ coverage = "*"
 pytest-cov = "*"
 
 [packages]
+networkx = "~=2.5"
 numpy = "~=1.20"
 pandas = "~=1.2"
 paramiko = "==2.7.2"
+psycopg2 = "~=2.8.5"
 scipy = "~=1.5"
 tqdm = "==4.29.1"
-psycopg2 = "~=2.8.5"
 requests = "~=2.25"
diff --git a/Pipfile.lock b/Pipfile.lock
index c5a34e335..5f23e354e 100644
--- a/Pipfile.lock
+++ b/Pipfile.lock
@@ -1,10 +1,12 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "84a6fc7c5ebb2d3bcdc557de8e5df841f7cdef2894cde2ecbf73b1331a73fd4b"
+            "sha256": "15fffec7dc4042ca1547cce2d0f44d496911088b5851551e65d95d6b90ef48b9"
         },
         "pipfile-spec": 6,
-        "requires": {},
+        "requires": {
+            "python_version": "3.8"
+        },
         "sources": [
             {
                 "name": "pypi",
@@ -14,6 +16,29 @@
         ]
     },
     "default": {
+        "appdirs": {
+            "hashes": [
+                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
+                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
+            ],
+            "version": "==1.4.4"
+        },
+        "atomicwrites": {
+            "hashes": [
+                "sha256:6d1784dea7c0c8d4a5172b6c620f40b6e4cbfdf96d783691f2e1302a7b88e197",
+                "sha256:ae70396ad1a434f9c7046fd2dd196fc04b12f9e91ffb859164193be8b6168a7a"
+            ],
+            "markers": "sys_platform == 'win32'",
+            "version": "==1.4.0"
+        },
+        "attrs": {
+            "hashes": [
+                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
+                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
+            "version": "==21.2.0"
+        },
         "bcrypt": {
             "hashes": [
                 "sha256:5b93c1726e50a93a033c36e5ca7fdcd29a5c7395af50a6892f5d9e7c6cfbfb29",
@@ -24,8 +49,17 @@
                 "sha256:cd1ea2ff3038509ea95f687256c46b79f5fc382ad0aa3664d200047546d511d1",
                 "sha256:cdcdcb3972027f83fe24a48b1e90ea4b584d35f1cc279d76de6fc4b13376239d"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==3.2.0"
         },
+        "black": {
+            "hashes": [
+                "sha256:1fc0e0a2c8ae7d269dfcf0c60a89afa299664f3e811395d40b1922dff8f854b5",
+                "sha256:e5cf21ebdffc7a9b29d73912b6a6a9a4df4ce70220d523c21647da2eae0751ef"
+            ],
+            "index": "pypi",
+            "version": "==21.5b2"
+        },
         "certifi": {
             "hashes": [
                 "sha256:2bbf76fd432960138b3ef6dda3dde0544f27cbf8546c458e60baf371917ba9ee",
@@ -92,8 +126,83 @@
                 "sha256:0d6f53a15db4120f2b08c94f11e7d93d2c911ee118b6b30a04ec3ee8310179fa",
                 "sha256:f864054d66fd9118f2e67044ac8981a54775ec5b67aed0441892edb553d21da5"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4'",
             "version": "==4.0.0"
         },
+        "click": {
+            "hashes": [
+                "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a",
+                "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"
+            ],
+            "markers": "python_version >= '3.6'",
+            "version": "==8.0.1"
+        },
+        "colorama": {
+            "hashes": [
+                "sha256:5941b2b48a20143d2267e95b1c2a7603ce057ee39fd88e7329b0c292aa16869b",
+                "sha256:9f47eda37229f68eee03b24b9748937c7dc3868f906e8ba69fbcbdd3bc5dc3e2"
+            ],
+            "markers": "sys_platform == 'win32' and platform_system == 'Windows'",
+            "version": "==0.4.4"
+        },
+        "coverage": {
+            "hashes": [
+                "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c",
+                "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6",
+                "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45",
+                "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a",
+                "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03",
+                "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529",
+                "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a",
+                "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a",
+                "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2",
+                "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6",
+                "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759",
+                "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53",
+                "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a",
+                "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4",
+                "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff",
+                "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502",
+                "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793",
+                "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb",
+                "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905",
+                "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821",
+                "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b",
+                "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81",
+                "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0",
+                "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b",
+                "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3",
+                "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184",
+                "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701",
+                "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a",
+                "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82",
+                "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638",
+                "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5",
+                "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083",
+                "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6",
+                "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90",
+                "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465",
+                "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a",
+                "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3",
+                "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e",
+                "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066",
+                "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf",
+                "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b",
+                "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae",
+                "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669",
+                "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873",
+                "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b",
+                "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6",
+                "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb",
+                "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160",
+                "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c",
+                "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079",
+                "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d",
+                "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6"
+            ],
+            "index": "pypi",
+            "version": "==5.5"
+        },
         "cryptography": {
             "hashes": [
                 "sha256:0f1212a66329c80d68aeeb39b8a16d54ef57071bf22ff4e521657b27372e327d",
@@ -109,44 +218,83 @@
                 "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d",
                 "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"
             ],
+            "markers": "python_version >= '3.6'",
             "version": "==3.4.7"
         },
+        "decorator": {
+            "hashes": [
+                "sha256:41fa54c2a0cc4ba648be4fd43cff00aedf5b9465c9bf18d64325bc225f08f760",
+                "sha256:e3a62f0520172440ca0dcc823749319382e377f37f140a0b99ef45fecb84bfe7"
+            ],
+            "version": "==4.4.2"
+        },
         "idna": {
             "hashes": [
                 "sha256:b307872f855b18632ce0c21c5e45be78c0ea7ae4c15c828c20788b26921eb3f6",
                 "sha256:b97d804b1e9b523befed77c48dacec60e6dcb0b5391d57af6a65a312a90648c0"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.10"
         },
+        "iniconfig": {
+            "hashes": [
+                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
+                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
+            ],
+            "version": "==1.1.1"
+        },
+        "mypy-extensions": {
+            "hashes": [
+                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
+                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
+            ],
+            "version": "==0.4.3"
+        },
+        "networkx": {
+            "hashes": [
+                "sha256:0635858ed7e989f4c574c2328380b452df892ae85084144c73d8cd819f0c4e06",
+                "sha256:109cd585cac41297f71103c3c42ac6ef7379f29788eb54cb751be5a663bb235a"
+            ],
+            "index": "pypi",
+            "version": "==2.5.1"
+        },
         "numpy": {
             "hashes": [
-                "sha256:2428b109306075d89d21135bdd6b785f132a1f5a3260c371cee1fae427e12727",
-                "sha256:377751954da04d4a6950191b20539066b4e19e3b559d4695399c5e8e3e683bf6",
-                "sha256:4703b9e937df83f5b6b7447ca5912b5f5f297aba45f91dbbbc63ff9278c7aa98",
-                "sha256:471c0571d0895c68da309dacee4e95a0811d0a9f9f532a48dc1bea5f3b7ad2b7",
-                "sha256:61d5b4cf73622e4d0c6b83408a16631b670fc045afd6540679aa35591a17fe6d",
-                "sha256:6c915ee7dba1071554e70a3664a839fbc033e1d6528199d4621eeaaa5487ccd2",
-                "sha256:6e51e417d9ae2e7848314994e6fc3832c9d426abce9328cf7571eefceb43e6c9",
-                "sha256:719656636c48be22c23641859ff2419b27b6bdf844b36a2447cb39caceb00935",
-                "sha256:780ae5284cb770ade51d4b4a7dce4faa554eb1d88a56d0e8b9f35fca9b0270ff",
-                "sha256:878922bf5ad7550aa044aa9301d417e2d3ae50f0f577de92051d739ac6096cee",
-                "sha256:924dc3f83de20437de95a73516f36e09918e9c9c18d5eac520062c49191025fb",
-                "sha256:97ce8b8ace7d3b9288d88177e66ee75480fb79b9cf745e91ecfe65d91a856042",
-                "sha256:9c0fab855ae790ca74b27e55240fe4f2a36a364a3f1ebcfd1fb5ac4088f1cec3",
-                "sha256:9cab23439eb1ebfed1aaec9cd42b7dc50fc96d5cd3147da348d9161f0501ada5",
-                "sha256:a8e6859913ec8eeef3dbe9aed3bf475347642d1cdd6217c30f28dee8903528e6",
-                "sha256:aa046527c04688af680217fffac61eec2350ef3f3d7320c07fd33f5c6e7b4d5f",
-                "sha256:abc81829c4039e7e4c30f7897938fa5d4916a09c2c7eb9b244b7a35ddc9656f4",
-                "sha256:bad70051de2c50b1a6259a6df1daaafe8c480ca98132da98976d8591c412e737",
-                "sha256:c73a7975d77f15f7f68dacfb2bca3d3f479f158313642e8ea9058eea06637931",
-                "sha256:d15007f857d6995db15195217afdbddfcd203dfaa0ba6878a2f580eaf810ecd6",
-                "sha256:d76061ae5cab49b83a8cf3feacefc2053fac672728802ac137dd8c4123397677",
-                "sha256:e8e4fbbb7e7634f263c5b0150a629342cc19b47c5eba8d1cd4363ab3455ab576",
-                "sha256:e9459f40244bb02b2f14f6af0cd0732791d72232bbb0dc4bab57ef88e75f6935",
-                "sha256:edb1f041a9146dcf02cd7df7187db46ab524b9af2515f392f337c7cbbf5b52cd"
+                "sha256:1676b0a292dd3c99e49305a16d7a9f42a4ab60ec522eac0d3dd20cdf362ac010",
+                "sha256:16f221035e8bd19b9dc9a57159e38d2dd060b48e93e1d843c49cb370b0f415fd",
+                "sha256:43909c8bb289c382170e0282158a38cf306a8ad2ff6dfadc447e90f9961bef43",
+                "sha256:4e465afc3b96dbc80cf4a5273e5e2b1e3451286361b4af70ce1adb2984d392f9",
+                "sha256:55b745fca0a5ab738647d0e4db099bd0a23279c32b31a783ad2ccea729e632df",
+                "sha256:5d050e1e4bc9ddb8656d7b4f414557720ddcca23a5b88dd7cff65e847864c400",
+                "sha256:637d827248f447e63585ca3f4a7d2dfaa882e094df6cfa177cc9cf9cd6cdf6d2",
+                "sha256:6690080810f77485667bfbff4f69d717c3be25e5b11bb2073e76bb3f578d99b4",
+                "sha256:66fbc6fed94a13b9801fb70b96ff30605ab0a123e775a5e7a26938b717c5d71a",
+                "sha256:67d44acb72c31a97a3d5d33d103ab06d8ac20770e1c5ad81bdb3f0c086a56cf6",
+                "sha256:6ca2b85a5997dabc38301a22ee43c82adcb53ff660b89ee88dded6b33687e1d8",
+                "sha256:6e51534e78d14b4a009a062641f465cfaba4fdcb046c3ac0b1f61dd97c861b1b",
+                "sha256:70eb5808127284c4e5c9e836208e09d685a7978b6a216db85960b1a112eeace8",
+                "sha256:830b044f4e64a76ba71448fce6e604c0fc47a0e54d8f6467be23749ac2cbd2fb",
+                "sha256:8b7bb4b9280da3b2856cb1fc425932f46fba609819ee1c62256f61799e6a51d2",
+                "sha256:a9c65473ebc342715cb2d7926ff1e202c26376c0dcaaee85a1fd4b8d8c1d3b2f",
+                "sha256:c1c09247ccea742525bdb5f4b5ceeacb34f95731647fe55774aa36557dbb5fa4",
+                "sha256:c5bf0e132acf7557fc9bb8ded8b53bbbbea8892f3c9a1738205878ca9434206a",
+                "sha256:db250fd3e90117e0312b611574cd1b3f78bec046783195075cbd7ba9c3d73f16",
+                "sha256:e515c9a93aebe27166ec9593411c58494fa98e5fcc219e47260d9ab8a1cc7f9f",
+                "sha256:e55185e51b18d788e49fe8305fd73ef4470596b33fc2c1ceb304566b99c71a69",
+                "sha256:ea9cff01e75a956dbee133fa8e5b68f2f92175233de2f88de3a682dd94deda65",
+                "sha256:f1452578d0516283c87608a5a5548b0cdde15b99650efdfd85182102ef7a7c17",
+                "sha256:f39a995e47cb8649673cfa0579fbdd1cdd33ea497d1728a6cb194d6252268e48"
             ],
             "index": "pypi",
-            "version": "==1.20.2"
+            "version": "==1.20.3"
+        },
+        "packaging": {
+            "hashes": [
+                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
+                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==20.9"
         },
         "pandas": {
             "hashes": [
@@ -178,6 +326,21 @@
             "index": "pypi",
             "version": "==2.7.2"
         },
+        "pathspec": {
+            "hashes": [
+                "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd",
+                "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"
+            ],
+            "version": "==0.8.1"
+        },
+        "pluggy": {
+            "hashes": [
+                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
+                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==0.13.1"
+        },
         "psycopg2": {
             "hashes": [
                 "sha256:00195b5f6832dbf2876b8bf77f12bdce648224c89c880719c745b90515233301",
@@ -199,11 +362,20 @@
             "index": "pypi",
             "version": "==2.8.6"
         },
+        "py": {
+            "hashes": [
+                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
+                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
+            "version": "==1.10.0"
+        },
         "pycparser": {
             "hashes": [
                 "sha256:2d475327684562c3a96cc71adf7dc8c4f0565175cf86b6d7a404ff4c771f15f0",
                 "sha256:7582ad22678f0fcd81102833f60ef8d0e57288b6b5fb00323d101be910e35705"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==2.20"
         },
         "pynacl": {
@@ -227,231 +399,47 @@
                 "sha256:ea6841bc3a76fa4942ce00f3bda7d436fda21e2d91602b9e21b7ca9ecab8f3ff",
                 "sha256:f8851ab9041756003119368c1e6cd0b9c631f46d686b3904b18c0139f4419f80"
             ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
             "version": "==1.4.0"
         },
-        "python-dateutil": {
-            "hashes": [
-                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
-                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
-            ],
-            "version": "==2.8.1"
-        },
-        "pytz": {
-            "hashes": [
-                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
-                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
-            ],
-            "version": "==2021.1"
-        },
-        "requests": {
-            "hashes": [
-                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
-                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
-            ],
-            "index": "pypi",
-            "version": "==2.25.1"
-        },
-        "scipy": {
-            "hashes": [
-                "sha256:01b38dec7e9f897d4db04f8de4e20f0f5be3feac98468188a0f47a991b796055",
-                "sha256:10dbcc7de03b8d635a1031cb18fd3eaa997969b64fdf78f99f19ac163a825445",
-                "sha256:19aeac1ad3e57338723f4657ac8520f41714804568f2e30bd547d684d72c392e",
-                "sha256:1b21c6e0dc97b1762590b70dee0daddb291271be0580384d39f02c480b78290a",
-                "sha256:1caade0ede6967cc675e235c41451f9fb89ae34319ddf4740194094ab736b88d",
-                "sha256:23995dfcf269ec3735e5a8c80cfceaf384369a47699df111a6246b83a55da582",
-                "sha256:2a799714bf1f791fb2650d73222b248d18d53fd40d6af2df2c898db048189606",
-                "sha256:3274ce145b5dc416c49c0cf8b6119f787f0965cd35e22058fe1932c09fe15d77",
-                "sha256:33d1677d46111cfa1c84b87472a0274dde9ef4a7ef2e1f155f012f5f1e995d8f",
-                "sha256:44d452850f77e65e25b1eb1ac01e25770323a782bfe3a1a3e43847ad4266d93d",
-                "sha256:9e3302149a369697c6aaea18b430b216e3c88f9a61b62869f6104881e5f9ef85",
-                "sha256:a75b014d3294fce26852a9d04ea27b5671d86736beb34acdfc05859246260707",
-                "sha256:ad7269254de06743fb4768f658753de47d8b54e4672c5ebe8612a007a088bd48",
-                "sha256:b30280fbc1fd8082ac822994a98632111810311a9ece71a0e48f739df3c555a2",
-                "sha256:b79104878003487e2b4639a20b9092b02e1bad07fc4cf924b495cf413748a777",
-                "sha256:d449d40e830366b4c612692ad19fbebb722b6b847f78a7b701b1e0d6cda3cc13",
-                "sha256:d647757373985207af3343301d89fe738d5a294435a4f2aafb04c13b4388c896",
-                "sha256:f68eb46b86b2c246af99fcaa6f6e37c7a7a413e1084a794990b877f2ff71f7b6",
-                "sha256:fdf606341cd798530b05705c87779606fcdfaf768a8129c348ea94441da15b04"
-            ],
-            "index": "pypi",
-            "version": "==1.6.3"
-        },
-        "six": {
-            "hashes": [
-                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
-                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
-            ],
-            "version": "==1.16.0"
-        },
-        "tqdm": {
-            "hashes": [
-                "sha256:b856be5cb6cfaee3b2733655c7c5bbc7751291bb5d1a4f54f020af4727570b3e",
-                "sha256:c9b9b5eeba13994a4c266aae7eef7aeeb0ba2973e431027e942b4faea139ef49"
-            ],
-            "index": "pypi",
-            "version": "==4.29.1"
-        },
-        "urllib3": {
-            "hashes": [
-                "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c",
-                "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"
-            ],
-            "index": "pypi",
-            "version": "==1.26.5"
-        }
-    },
-    "develop": {
-        "appdirs": {
-            "hashes": [
-                "sha256:7d5d0167b2b1ba821647616af46a749d1c653740dd0d2415100fe26e27afdf41",
-                "sha256:a841dacd6b99318a741b166adb07e19ee71a274450e68237b4650ca1055ab128"
-            ],
-            "version": "==1.4.4"
-        },
-        "attrs": {
-            "hashes": [
-                "sha256:149e90d6d8ac20db7a955ad60cf0e6881a3f20d37096140088356da6c716b0b1",
-                "sha256:ef6aaac3ca6cd92904cdd0d83f629a15f18053ec84e6432106f7a4d04ae4f5fb"
-            ],
-            "version": "==21.2.0"
-        },
-        "black": {
-            "hashes": [
-                "sha256:bff7067d8bc25eb21dcfdbc8c72f2baafd9ec6de4663241a52fb904b304d391f",
-                "sha256:fc9bcf3b482b05c1f35f6a882c079dc01b9c7795827532f4cc43c0ec88067bbc"
-            ],
-            "index": "pypi",
-            "version": "==21.4b2"
-        },
-        "click": {
-            "hashes": [
-                "sha256:8c04c11192119b1ef78ea049e0a6f0463e4c48ef00a30160c704337586f3ad7a",
-                "sha256:fba402a4a47334742d782209a7c79bc448911afe1149d07bdabdf480b3e2f4b6"
-            ],
-            "version": "==8.0.1"
-        },
-        "coverage": {
-            "hashes": [
-                "sha256:004d1880bed2d97151facef49f08e255a20ceb6f9432df75f4eef018fdd5a78c",
-                "sha256:01d84219b5cdbfc8122223b39a954820929497a1cb1422824bb86b07b74594b6",
-                "sha256:040af6c32813fa3eae5305d53f18875bedd079960822ef8ec067a66dd8afcd45",
-                "sha256:06191eb60f8d8a5bc046f3799f8a07a2d7aefb9504b0209aff0b47298333302a",
-                "sha256:13034c4409db851670bc9acd836243aeee299949bd5673e11844befcb0149f03",
-                "sha256:13c4ee887eca0f4c5a247b75398d4114c37882658300e153113dafb1d76de529",
-                "sha256:184a47bbe0aa6400ed2d41d8e9ed868b8205046518c52464fde713ea06e3a74a",
-                "sha256:18ba8bbede96a2c3dde7b868de9dcbd55670690af0988713f0603f037848418a",
-                "sha256:1aa846f56c3d49205c952d8318e76ccc2ae23303351d9270ab220004c580cfe2",
-                "sha256:217658ec7187497e3f3ebd901afdca1af062b42cfe3e0dafea4cced3983739f6",
-                "sha256:24d4a7de75446be83244eabbff746d66b9240ae020ced65d060815fac3423759",
-                "sha256:2910f4d36a6a9b4214bb7038d537f015346f413a975d57ca6b43bf23d6563b53",
-                "sha256:2949cad1c5208b8298d5686d5a85b66aae46d73eec2c3e08c817dd3513e5848a",
-                "sha256:2a3859cb82dcbda1cfd3e6f71c27081d18aa251d20a17d87d26d4cd216fb0af4",
-                "sha256:2cafbbb3af0733db200c9b5f798d18953b1a304d3f86a938367de1567f4b5bff",
-                "sha256:2e0d881ad471768bf6e6c2bf905d183543f10098e3b3640fc029509530091502",
-                "sha256:30c77c1dc9f253283e34c27935fded5015f7d1abe83bc7821680ac444eaf7793",
-                "sha256:3487286bc29a5aa4b93a072e9592f22254291ce96a9fbc5251f566b6b7343cdb",
-                "sha256:372da284cfd642d8e08ef606917846fa2ee350f64994bebfbd3afb0040436905",
-                "sha256:41179b8a845742d1eb60449bdb2992196e211341818565abded11cfa90efb821",
-                "sha256:44d654437b8ddd9eee7d1eaee28b7219bec228520ff809af170488fd2fed3e2b",
-                "sha256:4a7697d8cb0f27399b0e393c0b90f0f1e40c82023ea4d45d22bce7032a5d7b81",
-                "sha256:51cb9476a3987c8967ebab3f0fe144819781fca264f57f89760037a2ea191cb0",
-                "sha256:52596d3d0e8bdf3af43db3e9ba8dcdaac724ba7b5ca3f6358529d56f7a166f8b",
-                "sha256:53194af30d5bad77fcba80e23a1441c71abfb3e01192034f8246e0d8f99528f3",
-                "sha256:5fec2d43a2cc6965edc0bb9e83e1e4b557f76f843a77a2496cbe719583ce8184",
-                "sha256:6c90e11318f0d3c436a42409f2749ee1a115cd8b067d7f14c148f1ce5574d701",
-                "sha256:74d881fc777ebb11c63736622b60cb9e4aee5cace591ce274fb69e582a12a61a",
-                "sha256:7501140f755b725495941b43347ba8a2777407fc7f250d4f5a7d2a1050ba8e82",
-                "sha256:796c9c3c79747146ebd278dbe1e5c5c05dd6b10cc3bcb8389dfdf844f3ead638",
-                "sha256:869a64f53488f40fa5b5b9dcb9e9b2962a66a87dab37790f3fcfb5144b996ef5",
-                "sha256:8963a499849a1fc54b35b1c9f162f4108017b2e6db2c46c1bed93a72262ed083",
-                "sha256:8d0a0725ad7c1a0bcd8d1b437e191107d457e2ec1084b9f190630a4fb1af78e6",
-                "sha256:900fbf7759501bc7807fd6638c947d7a831fc9fdf742dc10f02956ff7220fa90",
-                "sha256:92b017ce34b68a7d67bd6d117e6d443a9bf63a2ecf8567bb3d8c6c7bc5014465",
-                "sha256:970284a88b99673ccb2e4e334cfb38a10aab7cd44f7457564d11898a74b62d0a",
-                "sha256:972c85d205b51e30e59525694670de6a8a89691186012535f9d7dbaa230e42c3",
-                "sha256:9a1ef3b66e38ef8618ce5fdc7bea3d9f45f3624e2a66295eea5e57966c85909e",
-                "sha256:af0e781009aaf59e25c5a678122391cb0f345ac0ec272c7961dc5455e1c40066",
-                "sha256:b6d534e4b2ab35c9f93f46229363e17f63c53ad01330df9f2d6bd1187e5eaacf",
-                "sha256:b7895207b4c843c76a25ab8c1e866261bcfe27bfaa20c192de5190121770672b",
-                "sha256:c0891a6a97b09c1f3e073a890514d5012eb256845c451bd48f7968ef939bf4ae",
-                "sha256:c2723d347ab06e7ddad1a58b2a821218239249a9e4365eaff6649d31180c1669",
-                "sha256:d1f8bf7b90ba55699b3a5e44930e93ff0189aa27186e96071fac7dd0d06a1873",
-                "sha256:d1f9ce122f83b2305592c11d64f181b87153fc2c2bbd3bb4a3dde8303cfb1a6b",
-                "sha256:d314ed732c25d29775e84a960c3c60808b682c08d86602ec2c3008e1202e3bb6",
-                "sha256:d636598c8305e1f90b439dbf4f66437de4a5e3c31fdf47ad29542478c8508bbb",
-                "sha256:deee1077aae10d8fa88cb02c845cfba9b62c55e1183f52f6ae6a2df6a2187160",
-                "sha256:ebe78fe9a0e874362175b02371bdfbee64d8edc42a044253ddf4ee7d3c15212c",
-                "sha256:f030f8873312a16414c0d8e1a1ddff2d3235655a2174e3648b4fa66b3f2f1079",
-                "sha256:f0b278ce10936db1a37e6954e15a3730bea96a0997c26d7fee88e6c396c2086d",
-                "sha256:f11642dddbb0253cc8853254301b51390ba0081750a8ac03f20ea8103f0c56b6"
-            ],
-            "index": "pypi",
-            "version": "==5.5"
-        },
-        "iniconfig": {
-            "hashes": [
-                "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3",
-                "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32"
-            ],
-            "version": "==1.1.1"
-        },
-        "mypy-extensions": {
-            "hashes": [
-                "sha256:090fedd75945a69ae91ce1303b5824f428daf5a028d2f6ab8a299250a846f15d",
-                "sha256:2d82818f5bb3e369420cb3c4060a7970edba416647068eb4c5343488a6c604a8"
-            ],
-            "version": "==0.4.3"
-        },
-        "packaging": {
-            "hashes": [
-                "sha256:5b327ac1320dc863dca72f4514ecc086f31186744b84a230374cc1fd776feae5",
-                "sha256:67714da7f7bc052e064859c05c595155bd1ee9f69f76557e21f051443c20947a"
-            ],
-            "version": "==20.9"
-        },
-        "pathspec": {
-            "hashes": [
-                "sha256:86379d6b86d75816baba717e64b1a3a3469deb93bb76d613c9ce79edc5cb68fd",
-                "sha256:aa0cb481c4041bf52ffa7b0d8fa6cd3e88a2ca4879c533c9153882ee2556790d"
-            ],
-            "version": "==0.8.1"
-        },
-        "pluggy": {
-            "hashes": [
-                "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0",
-                "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d"
-            ],
-            "version": "==0.13.1"
-        },
-        "py": {
-            "hashes": [
-                "sha256:21b81bda15b66ef5e1a777a21c4dcd9c20ad3efd0b3f817e7a809035269e1bd3",
-                "sha256:3b80836aa6d1feeaa108e046da6423ab8f6ceda6468545ae8d02d9d58d18818a"
-            ],
-            "version": "==1.10.0"
-        },
         "pyparsing": {
             "hashes": [
                 "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1",
                 "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b"
             ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==2.4.7"
         },
         "pytest": {
             "hashes": [
-                "sha256:671238a46e4df0f3498d1c3270e5deb9b32d25134c99b7d75370a68cfbe9b634",
-                "sha256:6ad9c7bdf517a808242b998ac20063c41532a570d088d77eec1ee12b0b5574bc"
+                "sha256:50bcad0a0b9c5a72c8e4e7c9855a3ad496ca6a881a3641b4260605450772c54b",
+                "sha256:91ef2131a9bd6be8f76f1f08eac5c5317221d6ad1e143ae03894b862e8976890"
             ],
             "index": "pypi",
-            "version": "==6.2.3"
+            "version": "==6.2.4"
         },
         "pytest-cov": {
             "hashes": [
-                "sha256:359952d9d39b9f822d9d29324483e7ba04a3a17dd7d05aa6beb7ea01e359e5f7",
-                "sha256:bdb9fdb0b85a7cc825269a4c56b48ccaa5c7e365054b6038772c32ddcdc969da"
+                "sha256:261bb9e47e65bd099c89c3edf92972865210c36813f80ede5277dceb77a4a62a",
+                "sha256:261ceeb8c227b726249b376b8526b600f38667ee314f910353fa318caa01f4d7"
             ],
             "index": "pypi",
-            "version": "==2.11.1"
+            "version": "==2.12.1"
+        },
+        "python-dateutil": {
+            "hashes": [
+                "sha256:73ebfe9dbf22e832286dafa60473e4cd239f8592f699aa5adaf10050e6e1823c",
+                "sha256:75bb3f31ea686f1197762692a9ee6a7550b59fc6ca3a1f4b5d7e32fb98e2da2a"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==2.8.1"
+        },
+        "pytz": {
+            "hashes": [
+                "sha256:83a4a90894bf38e243cf052c8b58f381bfe9a7a483f6a9cab140bc7f702ac4da",
+                "sha256:eb10ce3e7736052ed3623d49975ce333bcd712c7bb19a58b9e2089d4057d0798"
+            ],
+            "version": "==2021.1"
         },
         "regex": {
             "hashes": [
@@ -499,12 +487,71 @@
             ],
             "version": "==2021.4.4"
         },
+        "requests": {
+            "hashes": [
+                "sha256:27973dd4a904a4f13b263a19c866c13b92a39ed1c964655f025f3f8d3d75b804",
+                "sha256:c210084e36a42ae6b9219e00e48287def368a26d03a048ddad7bfee44f75871e"
+            ],
+            "index": "pypi",
+            "version": "==2.25.1"
+        },
+        "scipy": {
+            "hashes": [
+                "sha256:01b38dec7e9f897d4db04f8de4e20f0f5be3feac98468188a0f47a991b796055",
+                "sha256:10dbcc7de03b8d635a1031cb18fd3eaa997969b64fdf78f99f19ac163a825445",
+                "sha256:19aeac1ad3e57338723f4657ac8520f41714804568f2e30bd547d684d72c392e",
+                "sha256:1b21c6e0dc97b1762590b70dee0daddb291271be0580384d39f02c480b78290a",
+                "sha256:1caade0ede6967cc675e235c41451f9fb89ae34319ddf4740194094ab736b88d",
+                "sha256:23995dfcf269ec3735e5a8c80cfceaf384369a47699df111a6246b83a55da582",
+                "sha256:2a799714bf1f791fb2650d73222b248d18d53fd40d6af2df2c898db048189606",
+                "sha256:3274ce145b5dc416c49c0cf8b6119f787f0965cd35e22058fe1932c09fe15d77",
+                "sha256:33d1677d46111cfa1c84b87472a0274dde9ef4a7ef2e1f155f012f5f1e995d8f",
+                "sha256:44d452850f77e65e25b1eb1ac01e25770323a782bfe3a1a3e43847ad4266d93d",
+                "sha256:9e3302149a369697c6aaea18b430b216e3c88f9a61b62869f6104881e5f9ef85",
+                "sha256:a75b014d3294fce26852a9d04ea27b5671d86736beb34acdfc05859246260707",
+                "sha256:ad7269254de06743fb4768f658753de47d8b54e4672c5ebe8612a007a088bd48",
+                "sha256:b30280fbc1fd8082ac822994a98632111810311a9ece71a0e48f739df3c555a2",
+                "sha256:b79104878003487e2b4639a20b9092b02e1bad07fc4cf924b495cf413748a777",
+                "sha256:d449d40e830366b4c612692ad19fbebb722b6b847f78a7b701b1e0d6cda3cc13",
+                "sha256:d647757373985207af3343301d89fe738d5a294435a4f2aafb04c13b4388c896",
+                "sha256:f68eb46b86b2c246af99fcaa6f6e37c7a7a413e1084a794990b877f2ff71f7b6",
+                "sha256:fdf606341cd798530b05705c87779606fcdfaf768a8129c348ea94441da15b04"
+            ],
+            "index": "pypi",
+            "version": "==1.6.3"
+        },
+        "six": {
+            "hashes": [
+                "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926",
+                "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2'",
+            "version": "==1.16.0"
+        },
         "toml": {
             "hashes": [
                 "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b",
                 "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"
             ],
+            "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2'",
             "version": "==0.10.2"
+        },
+        "tqdm": {
+            "hashes": [
+                "sha256:b856be5cb6cfaee3b2733655c7c5bbc7751291bb5d1a4f54f020af4727570b3e",
+                "sha256:c9b9b5eeba13994a4c266aae7eef7aeeb0ba2973e431027e942b4faea139ef49"
+            ],
+            "index": "pypi",
+            "version": "==4.29.1"
+        },
+        "urllib3": {
+            "hashes": [
+                "sha256:753a0374df26658f99d826cfe40394a686d05985786d946fbe4165b5148f5a7c",
+                "sha256:a7acd0977125325f516bda9735fa7142b909a8d01e8b2e4c8108d0984e6e0098"
+            ],
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4' and python_version < '4'",
+            "version": "==1.26.5"
         }
-    }
+    },
+    "develop": {}
 }
diff --git a/requirements.txt b/requirements.txt
index 47f813e0f..45abfc3b3 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,3 +1,4 @@
+networkx~=2.5
 numpy~=1.20
 pandas~=1.2
 paramiko==2.7.2
diff --git a/setup.py b/setup.py
index e42ff3082..15433827d 100644
--- a/setup.py
+++ b/setup.py
@@ -1,6 +1,7 @@
 from setuptools import find_packages, setup
 
 install_requires = [
+    "networkx",
     "numpy",
     "pandas",
     "paramiko",

From 2f5f5ceeb9a999eb00dd94ab4bc30834d44f9aff Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Fri, 4 Jun 2021 12:02:14 -0700
Subject: [PATCH 48/60] chore: change type of interconnect_combinations from
 set to dict to serve check_grid

---
 powersimdata/network/usa_tamu/constants/zones.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/powersimdata/network/usa_tamu/constants/zones.py b/powersimdata/network/usa_tamu/constants/zones.py
index 530c2db0d..68c0adcb1 100644
--- a/powersimdata/network/usa_tamu/constants/zones.py
+++ b/powersimdata/network/usa_tamu/constants/zones.py
@@ -30,7 +30,10 @@
 mappings = {"loadzone", "state", "state_abbr", "interconnect"}
 
 # Define combinations of interconnects
-interconnect_combinations = {"USA", "Texas_Western"}
+interconnect_combinations = {
+    "USA": {"Eastern", "Western", "Texas"},
+    "Texas_Western": {"Western", "Texas"},
+}
 
 
 # Map state abbreviations to state name

From 7d80ed6169d4ad2f1d359a517bd677ab8654e1ae Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Thu, 3 Jun 2021 15:22:22 -0700
Subject: [PATCH 49/60] feat: add user-facing check_grid function and
 lower-level sub-functions

---
 powersimdata/input/check.py | 201 ++++++++++++++++++++++++++++++++++++
 1 file changed, 201 insertions(+)
 create mode 100644 powersimdata/input/check.py

diff --git a/powersimdata/input/check.py b/powersimdata/input/check.py
new file mode 100644
index 000000000..0b55c2409
--- /dev/null
+++ b/powersimdata/input/check.py
@@ -0,0 +1,201 @@
+import networkx as nx
+
+from powersimdata.input.grid import Grid
+
+
+def check_grid(grid):
+    """Check whether an object is an internally-consistent Grid object.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :raises TypeError: if ``grid`` is not a Grid object.
+    """
+    error_messages = []
+    if not isinstance(grid, Grid):
+        error_messages.append("grid must be a Grid object")
+    _check_attributes(grid, error_messages)
+    _check_for_islanded_buses(grid, error_messages)
+    _check_for_undescribed_buses(grid, error_messages)
+    _check_bus_against_bus2sub(grid, error_messages)
+    _check_ac_interconnects(grid, error_messages)
+    _check_transformer_substations(grid, error_messages)
+    _check_line_voltages(grid, error_messages)
+    _check_plant_against_gencost(grid, error_messages)
+    _check_connected_components(grid, error_messages)
+    if len(error_messages) > 0:
+        collected = "\n".join(error_messages)
+        raise ValueError(f"Problem(s) found with grid:\n{collected}")
+
+
+def _check_attributes(grid, error_messages):
+    """Check whether a Grid object has the required attributes.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        ``grid`` is missing one or more required attributes.
+    """
+    required = {
+        "branch",
+        "bus",
+        "bus2sub",
+        "dcline",
+        "data_loc",
+        "gencost",
+        "grid_model",
+        "interconnect",
+        "model_immutables",
+        "plant",
+        "storage",
+        "sub",
+    }
+    for r in required:
+        if not hasattr(grid, r):
+            error_messages.append(f"grid object must have attribute {r}.")
+
+
+def _check_for_islanded_buses(grid, error_messages):
+    """Check whether a transmission network (AC & DC) does not connect to one or more
+    buses.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        branches/DC lines exist in the ``grid``, but one or more buses are islanded.
+    """
+    if len(grid.branch) + len(grid.dcline) > 0:
+        connected_buses = set().union(
+            set(grid.branch.from_bus_id),
+            set(grid.branch.to_bus_id),
+            set(grid.dcline.from_bus_id),
+            set(grid.dcline.to_bus_id),
+        )
+        isolated_buses = set(grid.bus.index) - connected_buses
+        if len(isolated_buses) > 0:
+            error_messages.append(f"islanded buses detected: {isolated_buses}.")
+
+
+def _check_for_undescribed_buses(grid, error_messages):
+    """Check whether any transmission elements are connected to buses that are not
+    described in the bus table.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        any transmission elements are connected to buses that are not described in the
+        bus table of the ``grid``.
+    """
+    expected_buses = set().union(
+        set(grid.branch.from_bus_id),
+        set(grid.branch.to_bus_id),
+        set(grid.dcline.from_bus_id),
+        set(grid.dcline.to_bus_id),
+    )
+    undescribed_buses = expected_buses - set(grid.bus.index)
+    if len(undescribed_buses) > 0:
+        error_messages.append(
+            "buses present in transmission network but missing from bus table: "
+            f"{undescribed_buses}."
+        )
+
+
+def _check_bus_against_bus2sub(grid, error_messages):
+    """Check whether indices of bus and bus2sub tables match.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        indices of bus and bus2sub tables of the ``grid`` don't match.
+    """
+    if not set(grid.bus.index) == set(grid.bus2sub.index):
+        error_messages.append("indices for bus and bus2sub don't match.")
+
+
+def _check_ac_interconnects(grid, error_messages):
+    """Check whether any AC branches bridge across interconnections.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        any AC branches bridge across interconnections of the ``grid``.
+    """
+    from_interconnect = grid.branch.from_bus_id.map(grid.bus.interconnect)
+    to_interconnect = grid.branch.to_bus_id.map(grid.bus.interconnect)
+    if not all(from_interconnect == to_interconnect):
+        non_matching_ids = grid.branch.index[from_interconnect != to_interconnect]
+        error_messages.append(
+            "branch(es) connected across multiple interconnections: "
+            f"{non_matching_ids}."
+        )
+
+
+def _check_transformer_substations(grid, error_messages):
+    """Check whether any transformers are are not within exactly one same substation.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        any transformers in the ``grid`` are not within exactly one same substation.
+    """
+    txfmr_branch_types = {"Transformer", "TransformerWinding"}
+    branch = grid.branch
+    transformers = branch.loc[branch.branch_device_type.isin(txfmr_branch_types)]
+    from_sub = transformers.from_bus_id.map(grid.bus2sub.sub_id)
+    to_sub = transformers.to_bus_id.map(grid.bus2sub.sub_id)
+    if not all(from_sub == to_sub):
+        non_matching_transformers = transformers.index[from_sub != to_sub]
+        error_messages.append(
+            "transformer(s) connected across multiple substations: "
+            f"{non_matching_transformers}."
+        )
+
+
+def _check_line_voltages(grid, error_messages):
+    """Check whether any lines connect across different voltage levels.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        any lines in the ``grid`` connect across different voltage levels.
+    """
+    lines = grid.branch.query("branch_device_type == 'Line'")
+    from_kV = lines.from_bus_id.map(grid.bus.baseKV)  # noqa: N806
+    to_kV = lines.to_bus_id.map(grid.bus.baseKV)  # noqa: N806
+    if not all(from_kV == to_kV):
+        non_matching_lines = lines.index[from_kV != to_kV]
+        error_messages.append(
+            f"line(s) connected across multiple voltages: {non_matching_lines}."
+        )
+
+
+def _check_plant_against_gencost(grid, error_messages):
+    """Check whether indices of plant and gencost tables match.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        indices of plant and gencost tables of the ``grid`` don't match.
+    """
+    if not (
+        set(grid.plant.index)
+        == set(grid.gencost["before"].index)
+        == set(grid.gencost["after"].index)
+    ):
+        error_messages.append("indices for plant and gencost don't match.")
+
+
+def _check_connected_components(grid, error_messages):
+    """Check whether connected components and listed interconnects of a grid match.
+
+    :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
+    :param list error_messages: list, to be appended to with a str if:
+        connected components and listed interconnects of a ``grid`` don't match.
+    """
+    g = nx.from_pandas_edgelist(grid.branch, "from_bus_id", "to_bus_id")
+    num_connected_components = len([c for c in nx.connected_components(g)])
+    if len(grid.interconnect) == 1:
+        # Check for e.g. ['USA'] interconnect, which is really three interconnects
+        interconnect_aliases = grid.model_immutables.zones["interconnect_combinations"]
+        if grid.interconnect[0] in interconnect_aliases:
+            num_interconnects = len(interconnect_aliases[grid.interconnect[0]])
+        else:
+            num_interconnects = 1
+    else:
+        num_interconnects = len(grid.interconnect)
+    if num_interconnects != num_connected_components:
+        error_messages.append(
+            f"This grid contains {num_connected_components} connected components, "
+            f"but is specified as having {num_interconnects} interconnects: "
+            f"{grid.interconnect}."
+        )

From bdaae0e7eb56065f8b1cf5772ed73c5f3b63710a Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Fri, 4 Jun 2021 11:13:10 -0700
Subject: [PATCH 50/60] test: add tests for grid check

---
 powersimdata/input/tests/test_check.py | 27 ++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)
 create mode 100644 powersimdata/input/tests/test_check.py

diff --git a/powersimdata/input/tests/test_check.py b/powersimdata/input/tests/test_check.py
new file mode 100644
index 000000000..0e446a19a
--- /dev/null
+++ b/powersimdata/input/tests/test_check.py
@@ -0,0 +1,27 @@
+from powersimdata import Grid
+from powersimdata.input.check import check_grid
+
+
+def test_check_eastern():
+    grid = Grid("Eastern")
+    check_grid(grid)
+
+
+def test_check_western():
+    grid = Grid("Western")
+    check_grid(grid)
+
+
+def test_check_texas():
+    grid = Grid("Texas")
+    check_grid(grid)
+
+
+def test_check_western_texas():
+    grid = Grid(["Western", "Texas"])
+    check_grid(grid)
+
+
+def test_check_usa():
+    grid = Grid(["USA"])
+    check_grid(grid)

From bb8733cc5bda7cdc4da4f080c0ec9eb78dd56747 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Fri, 4 Jun 2021 11:22:41 -0700
Subject: [PATCH 51/60] data: change three Western offshore bus voltages to
 satisfy grid check

---
 powersimdata/network/usa_tamu/data/bus.csv | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/powersimdata/network/usa_tamu/data/bus.csv b/powersimdata/network/usa_tamu/data/bus.csv
index 7c211311a..4debec968 100644
--- a/powersimdata/network/usa_tamu/data/bus.csv
+++ b/powersimdata/network/usa_tamu/data/bus.csv
@@ -80051,13 +80051,13 @@ bus_id,type,Pd,Qd,Gs,Bs,zone_id,Vm,Va,baseKV,loss_zone,Vmax,Vmin,lam_P,lam_Q,mu_
 2090002,2,0,0,0,0,206,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
 2090003,2,0,0,0,0,206,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
 2090004,2,0,0,0,0,205,1.0,0.0,138,1,1.1,0.9,0,0,0,0,Western
-2090005,2,0,0,0,0,205,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
+2090005,2,0,0,0,0,205,1.0,0.0,138,1,1.1,0.9,0,0,0,0,Western
 2090006,2,0,0,0,0,204,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
 2090007,2,0,0,0,0,204,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
-2090008,2,0,0,0,0,204,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
+2090008,2,0,0,0,0,204,1.0,0.0,138,1,1.1,0.9,0,0,0,0,Western
 2090009,2,0,0,0,0,203,1.0,0.0,138,1,1.1,0.9,0,0,0,0,Western
 2090010,2,0,0,0,0,203,1.0,0.0,138,1,1.1,0.9,0,0,0,0,Western
-2090011,2,0,0,0,0,203,1.0,0.0,138,1,1.1,0.9,0,0,0,0,Western
+2090011,2,0,0,0,0,203,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
 2090012,2,0,0,0,0,203,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
 2090013,2,0,0,0,0,203,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western
 2090014,2,0,0,0,0,203,1.0,0.0,345,1,1.1,0.9,0,0,0,0,Western

From 09f3938d7f676621d9bfa74a503e7dca427c2b3e Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 7 Jun 2021 10:57:35 -0700
Subject: [PATCH 52/60] chore: raise early if wrong type

---
 powersimdata/input/check.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/powersimdata/input/check.py b/powersimdata/input/check.py
index 0b55c2409..5bb7758e0 100644
--- a/powersimdata/input/check.py
+++ b/powersimdata/input/check.py
@@ -8,10 +8,11 @@ def check_grid(grid):
 
     :param powersimdata.input.grid.Grid grid: grid or grid-like object to check.
     :raises TypeError: if ``grid`` is not a Grid object.
+    :raises ValueError: if ``grid`` has any inconsistency
     """
-    error_messages = []
     if not isinstance(grid, Grid):
-        error_messages.append("grid must be a Grid object")
+        raise TypeError("grid must be a Grid object")
+    error_messages = []
     _check_attributes(grid, error_messages)
     _check_for_islanded_buses(grid, error_messages)
     _check_for_undescribed_buses(grid, error_messages)

From 470477d1921a2597e6b64e80342d04099ed7ce70 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 7 Jun 2021 12:54:42 -0700
Subject: [PATCH 53/60] chore: error handling for each grid check

---
 powersimdata/input/check.py            | 28 +++++++++++++++++---------
 powersimdata/input/tests/test_check.py |  9 +++++++++
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/powersimdata/input/check.py b/powersimdata/input/check.py
index 5bb7758e0..a08932bc4 100644
--- a/powersimdata/input/check.py
+++ b/powersimdata/input/check.py
@@ -1,3 +1,5 @@
+import sys
+
 import networkx as nx
 
 from powersimdata.input.grid import Grid
@@ -13,15 +15,23 @@ def check_grid(grid):
     if not isinstance(grid, Grid):
         raise TypeError("grid must be a Grid object")
     error_messages = []
-    _check_attributes(grid, error_messages)
-    _check_for_islanded_buses(grid, error_messages)
-    _check_for_undescribed_buses(grid, error_messages)
-    _check_bus_against_bus2sub(grid, error_messages)
-    _check_ac_interconnects(grid, error_messages)
-    _check_transformer_substations(grid, error_messages)
-    _check_line_voltages(grid, error_messages)
-    _check_plant_against_gencost(grid, error_messages)
-    _check_connected_components(grid, error_messages)
+    for check in [
+        _check_attributes,
+        _check_for_islanded_buses,
+        _check_for_undescribed_buses,
+        _check_bus_against_bus2sub,
+        _check_ac_interconnects,
+        _check_transformer_substations,
+        _check_line_voltages,
+        _check_plant_against_gencost,
+        _check_connected_components,
+    ]:
+        try:
+            check(grid, error_messages)
+        except Exception:
+            error_messages.append(
+                f"Exception during {check.__name__}: {sys.exc_info()}"
+            )
     if len(error_messages) > 0:
         collected = "\n".join(error_messages)
         raise ValueError(f"Problem(s) found with grid:\n{collected}")
diff --git a/powersimdata/input/tests/test_check.py b/powersimdata/input/tests/test_check.py
index 0e446a19a..24cc0bec2 100644
--- a/powersimdata/input/tests/test_check.py
+++ b/powersimdata/input/tests/test_check.py
@@ -1,7 +1,16 @@
+import pytest
+
 from powersimdata import Grid
 from powersimdata.input.check import check_grid
 
 
+def test_error_handling():
+    grid = Grid("Western")
+    del grid.dcline
+    with pytest.raises(ValueError):
+        check_grid(grid)
+
+
 def test_check_eastern():
     grid = Grid("Eastern")
     check_grid(grid)

From ff960c7f338f4bc84ecf39d03da8b0df5931bd52 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 7 Jun 2021 13:18:10 -0700
Subject: [PATCH 54/60] chore: simplify error message

---
 powersimdata/input/check.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/powersimdata/input/check.py b/powersimdata/input/check.py
index a08932bc4..a8290c6c1 100644
--- a/powersimdata/input/check.py
+++ b/powersimdata/input/check.py
@@ -30,7 +30,7 @@ def check_grid(grid):
             check(grid, error_messages)
         except Exception:
             error_messages.append(
-                f"Exception during {check.__name__}: {sys.exc_info()}"
+                f"Exception during {check.__name__}: {sys.exc_info()[1]!r}"
             )
     if len(error_messages) > 0:
         collected = "\n".join(error_messages)

From bc864b660cf6e7ceeefb2fd44b2d54c6341f3c59 Mon Sep 17 00:00:00 2001
From: Jon Hagg <jon.hagg@breakthroughenergy.org>
Date: Mon, 7 Jun 2021 14:29:48 -0700
Subject: [PATCH 55/60] test: parametrize interconnect

---
 powersimdata/input/tests/test_check.py | 27 +++++---------------------
 1 file changed, 5 insertions(+), 22 deletions(-)

diff --git a/powersimdata/input/tests/test_check.py b/powersimdata/input/tests/test_check.py
index 24cc0bec2..004a3a44d 100644
--- a/powersimdata/input/tests/test_check.py
+++ b/powersimdata/input/tests/test_check.py
@@ -11,26 +11,9 @@ def test_error_handling():
         check_grid(grid)
 
 
-def test_check_eastern():
-    grid = Grid("Eastern")
-    check_grid(grid)
-
-
-def test_check_western():
-    grid = Grid("Western")
-    check_grid(grid)
-
-
-def test_check_texas():
-    grid = Grid("Texas")
-    check_grid(grid)
-
-
-def test_check_western_texas():
-    grid = Grid(["Western", "Texas"])
-    check_grid(grid)
-
-
-def test_check_usa():
-    grid = Grid(["USA"])
+@pytest.mark.parametrize(
+    "interconnect", ["Eastern", "Western", "Texas", ["Western", "Texas"], "USA"]
+)
+def test_check_grid(interconnect):
+    grid = Grid(interconnect)
     check_grid(grid)

From 88a179a081c6c55645dc7b1b1e57ed23358ebc39 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Tue, 8 Jun 2021 11:39:57 -0700
Subject: [PATCH 56/60] chore: move Grid column constants to
 powersimdata.input.const

---
 powersimdata/input/const.py | 237 ++++++++++++++++++++++++++++++++++++
 1 file changed, 237 insertions(+)
 create mode 100644 powersimdata/input/const.py

diff --git a/powersimdata/input/const.py b/powersimdata/input/const.py
new file mode 100644
index 000000000..17e087196
--- /dev/null
+++ b/powersimdata/input/const.py
@@ -0,0 +1,237 @@
+col_name_branch = [
+    "from_bus_id",
+    "to_bus_id",
+    "r",
+    "x",
+    "b",
+    "rateA",
+    "rateB",
+    "rateC",
+    "ratio",
+    "angle",
+    "status",
+    "angmin",
+    "angmax",
+    "Pf",
+    "Qf",
+    "Pt",
+    "Qt",
+    "mu_Sf",
+    "mu_St",
+    "mu_angmin",
+    "mu_angmax",
+]
+col_type_branch = [
+    "int",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+]
+
+col_name_bus = [
+    "bus_id",
+    "type",
+    "Pd",
+    "Qd",
+    "Gs",
+    "Bs",
+    "zone_id",
+    "Vm",
+    "Va",
+    "baseKV",
+    "loss_zone",
+    "Vmax",
+    "Vmin",
+    "lam_P",
+    "lam_Q",
+    "mu_Vmax",
+    "mu_Vmin",
+]
+col_type_bus = [
+    "int",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+]
+
+col_name_bus2sub = ["sub_id", "interconnect"]
+col_type_bus2sub = ["int", "str"]
+
+col_name_dcline = [
+    "from_bus_id",
+    "to_bus_id",
+    "status",
+    "Pf",
+    "Pt",
+    "Qf",
+    "Qt",
+    "Vf",
+    "Vt",
+    "Pmin",
+    "Pmax",
+    "QminF",
+    "QmaxF",
+    "QminT",
+    "QmaxT",
+    "loss0",
+    "loss1",
+    "muPmin",
+    "muPmax",
+    "muQminF",
+    "muQmaxF",
+    "muQminT",
+    "muQmaxT",
+]
+col_type_dcline = [
+    "int",
+    "int",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+]
+
+col_name_gencost = ["type", "startup", "shutdown", "n", "c2", "c1", "c0"]
+col_type_gencost = ["int", "float", "float", "int", "float", "float", "float"]
+
+col_name_heat_rate_curve = ["GenIOB", "GenIOC", "GenIOD"]
+col_type_heat_rate_curve = ["float", "float", "float"]
+
+
+col_name_plant = [
+    "bus_id",
+    "Pg",
+    "Qg",
+    "Qmax",
+    "Qmin",
+    "Vg",
+    "mBase",
+    "status",
+    "Pmax",
+    "Pmin",
+    "Pc1",
+    "Pc2",
+    "Qc1min",
+    "Qc1max",
+    "Qc2min",
+    "Qc2max",
+    "ramp_agc",
+    "ramp_10",
+    "ramp_30",
+    "ramp_q",
+    "apf",
+    "mu_Pmax",
+    "mu_Pmin",
+    "mu_Qmax",
+    "mu_Qmin",
+]
+col_type_plant = [
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+]
+
+col_name_storage_storagedata = [
+    "UnitIdx",
+    "InitialStorage",
+    "InitialStorageLowerBound",
+    "InitialStorageUpperBound",
+    "InitialStorageCost",
+    "TerminalStoragePrice",
+    "MinStorageLevel",
+    "MaxStorageLevel",
+    "OutEff",
+    "InEff",
+    "LossFactor",
+    "rho",
+    "ExpectedTerminalStorageMax",
+    "ExpectedTerminalStorageMin",
+]
+col_type_storage_storagedata = [
+    "int",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+    "float",
+]
+
+col_name_sub = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
+col_type_sub = ["str", "int", "float", "float", "str"]

From b350e1dbc7e8b0c38aa1136a3bdd85eb3e5f0125 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Tue, 8 Jun 2021 11:40:21 -0700
Subject: [PATCH 57/60] refactor: use powersimdata.input.const in scenario_grid

---
 powersimdata/input/scenario_grid.py | 223 ++--------------------------
 1 file changed, 15 insertions(+), 208 deletions(-)

diff --git a/powersimdata/input/scenario_grid.py b/powersimdata/input/scenario_grid.py
index 977515f42..449638d9e 100644
--- a/powersimdata/input/scenario_grid.py
+++ b/powersimdata/input/scenario_grid.py
@@ -4,6 +4,7 @@
 import pandas as pd
 from scipy.io import loadmat
 
+from powersimdata.input import const
 from powersimdata.input.abstract_grid import AbstractGrid
 from powersimdata.input.helpers import (
     add_coord_to_grid_data_frames,
@@ -218,111 +219,14 @@ def column_name_provider():
 
     :return: (*dict*) -- dictionary of data frame columns name.
     """
-    col_name_sub = ["name", "interconnect_sub_id", "lat", "lon", "interconnect"]
-    col_name_bus = [
-        "bus_id",
-        "type",
-        "Pd",
-        "Qd",
-        "Gs",
-        "Bs",
-        "zone_id",
-        "Vm",
-        "Va",
-        "baseKV",
-        "loss_zone",
-        "Vmax",
-        "Vmin",
-        "lam_P",
-        "lam_Q",
-        "mu_Vmax",
-        "mu_Vmin",
-    ]
-    col_name_bus2sub = ["sub_id", "interconnect"]
-    col_name_branch = [
-        "from_bus_id",
-        "to_bus_id",
-        "r",
-        "x",
-        "b",
-        "rateA",
-        "rateB",
-        "rateC",
-        "ratio",
-        "angle",
-        "status",
-        "angmin",
-        "angmax",
-        "Pf",
-        "Qf",
-        "Pt",
-        "Qt",
-        "mu_Sf",
-        "mu_St",
-        "mu_angmin",
-        "mu_angmax",
-    ]
-    col_name_dcline = [
-        "from_bus_id",
-        "to_bus_id",
-        "status",
-        "Pf",
-        "Pt",
-        "Qf",
-        "Qt",
-        "Vf",
-        "Vt",
-        "Pmin",
-        "Pmax",
-        "QminF",
-        "QmaxF",
-        "QminT",
-        "QmaxT",
-        "loss0",
-        "loss1",
-        "muPmin",
-        "muPmax",
-        "muQminF",
-        "muQmaxF",
-        "muQminT",
-        "muQmaxT",
-    ]
-    col_name_plant = [
-        "bus_id",
-        "Pg",
-        "Qg",
-        "Qmax",
-        "Qmin",
-        "Vg",
-        "mBase",
-        "status",
-        "Pmax",
-        "Pmin",
-        "Pc1",
-        "Pc2",
-        "Qc1min",
-        "Qc1max",
-        "Qc2min",
-        "Qc2max",
-        "ramp_agc",
-        "ramp_10",
-        "ramp_30",
-        "ramp_q",
-        "apf",
-        "mu_Pmax",
-        "mu_Pmin",
-        "mu_Qmax",
-        "mu_Qmin",
-    ]
-    col_name_heat_rate_curve = ["GenIOB", "GenIOC", "GenIOD"]
     col_name = {
-        "sub": col_name_sub,
-        "bus": col_name_bus,
-        "bus2sub": col_name_bus2sub,
-        "branch": col_name_branch,
-        "dcline": col_name_dcline,
-        "plant": col_name_plant,
-        "heat_rate_curve": col_name_heat_rate_curve,
+        "sub": const.col_name_sub,
+        "bus": const.col_name_bus,
+        "bus2sub": const.col_name_bus2sub,
+        "branch": const.col_name_branch,
+        "dcline": const.col_name_dcline,
+        "plant": const.col_name_plant,
+        "heat_rate_curve": const.col_name_heat_rate_curve,
     }
     return col_name
 
@@ -332,111 +236,14 @@ def column_type_provider():
 
     :return: (*dict*) -- dictionary of data frame columns type.
     """
-    col_type_sub = ["str", "int", "float", "float", "str"]
-    col_type_bus = [
-        "int",
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "int",
-        "float",
-        "float",
-        "float",
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-    ]
-    col_type_bus2sub = ["int", "str"]
-    col_type_branch = [
-        "int",
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-    ]
-    col_type_dcline = [
-        "int",
-        "int",
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-    ]
-    col_type_plant = [
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "int",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-        "float",
-    ]
-    col_type_heat_rate_curve = ["float", "float", "float"]
     col_type = {
-        "sub": col_type_sub,
-        "bus": col_type_bus,
-        "bus2sub": col_type_bus2sub,
-        "branch": col_type_branch,
-        "dcline": col_type_dcline,
-        "plant": col_type_plant,
-        "heat_rate_curve": col_type_heat_rate_curve,
+        "sub": const.col_type_sub,
+        "bus": const.col_type_bus,
+        "bus2sub": const.col_type_bus2sub,
+        "branch": const.col_type_branch,
+        "dcline": const.col_type_dcline,
+        "plant": const.col_type_plant,
+        "heat_rate_curve": const.col_type_heat_rate_curve,
     }
     return col_type
 

From 6b8a16d181511bd1f74e490c7f77058873dc3966 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Tue, 8 Jun 2021 11:44:23 -0700
Subject: [PATCH 58/60] refactor: use powersimdata.input.const in abstract_grid

---
 powersimdata/input/abstract_grid.py | 55 +++--------------------------
 1 file changed, 5 insertions(+), 50 deletions(-)

diff --git a/powersimdata/input/abstract_grid.py b/powersimdata/input/abstract_grid.py
index 4ca10998b..5c6525507 100644
--- a/powersimdata/input/abstract_grid.py
+++ b/powersimdata/input/abstract_grid.py
@@ -1,5 +1,7 @@
 import pandas as pd
 
+from powersimdata.input import const
+
 
 class AbstractGrid:
     """Grid Builder."""
@@ -26,56 +28,9 @@ def storage_template():
     :return: (*dict*) -- storage structure for MATPOWER/MOST
     """
     storage = {
-        "gen": pd.DataFrame(
-            columns=[
-                "bus_id",
-                "Pg",
-                "Qg",
-                "Qmax",
-                "Qmin",
-                "Vg",
-                "mBase",
-                "status",
-                "Pmax",
-                "Pmin",
-                "Pc1",
-                "Pc2",
-                "Qc1min",
-                "Qc1max",
-                "Qc2min",
-                "Qc2max",
-                "ramp_agc",
-                "ramp_10",
-                "ramp_30",
-                "ramp_q",
-                "apf",
-                "mu_Pmax",
-                "mu_Pmin",
-                "mu_Qmax",
-                "mu_Qmin",
-            ]
-        ),
-        "gencost": pd.DataFrame(
-            columns=["type", "startup", "shutdown", "n", "c2", "c1", "c0"]
-        ),
-        "StorageData": pd.DataFrame(
-            columns=[
-                "UnitIdx",
-                "InitialStorage",
-                "InitialStorageLowerBound",
-                "InitialStorageUpperBound",
-                "InitialStorageCost",
-                "TerminalStoragePrice",
-                "MinStorageLevel",
-                "MaxStorageLevel",
-                "OutEff",
-                "InEff",
-                "LossFactor",
-                "rho",
-                "ExpectedTerminalStorageMax",
-                "ExpectedTerminalStorageMin",
-            ]
-        ),
+        "gen": pd.DataFrame(columns=const.col_name_plant),
+        "gencost": pd.DataFrame(columns=const.col_name_gencost),
+        "StorageData": pd.DataFrame(columns=const.col_name_storage_storagedata),
         "genfuel": [],
         "duration": None,  # hours
         "min_stor": None,  # ratio

From b2e96ba78e9b3051d6e6ae988301d918d559ac63 Mon Sep 17 00:00:00 2001
From: Daniel Olsen <daniel.olsen@breakthroughenergy.org>
Date: Tue, 8 Jun 2021 11:50:14 -0700
Subject: [PATCH 59/60] refactor: use new powersimdata.input.const in mock_grid

---
 powersimdata/tests/mock_grid.py | 127 ++++----------------------------
 1 file changed, 13 insertions(+), 114 deletions(-)

diff --git a/powersimdata/tests/mock_grid.py b/powersimdata/tests/mock_grid.py
index ed90248c9..4092dd012 100644
--- a/powersimdata/tests/mock_grid.py
+++ b/powersimdata/tests/mock_grid.py
@@ -1,5 +1,6 @@
 import pandas as pd
 
+from powersimdata.input import const
 from powersimdata.input.grid import Grid
 from powersimdata.network.model import ModelImmutables
 
@@ -24,28 +25,7 @@
 
 bus2sub_columns = ["sub_id", "interconnect"]
 
-branch_columns = [
-    "from_bus_id",
-    "to_bus_id",
-    "r",
-    "x",
-    "b",
-    "rateA",
-    "rateB",
-    "rateC",
-    "ratio",
-    "angle",
-    "status",
-    "angmin",
-    "angmax",
-    "Pf",
-    "Qf",
-    "Pt",
-    "Qt",
-    "mu_Sf",
-    "mu_St",
-    "mu_angmin",
-    "mu_angmax",
+branch_augment_columns = [
     "branch_device_type",
     "interconnect",
     "from_zone_id",
@@ -57,85 +37,18 @@
     "to_lat",
     "to_lon",
 ]
+branch_columns = const.col_name_branch + branch_augment_columns
 
-bus_columns = [
-    "type",
-    "Pd",
-    "Qd",
-    "Gs",
-    "Bs",
-    "zone_id",
-    "Vm",
-    "Va",
-    "loss_zone",
-    "baseKV",
-    "Vmax",
-    "Vmin",
-    "lam_P",
-    "lam_Q",
-    "mu_Vmax",
-    "mu_Vmin",
-    "interconnect",
-    "lat",
-    "lon",
-]
+bus_augment_columns = ["interconnect", "lat", "lon"]
+bus_columns = const.col_name_bus + bus_augment_columns
 
-dcline_columns = [
-    "from_bus_id",
-    "to_bus_id",
-    "status",
-    "Pf",
-    "Pt",
-    "Qf",
-    "Qt",
-    "Vf",
-    "Vt",
-    "Pmin",
-    "Pmax",
-    "QminF",
-    "QmaxF",
-    "QminT",
-    "QmaxT",
-    "loss0",
-    "loss1",
-    "muPmin",
-    "muPmax",
-    "muQminF",
-    "muQmaxF",
-    "muQminT",
-    "muQmaxT",
-    "from_interconnect",
-    "to_interconnect",
-]
+dcline_augment_columns = ["from_interconnect", "to_interconnect"]
+dcline_columns = const.col_name_dcline + dcline_augment_columns
+
+gencost_augment_columns = ["interconnect"]
+gencost_columns = const.col_name_gencost + gencost_augment_columns
 
-gencost_columns = ["type", "startup", "shutdown", "n", "c2", "c1", "c0", "interconnect"]
-
-plant_columns = [
-    "bus_id",
-    "Pg",
-    "Qg",
-    "Qmax",
-    "Qmin",
-    "Vg",
-    "mBase",
-    "status",
-    "Pmax",
-    "Pmin",
-    "Pc1",
-    "Pc2",
-    "Qc1min",
-    "Qc1max",
-    "Qc2min",
-    "Qc2max",
-    "ramp_agc",
-    "ramp_10",
-    "ramp_30",
-    "ramp_q",
-    "apf",
-    "mu_Pmax",
-    "mu_Pmin",
-    "mu_Qmax",
-    "mu_Qmin",
+plant_augment_columns = [
     "type",
     "interconnect",
     "GenFuelCost",
@@ -147,26 +60,12 @@
     "lat",
     "lon",
 ]
+plant_columns = const.col_name_plant + plant_augment_columns
 
 storage_columns = {
     # The first 21 columns of plant are all that's necessary
     "gen": plant_columns[:21],
-    "StorageData": [
-        "UnitIdx",
-        "InitialStorage",
-        "InitialStorageLowerBound",
-        "InitialStorageUpperBound",
-        "InitialStorageCost",
-        "TerminalStoragePrice",
-        "MinStorageLevel",
-        "MaxStorageLevel",
-        "OutEff",
-        "InEff",
-        "LossFactor",
-        "rho",
-        "ExpectedTerminalStorageMax",
-        "ExpectedTerminalStorageMin",
-    ],
+    "StorageData": const.col_name_storage_storagedata,
 }
 
 

From 122516d84bd3bd375dca15bb571762dd41431ef4 Mon Sep 17 00:00:00 2001
From: danielolsen <danielolsen@users.noreply.github.com>
Date: Mon, 14 Jun 2021 13:30:35 -0700
Subject: [PATCH 60/60] chore: bump version number to v0.4.2 (#499)

---
 setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 15433827d..9f771df26 100644
--- a/setup.py
+++ b/setup.py
@@ -12,7 +12,7 @@
 
 setup(
     name="powersimdata",
-    version="0.4.1",
+    version="0.4.2",
     description="Power Simulation Data",
     url="https://github.com/Breakthrough-Energy/powersimdata",
     author="Kaspar Mueller",