Merge pull request hackforla#56 from hackforla/issues_with_missing_la…

…bels_over_time

add script to audit issues with missing labels

.github/workflows/schedule_run_to_update_data.yaml

@@ -56,4 +56,11 @@ jobs:
           API_KEY_GITHUB_PROJECTBOARD_DASHBOARD: ${{ secrets.API_KEY_GITHUB_PROJECTBOARD_DASHBOARD }}
           BASE64_PROJECT_BOARD_GOOGLECREDENTIAL: ${{ secrets.BASE64_PROJECT_BOARD_GOOGLECREDENTIAL }}
           API_TOKEN_USERNAME: ${{ secrets.API_TOKEN_USERNAME }}
-        run: python Project_Board_Dashboard_Script.py
+        run: python Project_Board_Dashboard_Script.py
+
+      - name: setup environment and run issues_with_missing_labels_over_time.py
+        env:
+          API_KEY_GITHUB_PROJECTBOARD_DASHBOARD: ${{ secrets.API_KEY_GITHUB_PROJECTBOARD_DASHBOARD }}
+          BASE64_PROJECT_BOARD_GOOGLECREDENTIAL: ${{ secrets.BASE64_PROJECT_BOARD_GOOGLECREDENTIAL }}
+          API_TOKEN_USERNAME: ${{ secrets.API_TOKEN_USERNAME }}
+        run: python issues_with_missing_labels_over_time.py

.gitignore

@@ -0,0 +1,160 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/#use-with-ide
+.pdm.toml
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

issues_with_missing_labels_over_time.py

@@ -0,0 +1,75 @@
+import base64
+import json
+import os
+from urllib.parse import parse_qs, urlparse
+
+import duckdb
+import requests
+from shillelagh.backends.apsw.db import connect
+
+
+github_token = os.environ["API_KEY_GITHUB_PROJECTBOARD_DASHBOARD"]
+github_user = os.environ["API_TOKEN_USERNAME"]
+response = requests.get("https://api.github.com/repos/hackforla/website/issues?state=all", auth=("hackforla", github_token))
+issues = response.json()
+
+# Link format
+# '<https://api.github.com/repositories/130000551/issues?page=2>; rel="next"'
+links = response.headers["Link"]
+links = links.split(",")
+next_link = links[1].split(";")[0].replace("<", "").replace(">", "").strip()
+last = parse_qs(urlparse(next_link).query)["page"][0]
+print("Last page:", last)
+page = 2
+while page <= int(last):
+    print(f"Fetching page: {page}/{last}")
+    response = requests.get("https://api.github.com/repos/hackforla/website/issues?state=all&page=" + str(page), headers={"Authorization": "Bearer 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"})
+    issues.extend(response.json())
+    page += 1
+print("Number of issues:", len(issues))
+
+for issue in issues:
+    issue["labels"] = ", ".join([label["name"] for label in issue["labels"]])
+
+with open("issues.json", "w") as f:
+    json.dump(issues, f)
+
+duckdb.read_json("issues.json")
+df = duckdb.sql(
+    """
+            SELECT
+                CURRENT_DATE as "Date",
+                SUM(CASE WHEN labels LIKE '%role missing%' AND state = 'open' THEN 1 ELSE 0 END) as "Role, Open",
+                SUM(CASE WHEN labels LIKE '%role missing%' AND state = 'closed' THEN 1 ELSE 0 END) as "Role, Closed",
+                SUM(CASE WHEN labels LIKE '%Complexity: Missing%' AND state = 'open' THEN 1 ELSE 0 END) as "Complexity, Open",
+                SUM(CASE WHEN labels LIKE '%Complexity: Missing%' AND state = 'closed' THEN 1 ELSE 0 END) as "Complexity, Closed",
+                SUM(CASE WHEN labels LIKE '%size: missing%' AND state = 'open' THEN 1 ELSE 0 END) as "Size, Open",
+                SUM(CASE WHEN labels LIKE '%size: missing%' AND state = 'closed' THEN 1 ELSE 0 END) as "Size, Closed",
+                SUM(CASE WHEN labels LIKE '%Feature Missing%' AND state = 'open' THEN 1 ELSE 0 END) as "Feature, Open",
+                SUM(CASE WHEN labels LIKE '%Feature Missing%' AND state = 'closed' THEN 1 ELSE 0 END) as "Feature, Closed"
+            FROM 'issues.json'
+        """
+).df()
+print(df)
+
+key_base64 = os.environ["BASE64_PROJECT_BOARD_GOOGLECREDENTIAL"]
+base64_bytes = key_base64.encode("ascii")
+key_base64_bytes = base64.b64decode(base64_bytes)
+key_content = key_base64_bytes.decode("ascii")
+
+service_account_info = json.loads(key_content)
+connection = connect(
+    ":memory:",
+    adapter_kwargs={
+        "gsheetsapi": {
+            "service_account_info": service_account_info,
+        },
+    },
+)
+
+cursor = connection.cursor()
+SQL = """
+INSERT INTO "https://docs.google.com/spreadsheets/d/16yC91C_ZTJoAhG0qVWqpEZ9kREPraubcARfZi9bkFcY/edit#gid=0"
+SELECT * FROM df
+"""
+cursor.execute(SQL)

requirements.txt

@@ -6,6 +6,10 @@ gspread == 5.9.0
 IPython
 gspread-dataframe
 pydrive
+shillelagh
+shillelagh[gsheetsapi]
+requests-cache
+duckdb
 
 # pip install --upgrade google-auth
 # pip install --upgrade google-api-python-client