Merge branch 'main' into fix-nox

laminlabs · Feb 10, 2025 · acb9772 · acb9772
2 parents 20bf60c + 7406925
commit acb9772
Show file tree

Hide file tree

Showing 18 changed files with 267 additions and 120 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -14,7 +14,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8"] # we want to be able to run this on 3.8
+        python-version: ["3.12"]
     timeout-minutes: 10
 
     steps:

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,64 +1,51 @@
+fail_fast: false
+default_language_version:
+  python: python3
+default_stages:
+  - pre-commit
+  - pre-push
+minimum_pre_commit_version: 2.12.0
 repos:
-  - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.2.0
-    hooks:
-      - id: trailing-whitespace
-      - id: end-of-file-fixer
-        exclude: |
-          (?x)(
-              .github/workflows/latest-changes.jinja2
-          )
-      - id: check-yaml
-      - id: check-added-large-files
-  - repo: https://github.com/psf/black
-    rev: 22.6.0
-    hooks:
-      - id: black-jupyter
-  - repo: https://github.com/pycqa/flake8
-    rev: 4.0.1
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: v4.0.0-alpha.4
     hooks:
-      - id: flake8
-        additional_dependencies:
-          - flake8-black==0.3.3
-          - flake8-typing-imports==1.10.0
-        language_version: python3
-        args:
-          - --max-line-length=88
-          - --ignore=E203,W503,BLK100,TYP001
+      - id: prettier
         exclude: |
           (?x)(
-              __init__.py
+            docs/changelog.md
           )
-  - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: v2.6.2
-    hooks:
-      - id: prettier
   - repo: https://github.com/kynan/nbstripout
-    rev: 0.3.9
+    rev: 0.6.1
     hooks:
       - id: nbstripout
         exclude: |
           (?x)(
               docs/examples/|
               docs/notes/
           )
-  - repo: https://github.com/Lucas-C/pre-commit-hooks
-    rev: v1.1.9
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: v0.9.1
     hooks:
-      - id: forbid-crlf
-      - id: remove-crlf
-  - repo: https://github.com/pre-commit/mirrors-isort
-    rev: v5.8.0
+      - id: ruff
+        args: [--fix, --exit-non-zero-on-fix, --unsafe-fixes]
+      - id: ruff-format
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.5.0
     hooks:
-      - id: isort
-        args: ["--profile", "black"]
+      - id: detect-private-key
+      - id: check-ast
+      - id: end-of-file-fixer
+        exclude: |
+          (?x)(
+              .github/workflows/latest-changes.jinja2
+            )
+      - id: mixed-line-ending
+        args: [--fix=lf]
+      - id: trailing-whitespace
+      - id: check-case-conflict
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v0.940
+    rev: v1.14.1
     hooks:
       - id: mypy
-  - repo: https://github.com/pycqa/pydocstyle
-    rev: 6.1.1
-    hooks:
-      - id: pydocstyle
-        args: # google style + __init__, see http://www.pydocstyle.org/en/stable/error_codes.html
-          - --ignore=D100,D101,D102,D103,D106,D107,D203,D204,D213,D215,D400,D401,D403,D404,D406,D407,D408,D409,D412,D413
+        args: [--no-strict-optional, --ignore-missing-imports]
+        additional_dependencies: ["types-requests", "types-attrs"]
diff --git a/README.md b/README.md
@@ -1,12 +1,13 @@
 # laminci: Internal CI tools
 
-
 From PyPI (sometimes outdated):
+
 ```
 pip install laminci
 ```
 
 From GitHub:
+
 ```
 pip install git+https://github.com/laminlabs/laminci
 ```
diff --git a/docs/changelog.md b/docs/changelog.md
@@ -3,6 +3,10 @@
 <!-- prettier-ignore -->
 Name | PR | Developer | Date | Version
 --- | --- | --- | --- | ---
+No longer install ourprojects  | [47](https://github.com/laminlabs/laminci/pull/47) | [Zethson](https://github.com/Zethson) | 2025-01-13 |
+🐛 Install consistent state | [46](https://github.com/laminlabs/laminci/pull/46) | [falexwolf](https://github.com/falexwolf) | 2025-01-11 |
+🎨 Remove lnschema-core installation | [44](https://github.com/laminlabs/laminci/pull/44) | [Zethson](https://github.com/Zethson) | 2025-01-07 |
+♻️ Name docs artifacts by repo-name instead of package name | [42](https://github.com/laminlabs/laminci/pull/42) | [falexwolf](https://github.com/falexwolf) | 2024-09-07 |
 ✨ Add install_lamindb to install lamindb from github | [40](https://github.com/laminlabs/laminci/pull/40) | [Koncopd](https://github.com/Koncopd) | 2024-07-26 |
 🚸 Publish to laminhub-public | [41](https://github.com/laminlabs/laminci/pull/41) | [falexwolf](https://github.com/falexwolf) | 2024-07-22 |
 ✨ Manage UI releases | [39](https://github.com/laminlabs/laminci/pull/39) | [falexwolf](https://github.com/falexwolf) | 2024-07-06 |

diff --git a/docs/notes/YYYY-MM-DD-my-design-choice.ipynb b/docs/notes/YYYY-MM-DD-my-design-choice.ipynb
@@ -14,9 +14,7 @@
    "id": "97ec6631-8473-4a2d-b488-def921bb83de",
    "metadata": {},
    "outputs": [],
-   "source": [
-    "from nbproject import header"
-   ]
+   "source": []
   }
  ],
  "metadata": {

diff --git a/laminci/__init__.py b/laminci/__init__.py
@@ -1,20 +1,18 @@
-"""Internal CI tools.
+"""Lamin's CI tools.
 
 Import the package::
 
    import laminci
 
 """
 
-__version__ = "0.13.2"  # denote a pre-release for 0.1.0 with 0.1a1
+__version__ = "0.14.0"  # denote a pre-release for 0.1.0 with 0.1a1
 
 from . import db, nox
-from ._artifacts import upload_docs_artifact
 from ._docs import (
     move_built_docs_to_docs_slash_project_slug,
     move_built_docs_to_slash_project_slug,
 )
+from ._docs_artifacts import upload_docs_artifact
 from ._env import get_package_name, get_schema_handle
-
-# backward compat
-upload_docs_dir = upload_docs_artifact
+from ._run_notebooks import run_notebooks
diff --git a/laminci/__main__.py b/laminci/__main__.py
@@ -8,24 +8,24 @@
 import subprocess
 from pathlib import Path
 from subprocess import PIPE, run
-from typing import Union
 
 from packaging.version import Version, parse
 
 from ._env import get_package_name
 
 parser = argparse.ArgumentParser("laminci")
 subparsers = parser.add_subparsers(dest="command")
-migr = subparsers.add_parser(
+release = subparsers.add_parser(
     "release",
     help="Help with release",
     description=(
         "Assumes you manually prepared the release commit!\n\n"
         "Please edit the version number in your package and prepare the release notes!"
     ),
 )
-aa = migr.add_argument
+aa = release.add_argument
 aa("--pypi", default=False, action="store_true", help="Publish to PyPI")
+aa("--changelog", default=None, help="Link to changelog entry")
 subparsers.add_parser(
     "doc-changes",
     help="Write latest changes",
@@ -34,7 +34,7 @@
 
 def update_readme_version(file_path, new_version):
     # Read the content of the file
-    with open(file_path, "r") as file:
+    with open(file_path) as file:
         content = file.read()
 
     # Use regex to find and replace the version
@@ -48,7 +48,7 @@ def update_readme_version(file_path, new_version):
 
 
 def get_last_version_from_tags():
-    proc = run(["git", "tag"], universal_newlines=True, stdout=PIPE)
+    proc = run(["git", "tag"], text=True, stdout=PIPE)
     tags = proc.stdout.splitlines()
     newest = "0.0.0"
     for tag in tags:
@@ -63,16 +63,18 @@ def validate_version(version_str: str):
         if not len(version.release) == 2:
             raise SystemExit(
                 f"Pre-releases should be of form 0.42a1 or 0.42rc1, yours is {version}"
-            )
+            ) from None
         else:
             return None
     if len(version.release) != 3:
-        raise SystemExit(f"Version should be of form 0.1.2, yours is {version}")
+        raise SystemExit(
+            f"Version should be of form 0.1.2, yours is {version}"
+        ) from None
 
 
 def publish_github_release(
     repo_name: str,
-    version: Union[str, Version],
+    version: str | Version,
     release_name: str,
     body: str = "",
     draft: bool = False,
@@ -84,7 +86,9 @@ def publish_github_release(
     try:
         cwd = Path.cwd() if cwd is None else Path(cwd)
         # account for repo_name sometimes being a package
-        assert repo_name.split("/")[1].replace("_", "-") == cwd.name
+        repo_name_standardized = repo_name.split("/")[1]
+        if not repo_name_standardized == cwd.name:
+            raise ValueError(f"Don't match: {repo_name_standardized} != {cwd.name}")
         subprocess.run(["gh", "--version"], check=True, stdout=subprocess.PIPE, cwd=cwd)
         try:
             command = [
@@ -105,7 +109,8 @@ def publish_github_release(
             print(f"\nrun: {' '.join(command)}")
             subprocess.run(command, check=True, stdout=subprocess.PIPE, cwd=cwd)
         except (subprocess.CalledProcessError, FileNotFoundError) as e:
-            raise SystemExit(f"Error creating GitHub release using `gh`: {e}")
+            if input("GitHub release error. Continue? ") != "y":
+                raise e
     except (subprocess.CalledProcessError, FileNotFoundError):
         try:
             from github import Github, GithubException
@@ -128,12 +133,14 @@ def publish_github_release(
                     generate_release_notes=generate_release_notes,
                 )
             except GithubException as e:
-                raise SystemExit(f"Error creating GitHub release using `PyGithub`: {e}")
+                raise SystemExit(
+                    f"Error creating GitHub release using `PyGithub`: {e}"
+                ) from None
         except ImportError:
             raise SystemExit(
                 "Neither the Github CLI ('gh') nor PyGithub were accessible.\n"
                 "Please install one of the two."
-            )
+            ) from None
 
 
 def main():
@@ -155,17 +162,35 @@ def main():
                     f"Your version ({version}) should increment the previous version"
                     f" ({previous_version})"
                 )
+            if package_name == "lamindb" and args.changelog is None:
+                raise SystemExit(
+                    "Please pass a link to the changelog entry via: --changelog"
+                    " 'your-link'"
+                ) from None
+            if Path("./LICENSE").exists() and not args.pypi:
+                raise SystemExit(
+                    "ERROR: Did you forget to add the `--pypi` flag? A LICENSE file"
+                    " exists and I assume this is an open-source package."
+                ) from None
+            repo_name = package_name.replace("_", "-")
         else:
-            assert Path.cwd().name == "laminhub"
+            assert Path.cwd().name == "laminhub"  # noqa: S101
+            repo_name = "laminhub"
             if not (Path.cwd().parent / "laminhub-public").exists():
                 raise ValueError(
                     "Please clone the laminhub-public repository into the same parent"
                     " directory as laminhub."
                 )
             is_laminhub = True
-            with open("ui/package.json", "r") as file:
+            with open("ui/package.json") as file:
                 version = json.load(file)["version"]
 
+        print(f"INFO: You will add this changelog link: {args.changelog}")
+        print(
+            "WARNING: This will run `git add -u` & commit everything into the release"
+            " commit. Please ensure all your current changes should appear in the"
+            " release commit. Typically, you only bump the version number. "
+        )
         pypi = " & publish to PyPI" if args.pypi else ""
         response = input(f"Bump {previous_version} to {version}{pypi}? (y/n)")
         if response != "y":
@@ -174,37 +199,45 @@ def main():
         commands = [
             "git add -u",
             f"git commit -m 'Release {version}'",
+            "git pull",
             "git push",
             f"git tag {version}",
             f"git push origin {version}",
         ]
         for command in commands:
             print(f"\nrun: {command}")
-            run(command, shell=True)
+            run(command, shell=True)  # noqa: S602
 
+        changelog_link = (
+            args.changelog
+            if args.changelog is not None
+            else "https://docs.lamin.ai/changelog"
+        )
+        # breakpoint()
         publish_github_release(
-            repo_name=f"laminlabs/{package_name}",
+            repo_name=f"laminlabs/{repo_name}",
             version=version,
             release_name=f"Release {version}",
-            body="See https://docs.lamin.ai/changelog",
+            body=f"See {changelog_link}",
         )
         if is_laminhub:
+            update_readme_version("../laminhub-public/README.md", version)
             for command in commands:
                 print(f"\nrun: {command}")
-                update_readme_version("../laminhub-public/README.md", version)
-                run(command, shell=True, cwd="../laminhub-public")
+                run(command, shell=True, cwd="../laminhub-public")  # noqa: S602
             publish_github_release(
                 repo_name="laminlabs/laminhub-public",
                 version=version,
-                body="See https://docs.lamin.ai/changelog",
+                body=f"See {changelog_link}",
                 release_name=f"Release {version}",
                 generate_release_notes=False,
+                cwd="../laminhub-public",
             )
 
         if args.pypi:
             command = "flit publish"
             print(f"\nrun: {command}")
-            run(command, shell=True)
+            run(command, shell=True)  # noqa: S602
     elif args.command == "doc-changes":
         from ._doc_changes import doc_changes
 

diff --git a/laminci/_db.py b/laminci/_db.py
@@ -25,9 +25,9 @@ def setup_local_test_postgres(name: str = "pgtest", version: Optional[str] = Non
         version = ":{version}"
     else:
         version = ""
-    process = run(
+    process = run(  # noqa: S602
         f"docker run --name {name} -e POSTGRES_PASSWORD=pwd"
-        f" -e POSTGRES_DB={name} -d -p 5432:5432 postgres{version}",  # noqa
+        f" -e POSTGRES_DB={name} -d -p 5432:5432 postgres{version}",
         shell=True,
     )
     if process.returncode == 0: