v2: Add lint and type checking for Python package (#515)

* ruff format

* Initial lint fix

* Initial ruff lint passing

* Add mypy type checking

* Add ci check

* Add py.typed

* ignore mypy errors in jupyter widget

* fix imports

* bump min version to Python 3.9

* numpy < 2

.github/workflows/build_test.yml

@@ -88,6 +88,29 @@ jobs:
           name: vegafusion-wasm-packages
           path: vegafusion-wasm/pkg/vegafusion-wasm-*.tgz
 
+  check-vegafusion-python:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # [email protected]
+      - uses: prefix-dev/[email protected]
+        with:
+          pixi-version: v0.30.0
+      - name: Cache
+        uses: actions/cache@v3
+        with:
+          key: ${{ runner.os }}-${{ hashFiles('pixi.lock', 'Cargo.lock') }}-build-vegafusion-wasm
+          path: |
+            ~/.cargo
+            target
+            .pixi
+      - name: Check format and lint
+        run: |
+          pixi run lint-check-py
+      - name: Type check
+        run: |
+          pixi run type-check-py
+
   # Use maturin action to build linux wheels within proper manylinux compatible containers
   # (This is why we don't use the pixi "build-py" action)
   build-vegafusion-python-linux-64:
@@ -325,7 +348,7 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@61a6322f88396a6271a6ee3565807d608ecaddd1 # [email protected]
         with:
-          python-version: '3.8'
+          python-version: '3.9'
       - name: Install Chrome
         uses: browser-actions/setup-chrome@f0ff752add8c926994566c80b3ceadfd03f24d12 # pin@latest
         with:
@@ -345,7 +368,7 @@ jobs:
           python -m pip install $vegafusion
 
           # Optional dependencies
-          python -m pip install pyarrow==10.0 altair==5.1.2 polars[timezone] "duckdb>=1.0" vl-convert-python 
+          python -m pip install pyarrow==10.0 altair==5.1.2 "numpy<2" polars[timezone] "duckdb>=1.0" vl-convert-python 
           
           # Test dependencies
           python -m pip install pytest altair vega-datasets scikit-image

pixi.toml

@@ -29,6 +29,11 @@ dev-py = { cmd = [
 # Build Python packages
 build-py = { cmd = "maturin build -m vegafusion-python/Cargo.toml --release --strip --sdist $0" }
 
+fmt-py = { cmd = "ruff format", cwd="vegafusion-python" }
+lint-fix-py = { cmd = "ruff format && ruff check --fix", cwd="vegafusion-python" }
+lint-check-py = { cmd = "ruff format --check && ruff check", cwd="vegafusion-python" }
+type-check-py = { cmd = "mypy", cwd="vegafusion-python" }
+
 # test python
 install-chromedriver-auto = """
 pip cache remove chromedriver-binary-auto &&
@@ -121,6 +126,10 @@ openjdk = "20.0.0.*"
 build = "0.7.0.*"
 minio-server = "2023.9.23.3.47.50.*"
 minio = "7.1.17.*"
+rust = "1.80.*"
+taplo = ">=0.9.3,<0.10"
+ruff = ">=0.6.9,<0.7"
+mypy = ">=1.11.2,<2"
 
 # Dependencies are those required at runtime by the Python packages
 [dependencies]
@@ -130,11 +139,10 @@ pandas = "2.0.3.*"
 altair = "5.4.*"
 protobuf = "4.23.3.*"
 ipywidgets = "8.1.0.*"
-rust = "1.80.*"
-vl-convert-python = "1.6.*"
+vl-convert-python = "1.7.*"
 anywidget = ">=0.9.6,<0.10"
 polars = "1.8.*"
-taplo = ">=0.9.3,<0.10"
+grpcio = ">=1.56.2,<2"
 
 [target.osx-arm64.build-dependencies]
 # These dependencies are for building node canvas from source on apple silicon

vegafusion-python/.mypy.ini

@@ -0,0 +1,7 @@
+[mypy]
+python_version = 3.9
+cache_dir = .mypy_cache
+strict = true
+files = vegafusion/
+
+ignore_missing_imports = true

vegafusion-python/pyproject.toml

@@ -3,14 +3,14 @@ name = "vegafusion"
 version = "1.6.9"
 description = "Core tools for using VegaFusion from Python"
 readme = "README.md"
-requires-python = ">=3.8"
+requires-python = ">=3.9"
 keywords = ["vega", "altair", "vegafusion", "arrow"]
 classifiers = [
-  "Programming Language :: Python :: 3.8",
   "Programming Language :: Python :: 3.9",
   "Programming Language :: Python :: 3.10",
   "Programming Language :: Python :: 3.11",
   "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: 3.13",
   "License :: OSI Approved :: BSD License",
   "Topic :: Scientific/Engineering :: Visualization",
 ]
@@ -32,7 +32,3 @@ Documentation = "https://vegafusion.io"
 
 [tool.maturin]
 module-name = "vegafusion._vegafusion"
-
-[tool.black]
-line-length = 88
-target_version = ["py38"]

vegafusion-python/ruff.toml

@@ -0,0 +1,65 @@
+line-length = 88
+indent-width = 4
+include = ["vegafusion/**/*.py", "tests/**/*.py"]
+exclude = ["vegafusion/datasource/_dfi_types.py", "tests/altair_mocks/**/*.py"]
+target-version = "py39"
+
+[lint]
+select = [
+  "E",   # pycodestyle Error
+  "F",   # Pyflakes
+  "W",   # pycodestyle Warning
+  "I",   # isort
+  "N",   # PEP8 Naming
+  "UP",  # pyupgrade
+  "YTT", # flake8-2020
+  "ANN", # flake8-annotations
+  "S",   # flake8-bandit
+  "BLE", # flake8-blind-except
+  "B",   # flake8-bugbear
+  "A",   # flake8-annotations-complexity
+  "C4",  # flake8-comprehensions
+  "EM",  # flake8-errmsg
+  "FA",  # flake8-future-annotations
+]
+
+ignore = [
+  # Allow assert statements
+  "S101",
+
+  # Rules that conflict with other rules
+  "D211", # No blank lines allowed before class docstring (no-blank-line-before-class-docstring)
+  "D213", # Multi-line docstring summary should start at the second line (multi-line-docstring-summary-start)
+  "EM101", # Exception must not use an string literal
+  "EM102",  # Exception must not use an f-string literal
+
+  # Not compatible with Python 3.9. Re-enable when upgrading to Python 3.10+
+  "UP007", # Using old-style union syntax (e.g., Union[int, str] instead of int | str)
+  "UP018", # Using non-native literals (e.g., using 'int' instead of 'Literal[int]')
+
+  # Deprecated and not useful
+  "ANN101", # Missing type annotation for self in method (missing-type-annotation-for-self)
+  "ANN102", # Missing type annotation for class (missing-type-cls)
+
+  "S608", # SQL injection
+
+  # Conflicting lint rules with Ruff formatter
+  "W191",   # Tab indentation
+  "E111",   # Indentation with invalid multiple
+  "E114",   # Indentation with invalid multiple comment
+  "E117",   # Over-indented
+  "D206",   # Indent with spaces
+  "D300",   # Triple single quotes
+  "Q000",   # Bad quotes inline string
+  "Q001",   # Bad quotes multiline string
+  "Q002",   # Bad quotes docstring
+  "Q003",   # Avoidable escaped quote
+  "COM812", # Missing trailing comma
+  "COM819", # Prohibited trailing comma
+  "ISC001", # Single-line implicit string concatenation
+  "ISC002", # Multi-line implicit string concatenation
+]
+
+[lint.per-file-ignores]
+# Ignore some lint checks in test directory
+"tests/*" = ["D", "ANN", "S", "I", "E501"]

vegafusion-python/tests/altair_mocks/area/cumulative_count/mock.py

@@ -8,7 +8,4 @@
 alt.Chart(source).transform_window(
     cumulative_count="count()",
     sort=[{"field": "IMDB_Rating"}],
-).mark_area().encode(
-    x="IMDB_Rating:Q",
-    y="cumulative_count:Q"
-)
+).mark_area().encode(x="IMDB_Rating:Q", y="cumulative_count:Q")

vegafusion-python/tests/altair_mocks/area/density_facet/mock.py

@@ -5,18 +5,10 @@
 source = data.iris()
 
 alt.Chart(source).transform_fold(
-    ['petalWidth',
-     'petalLength',
-     'sepalWidth',
-     'sepalLength'],
-    as_ = ['Measurement_type', 'value']
+    ["petalWidth", "petalLength", "sepalWidth", "sepalLength"],
+    as_=["Measurement_type", "value"],
 ).transform_density(
-    density='value',
-    bandwidth=0.3,
-    groupby=['Measurement_type'],
-    extent= [0, 8]
+    density="value", bandwidth=0.3, groupby=["Measurement_type"], extent=[0, 8]
 ).mark_area().encode(
-    alt.X('value:Q'),
-    alt.Y('density:Q'),
-    alt.Row('Measurement_type:N')
-).properties(width=300, height=50)
+    alt.X("value:Q"), alt.Y("density:Q"), alt.Row("Measurement_type:N")
+).properties(width=300, height=50)

vegafusion-python/tests/altair_mocks/area/density_stack/mock.py

@@ -6,20 +6,15 @@
 source = data.iris()
 
 alt.Chart(source).transform_fold(
-    ['petalWidth',
-     'petalLength',
-     'sepalWidth',
-     'sepalLength'],
-    as_ = ['Measurement_type', 'value']
+    ["petalWidth", "petalLength", "sepalWidth", "sepalLength"],
+    as_=["Measurement_type", "value"],
 ).transform_density(
-    density='value',
+    density="value",
     bandwidth=0.3,
-    groupby=['Measurement_type'],
-    extent= [0, 8],
-    counts = True,
-    steps=200
+    groupby=["Measurement_type"],
+    extent=[0, 8],
+    counts=True,
+    steps=200,
 ).mark_area().encode(
-    alt.X('value:Q'),
-    alt.Y('density:Q', stack='zero'),
-    alt.Color('Measurement_type:N')
+    alt.X("value:Q"), alt.Y("density:Q", stack="zero"), alt.Color("Measurement_type:N")
 ).properties(width=400, height=100)

vegafusion-python/tests/altair_mocks/area/gradient/mock.py

@@ -5,20 +5,17 @@
 
 source = data.stocks()
 
-alt.Chart(source).transform_filter(
-    'datum.symbol==="GOOG"'
-).mark_area(
-    line={'color':'darkgreen'},
+alt.Chart(source).transform_filter('datum.symbol==="GOOG"').mark_area(
+    line={"color": "darkgreen"},
     color=alt.Gradient(
-        gradient='linear',
-        stops=[alt.GradientStop(color='white', offset=0),
-               alt.GradientStop(color='darkgreen', offset=1)],
+        gradient="linear",
+        stops=[
+            alt.GradientStop(color="white", offset=0),
+            alt.GradientStop(color="darkgreen", offset=1),
+        ],
         x1=1,
         x2=1,
         y1=1,
-        y2=0
-    )
-).encode(
-    alt.X('date:T'),
-    alt.Y('price:Q')
-)
+        y2=0,
+    ),
+).encode(alt.X("date:T"), alt.Y("price:Q"))

vegafusion-python/tests/altair_mocks/area/horizon_graph/mock.py

@@ -3,35 +3,44 @@
 import altair as alt
 import pandas as pd
 
-source = pd.DataFrame([
-    {"x": 1,  "y": 28}, {"x": 2,  "y": 55},
-    {"x": 3,  "y": 43}, {"x": 4,  "y": 91},
-    {"x": 5,  "y": 81}, {"x": 6,  "y": 53},
-    {"x": 7,  "y": 19}, {"x": 8,  "y": 87},
-    {"x": 9,  "y": 52}, {"x": 10, "y": 48},
-    {"x": 11, "y": 24}, {"x": 12, "y": 49},
-    {"x": 13, "y": 87}, {"x": 14, "y": 66},
-    {"x": 15, "y": 17}, {"x": 16, "y": 27},
-    {"x": 17, "y": 68}, {"x": 18, "y": 16},
-    {"x": 19, "y": 49}, {"x": 20, "y": 15}
-])
+source = pd.DataFrame(
+    [
+        {"x": 1, "y": 28},
+        {"x": 2, "y": 55},
+        {"x": 3, "y": 43},
+        {"x": 4, "y": 91},
+        {"x": 5, "y": 81},
+        {"x": 6, "y": 53},
+        {"x": 7, "y": 19},
+        {"x": 8, "y": 87},
+        {"x": 9, "y": 52},
+        {"x": 10, "y": 48},
+        {"x": 11, "y": 24},
+        {"x": 12, "y": 49},
+        {"x": 13, "y": 87},
+        {"x": 14, "y": 66},
+        {"x": 15, "y": 17},
+        {"x": 16, "y": 27},
+        {"x": 17, "y": 68},
+        {"x": 18, "y": 16},
+        {"x": 19, "y": 49},
+        {"x": 20, "y": 15},
+    ]
+)
 
-area1 = alt.Chart(source).mark_area(
-    clip=True,
-    interpolate='monotone'
-).encode(
-    alt.X('x', scale=alt.Scale(zero=False, nice=False)),
-    alt.Y('y', scale=alt.Scale(domain=[0, 50]), title='y'),
-    opacity=alt.value(0.6)
-).properties(
-    width=500,
-    height=75
+area1 = (
+    alt.Chart(source)
+    .mark_area(clip=True, interpolate="monotone")
+    .encode(
+        alt.X("x", scale=alt.Scale(zero=False, nice=False)),
+        alt.Y("y", scale=alt.Scale(domain=[0, 50]), title="y"),
+        opacity=alt.value(0.6),
+    )
+    .properties(width=500, height=75)
 )
 
 area2 = area1.encode(
-    alt.Y('ny:Q', scale=alt.Scale(domain=[0, 50]))
-).transform_calculate(
-    "ny", alt.datum.y - 50
-)
+    alt.Y("ny:Q", scale=alt.Scale(domain=[0, 50]))
+).transform_calculate("ny", alt.datum.y - 50)
 
 area1 + area2

vegafusion-python/tests/altair_mocks/area/layered/mock.py

@@ -6,7 +6,5 @@
 source = data.iowa_electricity()
 
 alt.Chart(source).mark_area(opacity=0.3).encode(
-    x="year:T",
-    y=alt.Y("net_generation:Q", stack=None),
-    color="source:N"
+    x="year:T", y=alt.Y("net_generation:Q", stack=None), color="source:N"
 )

vegafusion-python/tests/altair_mocks/area/normalized_stacked/mock.py

@@ -6,7 +6,5 @@
 source = data.iowa_electricity()
 
 alt.Chart(source).mark_area().encode(
-    x="year:T",
-    y=alt.Y("net_generation:Q", stack="normalize"),
-    color="source:N"
+    x="year:T", y=alt.Y("net_generation:Q", stack="normalize"), color="source:N"
 )

vegafusion-python/tests/altair_mocks/area/streamgraph/mock.py

@@ -6,11 +6,7 @@
 source = data.unemployment_across_industries.url
 
 alt.Chart(source).mark_area().encode(
-    alt.X('yearmonth(date):T',
-          axis=alt.Axis(format='%Y', domain=False, tickSize=0)
-          ),
-    alt.Y('sum(count):Q', stack='center', axis=None),
-    alt.Color('series:N',
-              scale=alt.Scale(scheme='category20b')
-              )
+    alt.X("yearmonth(date):T", axis=alt.Axis(format="%Y", domain=False, tickSize=0)),
+    alt.Y("sum(count):Q", stack="center", axis=None),
+    alt.Color("series:N", scale=alt.Scale(scheme="category20b")),
 ).interactive()

vegafusion-python/tests/altair_mocks/area/trellis/mock.py

@@ -6,10 +6,5 @@
 source = data.iowa_electricity()
 
 alt.Chart(source).mark_area().encode(
-    x="year:T",
-    y="net_generation:Q",
-    color="source:N",
-    row="source:N"
-).properties(
-    height=100
-)
+    x="year:T", y="net_generation:Q", color="source:N", row="source:N"
+).properties(height=100)