chore: lint fixes

CHANGELOG.md

@@ -88,4 +88,3 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ## [v1.0.0](https://github.com/DataShades/ckanext-ingest/releases/tag/v1.0.0) - 2022-02-07
 
 <small>[Compare with first commit](https://github.com/DataShades/ckanext-ingest/compare/5218fb4ae2e6c806e027ff44a5a17bd41377967c...v1.0.0)</small>
-

LICENSE

@@ -658,4 +658,4 @@ specific requirements.
   You should also get your employer (if you work as a programmer) or school,
 if any, to sign a "copyright disclaimer" for the program, if necessary.
 For more information on this, and how to apply and follow the GNU AGPL, see
-<http://www.gnu.org/licenses/>.
+<http://www.gnu.org/licenses/>.

MANIFEST.in

@@ -1,4 +1,4 @@
 include README.rst
 include LICENSE
 include requirements.txt
-recursive-include ckanext/ingest *.html *.json *.js *.less *.css *.mo *.yaml *.yml
+recursive-include ckanext/ingest *.html *.json *.js *.less *.css *.mo *.yaml *.yml

ckanext/ingest/logic/action.py

@@ -32,7 +32,6 @@ def ingest_extract_records(
     `iter_records` function that produces an iterable over records.
 
     Args:
-
         source: str|FileStorage - data source for records
 
         strategy: str|None - record extraction strategy. If missing, strategy
@@ -62,7 +61,6 @@ def ingest_import_records(context: types.Context, data_dict: dict[str, Any]):
     `Record.ingest`, potentially creating/updating data.
 
     Args:
-
         source: str|FileStorage - data source for records
 
         strategy: str|None - record extraction strategy. If missing, strategy
@@ -80,7 +78,6 @@ def ingest_import_records(context: types.Context, data_dict: dict[str, Any]):
 
         take: int - max number of records that will be ingested
     """
-
     tk.check_access("ingest_import_records", context, data_dict)
 
     start = data_dict["skip"]

ckanext/ingest/logic/schema.py

@@ -15,7 +15,7 @@
 
 
 def into_uploaded_file(value: Any):
-    """Try converting value into shared.Storage object"""
+    """Try converting value into shared.Storage object."""
     if isinstance(value, shared.Storage):
         return value
 

ckanext/ingest/logic/validators.py

@@ -15,6 +15,7 @@ def ingest_into_uuid(value: str):
 def ingest_munge_name(value: str):
     return munge.munge_name(value)
 
+
 def ingest_strip_prefix(prefix: str):
     def validator(value: Any):
         if isinstance(value, str) and value.startswith(prefix):

ckanext/ingest/plugin.py

@@ -1,7 +1,10 @@
 from __future__ import annotations
 
+import contextlib
+
 import ckan.plugins.toolkit as tk
-from ckan import common, plugins
+from ckan import common
+from ckan import plugins as p
 
 from . import config, interfaces, shared
 
@@ -12,10 +15,10 @@
 @tk.blanket.cli
 @tk.blanket.blueprints
 @tk.blanket.config_declarations
-class IngestPlugin(plugins.SingletonPlugin):
-    plugins.implements(plugins.IConfigurer)
-    plugins.implements(plugins.IConfigurable)
-    plugins.implements(interfaces.IIngest, inherit=True)
+class IngestPlugin(p.SingletonPlugin):
+    p.implements(p.IConfigurer)
+    p.implements(p.IConfigurable)
+    p.implements(interfaces.IIngest, inherit=True)
 
     # IConfigurer
 
@@ -31,7 +34,7 @@ def configure(self, config_: common.CKANConfig):
         blacklist = config.disabled_strategies()
         name_mapping = config.name_mapping()
 
-        for plugin in plugins.PluginImplementations(interfaces.IIngest):
+        for plugin in p.PluginImplementations(interfaces.IIngest):
             for name, s in plugin.get_ingest_strategies().items():
                 final_name = name_mapping.get(f"{s.__module__}:{s.__name__}", name)
 
@@ -45,13 +48,16 @@ def configure(self, config_: common.CKANConfig):
 
     # IIngest
     def get_ingest_strategies(self) -> dict[str, type[shared.ExtractionStrategy]]:
-        from .strategy import csv, xlsx, zip
+        from .strategy import csv, zip
 
-        strategies = {
+        strategies: dict[str, type[shared.ExtractionStrategy]] = {
             "ingest:recursive_zip": zip.ZipStrategy,
             "ingest:scheming_csv": csv.CsvStrategy,
         }
-        if xlsx.is_installed:
-            strategies["ingest:xlsx"] = xlsx.XlsxStrategy
+
+        with contextlib.suppress(ImportError):
+            from .strategy.xlsx import XlsxStrategy
+
+            strategies["ingest:xlsx"] = XlsxStrategy
 
         return strategies

ckanext/ingest/shared.py

@@ -130,8 +130,7 @@ def must_handle(cls, mime: str | None, source: Storage) -> bool:
         return False
 
     def chunks(self, source: Storage, options: StrategyOptions) -> Iterable[Any]:
-        """Iterate over separate chunks of data with correpoinding options
-        suitable for Record creation."""
+        """Produce chunks of data."""
         return []
 
     def chunk_into_record(self, chunk: Any, options: StrategyOptions) -> Record:
@@ -187,24 +186,22 @@ def make_storage(
     name: str | None = None,
     mimetype: str | None = None,
 ):
-    """Transform bytes stream(BytesIO), string or bytes into source object
-    expected by ExtractionStrategy.
+    """Transform incoming stream into standard format.
 
+    Transform bytes stream(BytesIO), string or bytes into source object
+    expected by ExtractionStrategy.
     """
     if isinstance(stream, str):
         stream = stream.encode()
 
-    if isinstance(stream, bytes):
+    if isinstance(stream, (bytes, bytearray, memoryview)):
         stream = BytesIO(stream)
 
     return Storage(stream, name, content_type=mimetype)
 
 
 def get_extra(options: StrategyOptions | RecordOptions, key: str, default: T) -> T:
-    """Safely return an item from `extras` member of strategy or record
-    options.
-
-    """
+    """Safely return an item from strategy `extras`."""
     return options.setdefault("extras", {}).get(key, default)
 
 

ckanext/ingest/strategy/__init__.py

@@ -1 +0,0 @@
-

ckanext/ingest/strategy/csv.py

@@ -25,7 +25,9 @@ class CsvStrategy(shared.ExtractionStrategy):
     record_factory = PackageRecord
 
     def chunks(
-        self, source: shared.Storage, options: shared.StrategyOptions,
+        self,
+        source: shared.Storage,
+        options: shared.StrategyOptions,
     ) -> Iterable[dict[str, Any]]:
         reader_options: dict[str, Any] = shared.get_extra(options, "reader_options", {})
         str_stream = StringIO(source.read().decode())

ckanext/ingest/strategy/xlsx.py

@@ -4,20 +4,16 @@
 resources to rewrite it and create a proper base XLSX strategy.
 
 """
+
 from __future__ import annotations
 
 import logging
 from typing import IO, Callable, Iterable, TypedDict, cast
 
-from ckanext.ingest import shared
-
-try:
-    from openpyxl import Workbook, load_workbook
-    from openpyxl.worksheet.worksheet import Worksheet
+from openpyxl import Workbook, load_workbook
+from openpyxl.worksheet.worksheet import Worksheet
 
-    is_installed = True
-except ImportError:
-    is_installed = False
+from ckanext.ingest import shared
 
 log = logging.getLogger(__name__)
 
@@ -60,11 +56,13 @@ def chunks(
             yield {
                 "sheet": sheet,
                 "document": doc,
-                "locator": lambda name: doc[name] if name in doc else None,
+                "locator": lambda name: doc.get(name, None),  # type: ignore
             }
 
     def extract(
-        self, source: shared.Storage, options: shared.StrategyOptions,
+        self,
+        source: shared.Storage,
+        options: shared.StrategyOptions,
     ) -> Iterable[shared.Record]:
         extras = options.get("extras", {})
 

ckanext/ingest/strategy/zip.py

@@ -19,7 +19,7 @@ class ZipChunk(TypedDict):
     handler: shared.ExtractionStrategy
     name: str
     source: shared.Storage
-    locator: Callable[[str], IO[bytes]]
+    locator: Callable[[str], IO[bytes] | None]
 
 
 class ZipStrategy(shared.ExtractionStrategy):

ckanext/ingest/tests/logic/test_action.py

@@ -1,6 +1,7 @@
+from __future__ import annotations
+
 import mimetypes
 import os
-from typing import Optional
 
 import pytest
 
@@ -14,7 +15,7 @@
 def source():
     data = os.path.join(os.path.dirname(__file__), "data")
 
-    def reader(filename: str, mime: Optional[str] = None):
+    def reader(filename: str, mime: str | None = None):
         if mime is None:
             mime, _enc = mimetypes.guess_type(filename)
 

ckanext/ingest/transform.py

@@ -11,10 +11,13 @@
 
 _default = object()
 
+
 @dataclasses.dataclass
 class Options:
-    """Transformation options taken from `{profile}_options` attribute of field
-    in ckanext-scheming's schema
+    """Transformation options.
+
+    This information is taken from `{profile}_options` attribute of field in
+    ckanext-scheming's schema.
 
     These options define how raw value passed into the Record transforms into
     proper value that is suitable for the entity's schema. Workflow is the
@@ -161,7 +164,7 @@ def _normalize_choice(
     choices: list[dict[str, str]],
     separator: str,
 ) -> str | list[str] | None:
-    """Transform select label[s] into corresponding value[s]"""
+    """Transform select label[s] into corresponding value[s]."""
     if not value:
         return None
 

pyproject.toml

@@ -40,7 +40,7 @@ ignore = [
        "PLC1901", # simplify comparison to empty string: violated by SQLAlchemy filters
 ]
 
-[tool.ruff.per-file-ignores]
+[tool.ruff.lint.per-file-ignores]
 "ckanext/ingest/tests*" = ["S", "PL", "ANN"]
 "ckanext/ingest/logic/schema.py" = ["PLR0913"]
 "ckanext/ingest/logic/*" = [

setup.cfg

@@ -65,13 +65,3 @@ previous = true
 domain = ckanext-ingest
 directory = ckanext/ingest/i18n
 statistics = true
-
-
-[tool:pytest]
-filterwarnings =
-        ignore::sqlalchemy.exc.SADeprecationWarning
-        ignore::sqlalchemy.exc.SAWarning
-        ignore::DeprecationWarning
-
-addopts = --ckan-ini test.ini
-testpaths = ckanext/ingest/tests

test.ini

@@ -50,4 +50,4 @@ level = NOTSET
 formatter = generic
 
 [formatter_generic]
-format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s
+format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s