From 428d04f7a7cb18b65ef9225b33c0cf03e25b5ea1 Mon Sep 17 00:00:00 2001 From: Jan Rous Date: Thu, 30 Nov 2023 00:29:09 -0700 Subject: [PATCH] Break ferc_to_sqlite op monoliths. Refactor monolithic dbf2sqlite and xbrl2sqlite methods into per-dataset smaller ops that are invoked within the graphs. This should allow us to better make use of dagster parallelism and speed up ferc_to_sqlite processing. It seems that current unit/integration tests only use FERC1 raw data, so I've modified the fixtures to only run the relevant pieces of processing. --- src/pudl/extract/dbf.py | 28 ++++++++- src/pudl/extract/ferc.py | 37 ++--------- src/pudl/extract/xbrl.py | 42 +++++++------ src/pudl/ferc_to_sqlite/__init__.py | 24 ++++--- src/pudl/ferc_to_sqlite/cli.py | 11 +--- src/pudl/resources.py | 11 +++- test/conftest.py | 88 ++++++++++++-------------- test/unit/extract/xbrl_test.py | 98 ++++++++++++----------------- 8 files changed, 165 insertions(+), 174 deletions(-) diff --git a/src/pudl/extract/dbf.py b/src/pudl/extract/dbf.py index e48b9c3f25..838825cdf0 100644 --- a/src/pudl/extract/dbf.py +++ b/src/pudl/extract/dbf.py @@ -4,13 +4,14 @@ import importlib.resources import zipfile from collections import defaultdict -from collections.abc import Iterator +from collections.abc import Callable, Iterator from functools import lru_cache from pathlib import Path from typing import IO, Any, Protocol, Self import pandas as pd import sqlalchemy as sa +from dagster import op from dbfread import DBF, FieldParser import pudl @@ -18,6 +19,7 @@ from pudl.metadata.classes import DataSource from pudl.settings import FercToSqliteSettings, GenericDatasetSettings from pudl.workspace.datastore import Datastore +from pudl.workspace.setup import PudlPaths logger = pudl.logging_helpers.get_logger(__name__) @@ -464,6 +466,30 @@ def get_db_path(self) -> str: db_path = str(Path(self.output_path) / self.DATABASE_NAME) return f"sqlite:///{db_path}" + @classmethod + def get_dagster_op(cls) -> Callable: + """Returns dagstger op that runs this extractor.""" + + @op( + name=f"dbf_{cls.DATASET}", + required_resource_keys={ + "ferc_to_sqlite_settings", + "datastore", + "runtime_settings", + }, + ) + def inner_method(context) -> None: + """Instantiates dbf extractor and runs it.""" + dbf_extractor = cls( + datastore=context.resources.datastore, + settings=context.resources.ferc_to_sqlite_settings, + clobber=context.resources.runtime_settings.clobber, + output_path=PudlPaths().output_dir, + ) + dbf_extractor.execute() + + return inner_method + def execute(self): """Runs the extraction of the data from dbf to sqlite.""" logger.info( diff --git a/src/pudl/extract/ferc.py b/src/pudl/extract/ferc.py index bf7a8514f0..823f4024db 100644 --- a/src/pudl/extract/ferc.py +++ b/src/pudl/extract/ferc.py @@ -1,42 +1,17 @@ """Hooks to integrate ferc to sqlite functionality into dagster graph.""" -from dagster import Field, op - import pudl from pudl.extract.ferc1 import Ferc1DbfExtractor from pudl.extract.ferc2 import Ferc2DbfExtractor from pudl.extract.ferc6 import Ferc6DbfExtractor from pudl.extract.ferc60 import Ferc60DbfExtractor -from pudl.workspace.setup import PudlPaths logger = pudl.logging_helpers.get_logger(__name__) - -@op( - config_schema={ - "clobber": Field( - bool, description="Clobber existing ferc1 database.", default_value=False - ), - }, - required_resource_keys={"ferc_to_sqlite_settings", "datastore"}, -) -def dbf2sqlite(context) -> None: - """Clone the FERC Form 1 Visual FoxPro databases into SQLite.""" - # TODO(rousik): this thin wrapper seems to be somewhat quirky. Maybe there's a way - # to make the integration # between the class and dagster better? Investigate. - logger.info(f"dbf2sqlite settings: {context.resources.ferc_to_sqlite_settings}") - - extractors = [ - Ferc1DbfExtractor, - Ferc2DbfExtractor, - Ferc6DbfExtractor, - Ferc60DbfExtractor, - ] - for xclass in extractors: - xclass( - datastore=context.resources.datastore, - settings=context.resources.ferc_to_sqlite_settings, - clobber=context.op_config["clobber"], - output_path=PudlPaths().output_dir, - ).execute() +ALL_DBF_EXTRACTORS = [ + Ferc1DbfExtractor, + Ferc2DbfExtractor, + Ferc6DbfExtractor, + Ferc60DbfExtractor, +] diff --git a/src/pudl/extract/xbrl.py b/src/pudl/extract/xbrl.py index 4e54ed88f9..bc3812e369 100644 --- a/src/pudl/extract/xbrl.py +++ b/src/pudl/extract/xbrl.py @@ -4,10 +4,11 @@ from datetime import date from pathlib import Path -from dagster import ConfigurableResource, op +from dagster import op from ferc_xbrl_extractor.cli import run_main import pudl +from pudl.resources import RuntimeSettings from pudl.settings import FercGenericXbrlToSqliteSettings, XbrlFormNumber from pudl.workspace.datastore import Datastore from pudl.workspace.setup import PudlPaths @@ -15,14 +16,6 @@ logger = pudl.logging_helpers.get_logger(__name__) -class XbrlRuntimeSettings(ConfigurableResource): - """Encodes runtime setting for the XBRL extraction.""" - # TODO(rousik): Using BaseSettings here might allow configuring this via environment variables. - clobber: bool = False - num_workers: None | int = None - batch_size: int = 50 - - class FercXbrlDatastore: """Simple datastore wrapper for accessing ferc1 xbrl resources.""" @@ -52,22 +45,33 @@ def get_filings(self, year: int, form: XbrlFormNumber) -> io.BytesIO: ) ) + def xbrl2sqlite_op_factory(form: XbrlFormNumber) -> Callable: """Generates xbrl2sqlite op for a given FERC form.""" + @op( name=f"ferc{form.value}_xbrl", - required_resource_keys={"ferc_to_sqlite_settings", "datastore", "xbrl_runtime_settings"} + required_resource_keys={ + "ferc_to_sqlite_settings", + "datastore", + "runtime_settings", + }, ) - def inner_xbrl2sqlite(context) -> None: + def inner_op(context) -> None: output_path = PudlPaths().output_dir - runtime_settings: XbrlRuntimeSettings = context.resources.xbrl_runtime_settings - settings = context.resources.ferc_to_sqlite_settings.get_xbrl_dataset_settings(form) + runtime_settings: RuntimeSettings = context.resources.runtime_settings + settings = context.resources.ferc_to_sqlite_settings.get_xbrl_dataset_settings( + form + ) datastore = FercXbrlDatastore(context.resources.datastore) if settings is None or settings.disabled: - logger.info(f"Skipping dataset ferc{form}_xbrl: no config or is disabled.") - sql_path = PudlPaths().sqlite_db_path(f"ferc{form.value}_xbrl") + logger.info( + f"Skipping dataset ferc{form.value}_xbrl: no config or is disabled." + ) + return + sql_path = PudlPaths().sqlite_db_path(f"ferc{form.value}_xbrl") if sql_path.exists(): if runtime_settings.clobber: sql_path.unlink() @@ -82,10 +86,12 @@ def inner_xbrl2sqlite(context) -> None: datastore, output_path=output_path, sql_path=sql_path, - batch_size=runtime_settings.batch_size, - workers=runtime_settings.num_workers, + batch_size=runtime_settings.xbrl_batch_size, + workers=runtime_settings.xbrl_num_workers, ) - return inner_xbrl2sqlite + + return inner_op + def convert_form( form_settings: FercGenericXbrlToSqliteSettings, diff --git a/src/pudl/ferc_to_sqlite/__init__.py b/src/pudl/ferc_to_sqlite/__init__.py index 2b8c8f3df7..2c5848d0c9 100644 --- a/src/pudl/ferc_to_sqlite/__init__.py +++ b/src/pudl/ferc_to_sqlite/__init__.py @@ -4,9 +4,13 @@ from dagster import Definitions, graph import pudl -from pudl.extract.ferc import dbf2sqlite -from pudl.extract.xbrl import XbrlRuntimeSettings, xbrl2sqlite_op_factory -from pudl.resources import datastore, ferc_to_sqlite_settings +from pudl.extract.ferc import ALL_DBF_EXTRACTORS +from pudl.extract.ferc1 import Ferc1DbfExtractor +from pudl.extract.ferc2 import Ferc2DbfExtractor +from pudl.extract.ferc6 import Ferc6DbfExtractor +from pudl.extract.ferc60 import Ferc60DbfExtractor +from pudl.extract.xbrl import xbrl2sqlite_op_factory +from pudl.resources import RuntimeSettings, datastore, ferc_to_sqlite_settings from pudl.settings import EtlSettings, XbrlFormNumber logger = pudl.logging_helpers.get_logger(__name__) @@ -15,7 +19,8 @@ @graph def ferc_to_sqlite(): """Clone the FERC FoxPro databases and XBRL filings into SQLite.""" - dbf2sqlite() + for extractor in ALL_DBF_EXTRACTORS: + extractor.get_dagster_op()() for form in XbrlFormNumber: xbrl2sqlite_op_factory(form)() @@ -23,7 +28,8 @@ def ferc_to_sqlite(): @graph def ferc_to_sqlite_dbf_only(): """Clone the FERC FoxPro databases into SQLite.""" - dbf2sqlite() + for extractor in ALL_DBF_EXTRACTORS: + extractor.get_dagster_op()() @graph @@ -32,9 +38,10 @@ def ferc_to_sqlite_xbrl_only(): for form in XbrlFormNumber: xbrl2sqlite_op_factory(form)() + default_resources_defs = { "ferc_to_sqlite_settings": ferc_to_sqlite_settings, - "xbrl_runtime_settings": XbrlRuntimeSettings(), + "runtime_settings": RuntimeSettings(), "datastore": datastore, } @@ -55,10 +62,9 @@ def ferc_to_sqlite_xbrl_only(): "ferc_to_sqlite_settings": { "config": ferc_to_sqlite_fast_settings.model_dump(), }, - "xbrl_runtime_settings": { - # TODO(rousik): do we need to set some defaults here? + "runtime_settings": { "config": {}, - } + }, }, }, ) diff --git a/src/pudl/ferc_to_sqlite/cli.py b/src/pudl/ferc_to_sqlite/cli.py index 5754ee97df..067d4beb74 100644 --- a/src/pudl/ferc_to_sqlite/cli.py +++ b/src/pudl/ferc_to_sqlite/cli.py @@ -154,18 +154,13 @@ def main(): # noqa: C901 else "", }, }, - }, - "ops": { - "xbrl2sqlite": { + "runtime_settings": { "config": { - "workers": args.workers, - "batch_size": args.batch_size, "clobber": args.clobber, + "xbrl_num_workers": args.workers, + "xbrl_batch_size": args.batch_size, }, }, - "dbf2sqlite": { - "config": {"clobber": args.clobber}, - }, }, }, raise_on_error=True, diff --git a/src/pudl/resources.py b/src/pudl/resources.py index 13d2a50471..ab58b715b5 100644 --- a/src/pudl/resources.py +++ b/src/pudl/resources.py @@ -1,12 +1,21 @@ """Collection of Dagster resources for PUDL.""" -from dagster import Field, resource +from dagster import ConfigurableResource, Field, resource from pudl.settings import DatasetsSettings, FercToSqliteSettings, create_dagster_config from pudl.workspace.datastore import Datastore from pudl.workspace.setup import PudlPaths +class RuntimeSettings(ConfigurableResource): + """Encodes runtime settings for the ferc_to_sqlite graphs.""" + + # TODO(rousik): Using BaseSettings here might allow configuring this via environment variables. + clobber: bool = False + xbrl_num_workers: None | int = None + xbrl_batch_size: int = 50 + + @resource(config_schema=create_dagster_config(DatasetsSettings())) def dataset_settings(init_context) -> DatasetsSettings: """Dagster resource for parameterizing PUDL ETL assets. diff --git a/test/conftest.py b/test/conftest.py index a09dc516f0..dbb5b81915 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -9,13 +9,13 @@ import pytest import sqlalchemy as sa -from dagster import build_init_resource_context, materialize_to_memory +from dagster import build_init_resource_context, graph, materialize_to_memory import pudl from pudl import resources from pudl.cli.etl import pudl_etl_job_factory -from pudl.extract.ferc1 import raw_xbrl_metadata_json -from pudl.ferc_to_sqlite.cli import ferc_to_sqlite_job_factory +from pudl.extract.ferc1 import Ferc1DbfExtractor, raw_xbrl_metadata_json +from pudl.extract.xbrl import xbrl2sqlite_op_factory from pudl.io_managers import ( PudlSQLiteIOManager, ferc1_dbf_sqlite_io_manager, @@ -24,7 +24,12 @@ ) from pudl.metadata.classes import Package from pudl.output.pudltabl import PudlTabl -from pudl.settings import DatasetsSettings, EtlSettings, FercToSqliteSettings +from pudl.settings import ( + DatasetsSettings, + EtlSettings, + FercToSqliteSettings, + XbrlFormNumber, +) from pudl.workspace.datastore import Datastore from pudl.workspace.setup import PudlPaths @@ -180,14 +185,22 @@ def pudl_out_orig(live_dbs: bool, pudl_engine: sa.Engine) -> PudlTabl: @pytest.fixture(scope="session") -def ferc_to_sqlite_dbf_only( - live_dbs: bool, pudl_datastore_config, etl_settings: EtlSettings +def ferc1_dbf_extract( + live_dbs: bool, + pudl_datastore_config, + etl_settings: EtlSettings, ): - """Create raw FERC 1 SQLite DBs, but only based on DBF sources.""" + """Creates raw FERC 1 SQlite DBs, based only on DBF sources.""" + + @graph + def local_dbf_ferc1_graph(): + Ferc1DbfExtractor.get_dagster_op()() + if not live_dbs: - ferc_to_sqlite_job_factory( - enable_xbrl=False, - )().execute_in_process( + local_dbf_ferc1_graph.to_job( + name="ferc_to_sqlite_dbf_ferc1", + resource_defs=pudl.ferc_to_sqlite.default_resources_defs, + ).execute_in_process( run_config={ "resources": { "ferc_to_sqlite_settings": { @@ -196,62 +209,43 @@ def ferc_to_sqlite_dbf_only( "datastore": { "config": pudl_datastore_config, }, + "runtime_settings": {"config": {}}, }, }, ) @pytest.fixture(scope="session") -def ferc_to_sqlite_xbrl_only( +def ferc1_xbrl_extract( live_dbs: bool, pudl_datastore_config, etl_settings: EtlSettings ): - """Create raw FERC 1 SQLite DBs, but only based on XBRL sources.""" - if not live_dbs: - ferc_to_sqlite_job_factory( - enable_dbf=False, - )().execute_in_process( - run_config={ - "resources": { - "ferc_to_sqlite_settings": { - "config": etl_settings.ferc_to_sqlite_settings.model_dump() - }, - "datastore": { - "config": pudl_datastore_config, - }, - }, - }, - ) - + """Runs ferc_to_sqlite dagster job for FERC Form 1 XBRL data.""" -@pytest.fixture(scope="session") -def ferc_to_sqlite(live_dbs, pudl_datastore_config, etl_settings: EtlSettings): - """Create raw FERC 1 SQLite DBs. + @graph + def local_xbrl_ferc1_graph(): + xbrl2sqlite_op_factory(XbrlFormNumber.FERC1)() - If we are using the test database, we initialize it from scratch first. If we're - using the live database, then the sql engine fixtures will return connections to the - existing databases - """ if not live_dbs: - logger.info( - f"ferc_to_sqlite_settings: {etl_settings.ferc_to_sqlite_settings.model_dump()}" - ) - logger.info(f"ferc_to_sqlite PUDL_OUTPUT: {os.getenv('PUDL_OUTPUT')}") - ferc_to_sqlite_job_factory()().execute_in_process( + local_xbrl_ferc1_graph.to_job( + name="ferc_to_sqlite_xbrl_ferc1", + resource_defs=pudl.ferc_to_sqlite.default_resources_defs, + ).execute_in_process( run_config={ "resources": { "ferc_to_sqlite_settings": { - "config": etl_settings.ferc_to_sqlite_settings.model_dump() + "config": etl_settings.ferc_to_sqlite_settings.model_dump(), }, "datastore": { "config": pudl_datastore_config, }, + "runtime_settings": {"config": {}}, }, - }, + } ) @pytest.fixture(scope="session", name="ferc1_engine_dbf") -def ferc1_dbf_sql_engine(ferc_to_sqlite_dbf_only: FercToSqliteSettings) -> sa.Engine: +def ferc1_dbf_sql_engine(ferc1_dbf_extract, dataset_settings_config) -> sa.Engine: """Grab a connection to the FERC Form 1 DB clone.""" context = build_init_resource_context( resources={"dataset_settings": dataset_settings_config} @@ -260,9 +254,7 @@ def ferc1_dbf_sql_engine(ferc_to_sqlite_dbf_only: FercToSqliteSettings) -> sa.En @pytest.fixture(scope="session", name="ferc1_engine_xbrl") -def ferc1_xbrl_sql_engine( - ferc_to_sqlite_xbrl_only: FercToSqliteSettings, dataset_settings_config -) -> sa.Engine: +def ferc1_xbrl_sql_engine(ferc1_xbrl_extract, dataset_settings_config) -> sa.Engine: """Grab a connection to the FERC Form 1 DB clone.""" context = build_init_resource_context( resources={"dataset_settings": dataset_settings_config} @@ -342,9 +334,7 @@ def configure_paths_for_tests(tmp_path_factory, request): gha = os.environ.get("GITHUB_ACTIONS", False) # Under what circumstances do we want to use a temporary input directory? # This will force a re-download of raw inputs from Zenodo or the GCS cache: - if (gha and "PUDL_INPUT" not in os.environ) or ( - request.config.getoption("--tmp-data") - ): + if request.config.getoption("--tmp-data") or ("PUDL_INPUT" not in os.environ): in_tmp = pudl_tmpdir / "input" in_tmp.mkdir() PudlPaths.set_path_overrides( diff --git a/test/unit/extract/xbrl_test.py b/test/unit/extract/xbrl_test.py index 61ff1bdb07..12f13caca9 100644 --- a/test/unit/extract/xbrl_test.py +++ b/test/unit/extract/xbrl_test.py @@ -1,9 +1,11 @@ """Tests for xbrl extraction module.""" import pytest -from dagster import build_op_context +from dagster import ResourceDefinition, build_op_context -from pudl.extract.xbrl import FercXbrlDatastore, convert_form, xbrl2sqlite +from pudl.extract.xbrl import FercXbrlDatastore, convert_form, xbrl2sqlite_op_factory +from pudl.ferc_to_sqlite import ferc_to_sqlite_xbrl_only +from pudl.resources import RuntimeSettings from pudl.settings import ( Ferc1DbfToSqliteSettings, Ferc1XbrlToSqliteSettings, @@ -99,28 +101,21 @@ def test_xbrl2sqlite(settings, forms, mocker, tmp_path): mock_datastore = mocker.MagicMock() mocker.patch("pudl.extract.xbrl.FercXbrlDatastore", return_value=mock_datastore) - # always use tmp path here so that we don't clobber the live DB when --live-dbs is passed - mock_pudl_paths = mocker.MagicMock( - spec=PudlPaths(), - sqlite_db_path=lambda form_name: tmp_path / f"{form_name}.sqlite", - output_dir=PudlPaths().output_dir, - ) - mocker.patch("pudl.extract.xbrl.PudlPaths", return_value=mock_pudl_paths) - - # Construct xbrl2sqlite op context - context = build_op_context( + # always use tmp ath here so that we don't clobber the live DB when --live-dbs is passed + ferc_to_sqlite_xbrl_only.execute_in_process( resources={ "ferc_to_sqlite_settings": settings, - "datastore": "datastore", - }, - config={ - "workers": 10, - "batch_size": 20, - "clobber": True, - }, + "datastore": ResourceDefinition.mock_resource(), + "runtime_settings": RuntimeSettings( + xbrl_batch_size=20, + xbrl_num_workers=10, + clobber=True, + ), + } ) - xbrl2sqlite(context) + # TODO(rousik): do we need to use this, or can we simply set PUDL_OUTPUT env + # variable to some random path? assert convert_form_mock.call_count == len(forms) @@ -130,13 +125,16 @@ def test_xbrl2sqlite(settings, forms, mocker, tmp_path): form, mock_datastore, output_path=PudlPaths().output_dir, - sql_path=tmp_path / f"ferc{form.value}_xbrl.sqlite", + sql_path=PudlPaths().output_dir / f"ferc{form.value}_xbrl.sqlite", batch_size=20, workers=10, ) -def test_xbrl2sqlite_db_exists_no_clobber(mocker): +def test_xbrl2sqlite_db_exists_no_clobber(mocker, live_dbs): + if live_dbs: + return + convert_form_mock = mocker.MagicMock() mocker.patch("pudl.extract.xbrl.convert_form", new=convert_form_mock) @@ -147,31 +145,30 @@ def test_xbrl2sqlite_db_exists_no_clobber(mocker): ferc1_sqlite_path = PudlPaths().output_dir / "ferc1_xbrl.sqlite" ferc1_sqlite_path.touch() settings = FercToSqliteSettings( - ferc1_dbf_to_sqlite_settings=Ferc1DbfToSqliteSettings(), ferc1_xbrl_to_sqlite_settings=Ferc1XbrlToSqliteSettings(), - ferc2_xbrl_to_sqlite_settings=None, - ferc6_xbrl_to_sqlite_settings=None, - ferc60_xbrl_to_sqlite_settings=None, - ferc714_xbrl_to_sqlite_settings=None, ) - # Construct xbrl2sqlite op context context = build_op_context( resources={ "ferc_to_sqlite_settings": settings, "datastore": "datastore", - }, - config={ - "workers": 10, - "batch_size": 20, - "clobber": False, + "runtime_settings": RuntimeSettings( + clobber=False, + xbrl_batch_size=20, + xbrl_num_workers=10, + ), }, ) + assert ferc1_sqlite_path.exists() with pytest.raises(RuntimeError, match="Found existing DB"): - xbrl2sqlite(context) + xbrl2sqlite_op_factory(XbrlFormNumber.FORM1)(context) + assert ferc1_sqlite_path.exists() + +def test_xbrl2sqlite_db_exists_yes_clobber(mocker, live_dbs): + if live_dbs: + return -def test_xbrl2sqlite_db_exists_yes_clobber(mocker, tmp_path): convert_form_mock = mocker.MagicMock() mocker.patch("pudl.extract.xbrl.convert_form", new=convert_form_mock) @@ -180,40 +177,27 @@ def test_xbrl2sqlite_db_exists_yes_clobber(mocker, tmp_path): mocker.patch("pudl.extract.xbrl.FercXbrlDatastore", return_value=mock_datastore) # always use tmp path here so that we don't clobber the live DB when --live-dbs is passed - ferc1_sqlite_path = tmp_path / "ferc1_xbrl.sqlite" + ferc1_sqlite_path = PudlPaths().output_dir / "ferc1_xbrl.sqlite" ferc1_sqlite_path.touch() - - # mock the db path so we can assert it gets clobbered - mock_db_path = mocker.MagicMock(spec=ferc1_sqlite_path) - mock_pudl_paths = mocker.MagicMock( - spec=PudlPaths(), sqlite_db_path=lambda _x: mock_db_path - ) - mocker.patch("pudl.extract.xbrl.PudlPaths", return_value=mock_pudl_paths) - settings = FercToSqliteSettings( - ferc1_dbf_to_sqlite_settings=Ferc1DbfToSqliteSettings(), ferc1_xbrl_to_sqlite_settings=Ferc1XbrlToSqliteSettings(), - ferc2_xbrl_to_sqlite_settings=None, - ferc6_xbrl_to_sqlite_settings=None, - ferc60_xbrl_to_sqlite_settings=None, - ferc714_xbrl_to_sqlite_settings=None, ) context = build_op_context( resources={ "ferc_to_sqlite_settings": settings, "datastore": "datastore", - }, - config={ - "workers": 10, - "batch_size": 20, - "clobber": True, + "runtime_settings": RuntimeSettings( + clobber=True, + xbrl_batch_size=20, + xbrl_num_workers=10, + ), }, ) - xbrl2sqlite(context) - - mock_db_path.unlink.assert_any_call() + assert ferc1_sqlite_path.exists() + xbrl2sqlite_op_factory(XbrlFormNumber.FORM1)(context) + assert not ferc1_sqlite_path.exists() def test_convert_form(mocker):