From 333a1e59bed45dc2c42feab409154791812b1c99 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 19 Apr 2024 13:19:08 -0400 Subject: [PATCH] Add and configure `pre-commit` (#62) --- .github/workflows/code-quality.yml | 28 +- .github/workflows/integration-tests.yml | 9 - .github/workflows/unit-tests.yml | 8 - .pre-commit-config.yaml | 55 +++ CHANGELOG.md | 1 - CONTRIBUTING.md | 2 +- dbt/adapters/postgres/relation.py | 30 +- .../postgres/relation_configs/index.py | 2 +- pyproject.toml | 72 +--- tests/functional/artifacts/test_artifacts.py | 11 - .../artifacts/test_previous_version_state.py | 2 +- .../context_methods/test_builtin_functions.py | 1 - tests/functional/dbt_debug/test_dbt_debug.py | 5 +- tests/functional/dbt_runner.py | 4 +- tests/functional/profiles/test_profile_dir.py | 168 -------- .../functional/profiles/test_profiles_yml.py | 65 --- .../schema_tests/test_schema_v2_tests.py | 4 +- .../shared_tests/test_hooks/test_hooks.py | 1 + .../test_simple_seed/test_simple_seed.py | 1 + tests/functional/source_overrides/fixtures.py | 387 ------------------ .../test_simple_source_override.py | 146 ------- .../test_source_overrides_duplicate_model.py | 68 --- tests/functional/test_connection_manager.py | 7 +- tests/functional/test_dbt_runner.py | 14 +- tests/functional/test_init.py | 1 - tests/unit/test_connection.py | 6 +- tests/unit/test_renamed_relations.py | 12 +- 27 files changed, 121 insertions(+), 989 deletions(-) create mode 100644 .pre-commit-config.yaml delete mode 100644 tests/functional/profiles/test_profile_dir.py delete mode 100644 tests/functional/profiles/test_profiles_yml.py delete mode 100644 tests/functional/source_overrides/fixtures.py delete mode 100644 tests/functional/source_overrides/test_simple_source_override.py delete mode 100644 tests/functional/source_overrides/test_source_overrides_duplicate_model.py diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml index 081079fc..1ca2ded0 100644 --- a/.github/workflows/code-quality.yml +++ b/.github/workflows/code-quality.yml @@ -9,16 +9,14 @@ on: workflow_dispatch: inputs: dbt_adapters_branch: - description: "The branch of dbt-adapters to evaluate" - type: string - required: true - default: "main" + description: "The branch of dbt-adapters to evaluate" + type: string + default: "main" workflow_call: inputs: dbt_adapters_branch: description: "The branch of dbt-adapters to evaluate" type: string - required: true default: "main" permissions: read-all @@ -27,16 +25,12 @@ permissions: read-all concurrency: group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} cancel-in-progress: true -env: - # set DBT_ADAPTERS_BRANCH to the input value if the event is a workflow_dispatch (workflow_call uses the same event_name), - # otherwise use 'main' - DBT_ADAPTERS_BRANCH: ${{ github.event_name == 'workflow_dispatch' && inputs.dbt_adapters_branch || 'main' }} + jobs: code-quality: name: Code Quality runs-on: ubuntu-latest - steps: - name: Check out repository uses: actions/checkout@v4 @@ -44,17 +38,13 @@ jobs: persist-credentials: false - name: Update Adapters and Core branches + if: ${{ contains(github.event_name, 'workflow_') }} shell: bash - run: | - ./.github/scripts/update_dev_packages.sh \ - $DBT_ADAPTERS_BRANCH \ - "main" + run: ./.github/scripts/update_dev_packages.sh ${{ inputs.dbt_adapters_branch }} "main" - name: Setup `hatch` uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - name: Run linters - run: hatch run lint:all - - - name: Run typechecks - run: hatch run typecheck:all + - name: Run code quality + shell: bash + run: hatch run code-quality diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml index 6bb37ec5..4bb423fa 100644 --- a/.github/workflows/integration-tests.yml +++ b/.github/workflows/integration-tests.yml @@ -31,7 +31,6 @@ on: required: false default: "main" - permissions: read-all # will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise @@ -101,14 +100,6 @@ jobs: POSTGRES_TEST_DATABASE: dbt POSTGRES_TEST_THREADS: 4 - - name: Publish results - uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main - if: always() - with: - source-file: "results.csv" - file-name: "integration_results" - python-version: ${{ matrix.python-version }} - psycopg2-check: name: "Test psycopg2 build version" runs-on: ${{ matrix.scenario.platform }} diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml index 69681c0f..91d8c578 100644 --- a/.github/workflows/unit-tests.yml +++ b/.github/workflows/unit-tests.yml @@ -39,11 +39,3 @@ jobs: - name: Run unit tests run: hatch run unit-tests:all shell: bash - - - name: Publish results - uses: dbt-labs/dbt-adapters/.github/actions/publish-results@main - if: always() - with: - source-file: "results.csv" - file-name: "unit_results" - python-version: ${{ matrix.python-version }} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..c869fe86 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +default_language_version: + python: python3 + +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/dbt-labs/pre-commit-hooks + rev: v0.1.0a1 + hooks: + - id: dbt-core-in-adapters-check + +- repo: https://github.com/psf/black + rev: 24.4.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + - --target-version=py39 + - --target-version=py310 + - --target-version=py311 + +- repo: https://github.com/pycqa/flake8 + rev: 7.0.0 + hooks: + - id: flake8 + exclude: tests/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.9.0 + hooks: + - id: mypy + args: + - --explicit-package-bases + - --ignore-missing-imports + - --pretty + - --show-error-codes + files: ^dbt/adapters/postgres + additional_dependencies: + - types-PyYAML + - types-protobuf + - types-pytz diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c13880e..f9ef4153 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,4 +29,3 @@ and is generated by [Changie](https://github.com/miniscruff/changie). ### Under the Hood * Update dependabot configuration to cover GHA - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f39e4cb5..427d2de5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -164,7 +164,7 @@ Remember to commit and push the file that's created. ### Signing the CLA -> **_NOTE:_** All contributors to `dbt-postgres` must sign the +> **_NOTE:_** All contributors to `dbt-postgres` must sign the > [Contributor License Agreement](https://docs.getdbt.com/docs/contributor-license-agreements)(CLA). Maintainers will be unable to merge contributions until the contributor signs the CLA. diff --git a/dbt/adapters/postgres/relation.py b/dbt/adapters/postgres/relation.py index 677b12ac..05d55237 100644 --- a/dbt/adapters/postgres/relation.py +++ b/dbt/adapters/postgres/relation.py @@ -20,19 +20,23 @@ @dataclass(frozen=True, eq=False, repr=False) class PostgresRelation(BaseRelation): - renameable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - } - )) - replaceable_relations: FrozenSet[RelationType] = field(default_factory=lambda: frozenset( - { - RelationType.View, - RelationType.Table, - } - )) + renameable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + ) + ) + replaceable_relations: FrozenSet[RelationType] = field( + default_factory=lambda: frozenset( + { + RelationType.View, + RelationType.Table, + } + ) + ) def __post_init__(self): # Check for length of Postgres table/view names. diff --git a/dbt/adapters/postgres/relation_configs/index.py b/dbt/adapters/postgres/relation_configs/index.py index c4863073..0ed3e8aa 100644 --- a/dbt/adapters/postgres/relation_configs/index.py +++ b/dbt/adapters/postgres/relation_configs/index.py @@ -23,7 +23,7 @@ class PostgresIndexMethod(StrEnum): @classmethod def default(cls) -> "PostgresIndexMethod": - return cls.btree + return cls("btree") @dataclass(frozen=True, eq=True, unsafe_hash=True) diff --git a/pyproject.toml b/pyproject.toml index 996ea46a..0e93423c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,56 +59,30 @@ path = "dbt/adapters/postgres/__version__.py" dependencies = [ "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + 'pre-commit==3.7.0;python_version>="3.9"', + 'pre-commit==3.5.0;python_version=="3.8"', ] - -[tool.hatch.envs.lint] -detached = true -dependencies = [ - "black>=24.3", - "flake8", - "Flake8-pyproject", -] -[tool.hatch.envs.lint.scripts] -all = [ - "black", - "flake8", -] -black = "python -m black ." -flake8 = "python -m flake8 ." - -[tool.hatch.envs.typecheck] -dependencies = [ - "mypy", - "types-protobuf", - "types-pytz", -] -[tool.hatch.envs.typecheck.scripts] -all = "python -m mypy ." +[tool.hatch.envs.default.scripts] +dev = "pre-commit install" +code-quality = "pre-commit run --all-files" [tool.hatch.envs.unit-tests] dependencies = [ + "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", + "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", + "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", "freezegun", "pytest", "pytest-dotenv", "pytest-mock", "pytest-xdist", ] -extra-dependencies = [ - # TODO: remove `dbt-core` dependencies from unit tests - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", - "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", -] [tool.hatch.envs.unit-tests.scripts] all = "python -m pytest {args:tests/unit}" [tool.hatch.envs.integration-tests] template = "unit-tests" extra-dependencies = [ - # TODO: remove `dbt-core` dependencies from integration tests - "dbt-adapters @ git+https://github.com/dbt-labs/dbt-adapters.git", - "dbt-common @ git+https://github.com/dbt-labs/dbt-common.git", - "dbt-core @ git+https://github.com/dbt-labs/dbt-core.git#subdirectory=core", "dbt-tests-adapter @ git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter", ] [tool.hatch.envs.integration-tests.env-vars] @@ -125,8 +99,6 @@ dependencies = [ "twine", "check-wheel-contents", ] - - [tool.hatch.envs.build.scripts] check-all = [ "- check-wheel", @@ -143,35 +115,9 @@ check-sdist = [ "pip freeze | grep dbt-postgres", ] -[tool.black] -line-length = 99 -target-version = ['py38'] - -[tool.flake8] -select = ["E", "W", "F"] -ignore = ["E203", "E501", "E741", "W503", "W504"] -exclude = ["tests", "venv", ".hatch_venvs"] -per-file-ignores = ["*/__init__.py: F401"] - -[tool.mypy] -namespace_packages = true -show_error_codes = true -explicit_package_bases = true -ignore_missing_imports = true -pretty = true -files = [ - "dbt/adapters/postgres", - "tests/unit", -] -exclude = [ - "tests/functional", - "venv", - ".hatch_venvs", -] - [tool.pytest] env_files = ["test.env"] testpaths = [ "tests/functional", "tests/unit", -] \ No newline at end of file +] diff --git a/tests/functional/artifacts/test_artifacts.py b/tests/functional/artifacts/test_artifacts.py index 756ee73f..ca2c03f0 100644 --- a/tests/functional/artifacts/test_artifacts.py +++ b/tests/functional/artifacts/test_artifacts.py @@ -1,4 +1,3 @@ -from datetime import datetime import jsonschema import os @@ -12,16 +11,6 @@ ) import pytest -from tests.functional.artifacts.expected_manifest import ( - expected_references_manifest, - expected_seeded_manifest, - expected_versions_manifest, -) -from tests.functional.artifacts.expected_run_results import ( - expected_references_run_results, - expected_run_results, - expected_versions_run_results, -) from tests.functional.utils import run_dbt, run_dbt_and_capture diff --git a/tests/functional/artifacts/test_previous_version_state.py b/tests/functional/artifacts/test_previous_version_state.py index a19a2486..5ed37b9a 100644 --- a/tests/functional/artifacts/test_previous_version_state.py +++ b/tests/functional/artifacts/test_previous_version_state.py @@ -5,7 +5,7 @@ from dbt.artifacts.schemas.base import get_artifact_schema_version from dbt.artifacts.schemas.run import RunResultsArtifact from dbt.contracts.graph.manifest import WritableManifest -from dbt.exceptions import IncompatibleSchemaError +from dbt.artifacts.exceptions import IncompatibleSchemaError from dbt.tests.util import get_manifest import pytest diff --git a/tests/functional/context_methods/test_builtin_functions.py b/tests/functional/context_methods/test_builtin_functions.py index f5d7b92f..b8a47b34 100644 --- a/tests/functional/context_methods/test_builtin_functions.py +++ b/tests/functional/context_methods/test_builtin_functions.py @@ -1,5 +1,4 @@ import json -import os from dbt.tests.util import write_file from dbt_common.exceptions import CompilationError diff --git a/tests/functional/dbt_debug/test_dbt_debug.py b/tests/functional/dbt_debug/test_dbt_debug.py index 3e2a182e..e318322e 100644 --- a/tests/functional/dbt_debug/test_dbt_debug.py +++ b/tests/functional/dbt_debug/test_dbt_debug.py @@ -1,9 +1,6 @@ import pytest -import os -import re -import yaml -from tests.functional.utils import run_dbt, run_dbt_and_capture +from tests.functional.utils import run_dbt_and_capture MODELS__MODEL_SQL = """ seled 1 as id diff --git a/tests/functional/dbt_runner.py b/tests/functional/dbt_runner.py index 01ebc873..ba7aa2ba 100644 --- a/tests/functional/dbt_runner.py +++ b/tests/functional/dbt_runner.py @@ -25,13 +25,15 @@ def assert_run_results_have_compiled_node_attributes( class dbtTestRunner(dbtRunner): + exit_assertions: List[Callable[[List[str], dbtRunnerResult], None]] + def __init__( self, manifest: Optional[Manifest] = None, callbacks: Optional[List[Callable[[EventMsg], None]]] = None, exit_assertions: Optional[List[Callable[[List[str], dbtRunnerResult], None]]] = None, ): - self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS + self.exit_assertions = exit_assertions if exit_assertions else _STANDARD_ASSERTIONS # type: ignore super().__init__(manifest, callbacks) def invoke(self, args: List[str], **kwargs) -> dbtRunnerResult: diff --git a/tests/functional/profiles/test_profile_dir.py b/tests/functional/profiles/test_profile_dir.py deleted file mode 100644 index 282c978c..00000000 --- a/tests/functional/profiles/test_profile_dir.py +++ /dev/null @@ -1,168 +0,0 @@ -from contextlib import contextmanager -import os -from pathlib import Path - -from dbt.tests.util import rm_file, write_file -import pytest -import yaml - -from tests.functional.utils import run_dbt, run_dbt_and_capture - - -@pytest.fixture(scope="class") -def profiles_yml(profiles_root, dbt_profile_data): - write_file(yaml.safe_dump(dbt_profile_data), profiles_root, "profiles.yml") - return dbt_profile_data - - -@pytest.fixture(scope="class") -def profiles_home_root(): - return os.path.join(os.path.expanduser("~"), ".dbt") - - -@pytest.fixture(scope="class") -def profiles_env_root(tmpdir_factory): - path = tmpdir_factory.mktemp("profile_env") - # environment variables are lowercased for some reason in _get_flag_value_from_env within dbt.flags - return str(path).lower() - - -@pytest.fixture(scope="class") -def profiles_flag_root(tmpdir_factory): - return tmpdir_factory.mktemp("profile_flag") - - -@pytest.fixture(scope="class") -def profiles_project_root(project): - return project.project_root - - -@pytest.fixture(scope="class") -def cwd(): - return os.getcwd() - - -@pytest.fixture(scope="class") -def cwd_parent(cwd): - return os.path.dirname(cwd) - - -@pytest.fixture(scope="class") -def cwd_child(): - # pick any child directory of the dbt project - return Path(os.getcwd()) / "macros" - - -@pytest.fixture -def write_profiles_yml(request): - def _write_profiles_yml(profiles_dir, dbt_profile_contents): - def cleanup(): - rm_file(Path(profiles_dir) / "profiles.yml") - - request.addfinalizer(cleanup) - write_file(yaml.safe_dump(dbt_profile_contents), profiles_dir, "profiles.yml") - - return _write_profiles_yml - - -# https://gist.github.com/igniteflow/7267431?permalink_comment_id=2551951#gistcomment-2551951 -@contextmanager -def environ(env): - """Temporarily set environment variables inside the context manager and - fully restore previous environment afterwards - """ - original_env = {key: os.getenv(key) for key in env} - os.environ.update(env) - try: - yield - finally: - for key, value in original_env.items(): - if value is None: - del os.environ[key] - else: - os.environ[key] = value - - -class TestProfilesMayNotExist: - def test_debug(self, project): - # The database will not be able to connect; expect neither a pass or a failure (but not an exception) - run_dbt(["debug", "--profiles-dir", "does_not_exist"], expect_pass=None) - - def test_deps(self, project): - run_dbt(["deps", "--profiles-dir", "does_not_exist"]) - - -class TestProfiles: - def dbt_debug(self, project_dir_cli_arg=None, profiles_dir_cli_arg=None): - # begin with no command-line args or user config (from profiles.yml) - # flags.set_from_args(Namespace(), {}) - command = ["debug"] - - if project_dir_cli_arg: - command.extend(["--project-dir", str(project_dir_cli_arg)]) - - if profiles_dir_cli_arg: - command.extend(["--profiles-dir", str(profiles_dir_cli_arg)]) - - # get the output of `dbt debug` regardless of the exit code - return run_dbt_and_capture(command, expect_pass=None) - - @pytest.mark.parametrize( - "project_dir_cli_arg, working_directory", - [ - # 3 different scenarios for `--project-dir` flag and current working directory - (None, "cwd"), # no --project-dir flag and cwd is project directory - (None, "cwd_child"), # no --project-dir flag and cwd is a project subdirectory - ("cwd", "cwd_parent"), # use --project-dir flag and cwd is outside of it - ], - ) - def test_profiles( - self, - project_dir_cli_arg, - working_directory, - write_profiles_yml, - dbt_profile_data, - profiles_home_root, - profiles_project_root, - profiles_flag_root, - profiles_env_root, - request, - ): - """Verify priority order to search for profiles.yml configuration. - - Reverse priority order: - 1. HOME directory - 2. DBT_PROFILES_DIR environment variable - 3. --profiles-dir command-line argument - - Specification later in this list will take priority over earlier ones, even when both are provided. - """ - - # https://pypi.org/project/pytest-lazy-fixture/ is an alternative to using request.getfixturevalue - if project_dir_cli_arg is not None: - project_dir_cli_arg = request.getfixturevalue(project_dir_cli_arg) - - if working_directory is not None: - working_directory = request.getfixturevalue(working_directory) - - # start in the specified directory - if working_directory is not None: - os.chdir(working_directory) - # default case with profiles.yml in the HOME directory - _, stdout = self.dbt_debug(project_dir_cli_arg) - assert f"Using profiles.yml file at {profiles_home_root}" in stdout - - # set DBT_PROFILES_DIR environment variable for the remainder of the cases - env_vars = {"DBT_PROFILES_DIR": profiles_env_root} - with environ(env_vars): - _, stdout = self.dbt_debug(project_dir_cli_arg) - assert f"Using profiles.yml file at {profiles_env_root}" in stdout - - # This additional case is also within the context manager because we want to verify - # that it takes priority even when the relevant environment variable is also set - - # set --profiles-dir on the command-line - _, stdout = self.dbt_debug( - project_dir_cli_arg, profiles_dir_cli_arg=profiles_flag_root - ) - assert f"Using profiles.yml file at {profiles_flag_root}" in stdout diff --git a/tests/functional/profiles/test_profiles_yml.py b/tests/functional/profiles/test_profiles_yml.py deleted file mode 100644 index c4eeabbd..00000000 --- a/tests/functional/profiles/test_profiles_yml.py +++ /dev/null @@ -1,65 +0,0 @@ -from pathlib import Path - -from dbt.cli.main import dbtRunner -from test_profile_dir import environ - - -jinjaesque_password = "no{{jinja{%re{#ndering" - -profile_with_jinjaesque_password = f"""test: - outputs: - default: - dbname: my_db - host: localhost - password: {jinjaesque_password} - port: 12345 - schema: dummy - threads: 4 - type: postgres - user: peter.webb - target: default -""" - -profile_with_env_password = """test: - outputs: - default: - dbname: my_db - host: localhost - password: "{{ env_var('DBT_PASSWORD') }}" - port: 12345 - schema: dummy - threads: 4 - type: postgres - user: peter.webb - target: default -""" - - -class TestProfileParsing: - def write_profiles_yml(self, profiles_root, content) -> None: - with open(Path(profiles_root, "profiles.yml"), "w") as profiles_yml: - profiles_yml.write(content) - - def test_password_not_jinja_rendered_when_invalid(self, project, profiles_root) -> None: - """Verifies that passwords that contain Jinja control characters, but which are - not valid Jinja, do not cause errors.""" - self.write_profiles_yml(profiles_root, profile_with_jinjaesque_password) - - events = [] - result = dbtRunner(callbacks=[events.append]).invoke(["parse"]) - assert result.success - - for e in events: - assert "no{{jinja{%re{#ndering" not in e.info.msg - - def test_password_jinja_rendered_when_valid(self, project, profiles_root) -> None: - """Verifies that a password value that is valid Jinja is rendered as such, - and that it doesn't cause problems if the resulting value looks like Jinja""" - self.write_profiles_yml(profiles_root, profile_with_env_password) - - events = [] - with environ({"DBT_PASSWORD": jinjaesque_password}): - result = dbtRunner(callbacks=[events.append]).invoke(["parse"]) - - assert result.success - assert project.adapter.config.credentials.password == jinjaesque_password diff --git a/tests/functional/schema_tests/test_schema_v2_tests.py b/tests/functional/schema_tests/test_schema_v2_tests.py index a268f096..aae164a1 100644 --- a/tests/functional/schema_tests/test_schema_v2_tests.py +++ b/tests/functional/schema_tests/test_schema_v2_tests.py @@ -2,7 +2,7 @@ import re from dbt.contracts.results import TestStatus -from dbt.exceptions import ParsingError, DuplicateResourceNameError +from dbt.exceptions import ParsingError from dbt.tests.fixtures.project import write_project_files from dbt.tests.util import run_dbt, write_file from dbt_common.exceptions import CompilationError @@ -20,8 +20,6 @@ custom_generic_test_names__schema_yml, custom_generic_test_names_alt_format__model_a, custom_generic_test_names_alt_format__schema_yml, - dupe_generic_tests_collide__model_a, - dupe_generic_tests_collide__schema_yml, ephemeral__ephemeral_sql, ephemeral__schema_yml, invalid_schema_models__model_sql, diff --git a/tests/functional/shared_tests/test_hooks/test_hooks.py b/tests/functional/shared_tests/test_hooks/test_hooks.py index 84381130..7e832038 100644 --- a/tests/functional/shared_tests/test_hooks/test_hooks.py +++ b/tests/functional/shared_tests/test_hooks/test_hooks.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it uses a `data` directory. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.hooks.test_model_hooks import ( BasePrePostModelHooks, BaseHookRefs, diff --git a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py index cd849788..61664ca9 100644 --- a/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py +++ b/tests/functional/shared_tests/test_simple_seed/test_simple_seed.py @@ -2,6 +2,7 @@ This file needs to be in its own directory because it creates a `data` directory at run time. Placing this file in its own directory avoids collisions. """ + from dbt.tests.adapter.simple_seed.test_seed import ( BaseBasicSeedTests, BaseSeedConfigFullRefreshOn, diff --git a/tests/functional/source_overrides/fixtures.py b/tests/functional/source_overrides/fixtures.py deleted file mode 100644 index f7f49235..00000000 --- a/tests/functional/source_overrides/fixtures.py +++ /dev/null @@ -1,387 +0,0 @@ -import pytest - - -dupe_models__schema2_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - freshness: null - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - data_tests: - - not_null - - unique - - name: color - data_tests: - - unique - - name: my_other_table - freshness: null - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - - freshness: - error_after: {count: 1, period: day} - -""" - -dupe_models__schema1_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - freshness: null - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - data_tests: - - not_null - - unique - - name: color - data_tests: - - unique - - name: my_other_table - freshness: null - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: day} - -""" - -local_dependency__dbt_project_yml = """ -config-version: 2 -name: localdep - -version: '1.0' - -profile: 'default' - -seeds: - quote_columns: False - -seed-paths: ['seeds'] - -""" - -local_dependency__models__schema_yml = """ -version: 2 -sources: - - name: my_source - schema: invalid_schema - database: invalid_database - freshness: - error_after: {count: 3, period: hour} - tables: - - name: my_table - freshness: null - identifier: my_seed - columns: - - name: id - data_tests: - - unique - - not_null - - name: color - data_tests: - - unique - - not_null - - name: my_other_table - identifier: my_other_seed - columns: - - name: id - data_tests: - - unique - - not_null - - name: letter - data_tests: - - unique - - not_null - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: hour} - - name: my_other_source - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 1, period: day} - tables: - - name: never_fresh - loaded_at_field: updated_at - -""" - -local_dependency__models__my_model_sql = """ - -{{ config(materialized="table") }} - -with colors as ( - select id, color from {{ source('my_source', 'my_table') }} -), -letters as ( - select id, letter from {{ source('my_source', 'my_other_table') }} -) -select letter, color from colors join letters using (id) - -""" - -local_dependency__seeds__my_other_seed_csv = """id,letter -1,r -2,g -3,b -""" - -local_dependency__seeds__my_seed_csv = """id,color -1,red -2,green -3,blue -""" - -local_dependency__seeds__keep__never_fresh_csv = """favorite_color,id,first_name,email,ip_address,updated_at -blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 -blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 -blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 -blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 -blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 -blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 -blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 -blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 -blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 -blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 -blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 -blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 -blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 -blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 -blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 -blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 -blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 -blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 -blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 -blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 -blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 -blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 -blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 -blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 -blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 -blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 -blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 -blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 -blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 -blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 -blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 -blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 -blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 -blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 -blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 -blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 -blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 -blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 -blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 -blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 -blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 -blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 -blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 -blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 -blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 -blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 -blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 -blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 -blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 -blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 -""" - -local_dependency__seeds__keep__snapshot_freshness_base_csv = """favorite_color,id,first_name,email,ip_address,updated_at -blue,1,Larry,lking0@miitbeian.gov.cn,'69.135.206.194',2008-09-12 19:08:31 -blue,2,Larry,lperkins1@toplist.cz,'64.210.133.162',1978-05-09 04:15:14 -blue,3,Anna,amontgomery2@miitbeian.gov.cn,'168.104.64.114',2011-10-16 04:07:57 -blue,4,Sandra,sgeorge3@livejournal.com,'229.235.252.98',1973-07-19 10:52:43 -blue,5,Fred,fwoods4@google.cn,'78.229.170.124',2012-09-30 16:38:29 -blue,6,Stephen,shanson5@livejournal.com,'182.227.157.105',1995-11-07 21:40:50 -blue,7,William,wmartinez6@upenn.edu,'135.139.249.50',1982-09-05 03:11:59 -blue,8,Jessica,jlong7@hao123.com,'203.62.178.210',1991-10-16 11:03:15 -blue,9,Douglas,dwhite8@tamu.edu,'178.187.247.1',1979-10-01 09:49:48 -blue,10,Lisa,lcoleman9@nydailynews.com,'168.234.128.249',2011-05-26 07:45:49 -blue,11,Ralph,rfieldsa@home.pl,'55.152.163.149',1972-11-18 19:06:11 -blue,12,Louise,lnicholsb@samsung.com,'141.116.153.154',2014-11-25 20:56:14 -blue,13,Clarence,cduncanc@sfgate.com,'81.171.31.133',2011-11-17 07:02:36 -blue,14,Daniel,dfranklind@omniture.com,'8.204.211.37',1980-09-13 00:09:04 -blue,15,Katherine,klanee@auda.org.au,'176.96.134.59',1997-08-22 19:36:56 -blue,16,Billy,bwardf@wikia.com,'214.108.78.85',2003-10-19 02:14:47 -blue,17,Annie,agarzag@ocn.ne.jp,'190.108.42.70',1988-10-28 15:12:35 -blue,18,Shirley,scolemanh@fastcompany.com,'109.251.164.84',1988-08-24 10:50:57 -blue,19,Roger,rfrazieri@scribd.com,'38.145.218.108',1985-12-31 15:17:15 -blue,20,Lillian,lstanleyj@goodreads.com,'47.57.236.17',1970-06-08 02:09:05 -blue,21,Aaron,arodriguezk@nps.gov,'205.245.118.221',1985-10-11 23:07:49 -blue,22,Patrick,pparkerl@techcrunch.com,'19.8.100.182',2006-03-29 12:53:56 -blue,23,Phillip,pmorenom@intel.com,'41.38.254.103',2011-11-07 15:35:43 -blue,24,Henry,hgarcian@newsvine.com,'1.191.216.252',2008-08-28 08:30:44 -blue,25,Irene,iturnero@opera.com,'50.17.60.190',1994-04-01 07:15:02 -blue,26,Andrew,adunnp@pen.io,'123.52.253.176',2000-11-01 06:03:25 -blue,27,David,dgutierrezq@wp.com,'238.23.203.42',1988-01-25 07:29:18 -blue,28,Henry,hsanchezr@cyberchimps.com,'248.102.2.185',1983-01-01 13:36:37 -blue,29,Evelyn,epetersons@gizmodo.com,'32.80.46.119',1979-07-16 17:24:12 -blue,30,Tammy,tmitchellt@purevolume.com,'249.246.167.88',2001-04-03 10:00:23 -blue,31,Jacqueline,jlittleu@domainmarket.com,'127.181.97.47',1986-02-11 21:35:50 -blue,32,Earl,eortizv@opera.com,'166.47.248.240',1996-07-06 08:16:27 -blue,33,Juan,jgordonw@sciencedirect.com,'71.77.2.200',1987-01-31 03:46:44 -blue,34,Diane,dhowellx@nyu.edu,'140.94.133.12',1994-06-11 02:30:05 -blue,35,Randy,rkennedyy@microsoft.com,'73.255.34.196',2005-05-26 20:28:39 -blue,36,Janice,jriveraz@time.com,'22.214.227.32',1990-02-09 04:16:52 -blue,37,Laura,lperry10@diigo.com,'159.148.145.73',2015-03-17 05:59:25 -blue,38,Gary,gray11@statcounter.com,'40.193.124.56',1970-01-27 10:04:51 -blue,39,Jesse,jmcdonald12@typepad.com,'31.7.86.103',2009-03-14 08:14:29 -blue,40,Sandra,sgonzalez13@goodreads.com,'223.80.168.239',1993-05-21 14:08:54 -blue,41,Scott,smoore14@archive.org,'38.238.46.83',1980-08-30 11:16:56 -blue,42,Phillip,pevans15@cisco.com,'158.234.59.34',2011-12-15 23:26:31 -blue,43,Steven,sriley16@google.ca,'90.247.57.68',2011-10-29 19:03:28 -blue,44,Deborah,dbrown17@hexun.com,'179.125.143.240',1995-04-10 14:36:07 -blue,45,Lori,lross18@ow.ly,'64.80.162.180',1980-12-27 16:49:15 -blue,46,Sean,sjackson19@tumblr.com,'240.116.183.69',1988-06-12 21:24:45 -blue,47,Terry,tbarnes1a@163.com,'118.38.213.137',1997-09-22 16:43:19 -blue,48,Dorothy,dross1b@ebay.com,'116.81.76.49',2005-02-28 13:33:24 -blue,49,Samuel,swashington1c@house.gov,'38.191.253.40',1989-01-19 21:15:48 -blue,50,Ralph,rcarter1d@tinyurl.com,'104.84.60.174',2007-08-11 10:21:49 -green,51,Wayne,whudson1e@princeton.edu,'90.61.24.102',1983-07-03 16:58:12 -green,52,Rose,rjames1f@plala.or.jp,'240.83.81.10',1995-06-08 11:46:23 -green,53,Louise,lcox1g@theglobeandmail.com,'105.11.82.145',2016-09-19 14:45:51 -green,54,Kenneth,kjohnson1h@independent.co.uk,'139.5.45.94',1976-08-17 11:26:19 -green,55,Donna,dbrown1i@amazon.co.uk,'19.45.169.45',2006-05-27 16:51:40 -green,56,Johnny,jvasquez1j@trellian.com,'118.202.238.23',1975-11-17 08:42:32 -green,57,Patrick,pramirez1k@tamu.edu,'231.25.153.198',1997-08-06 11:51:09 -green,58,Helen,hlarson1l@prweb.com,'8.40.21.39',1993-08-04 19:53:40 -green,59,Patricia,pspencer1m@gmpg.org,'212.198.40.15',1977-08-03 16:37:27 -green,60,Joseph,jspencer1n@marriott.com,'13.15.63.238',2005-07-23 20:22:06 -green,61,Phillip,pschmidt1o@blogtalkradio.com,'177.98.201.190',1976-05-19 21:47:44 -green,62,Joan,jwebb1p@google.ru,'105.229.170.71',1972-09-07 17:53:47 -green,63,Phyllis,pkennedy1q@imgur.com,'35.145.8.244',2000-01-01 22:33:37 -green,64,Katherine,khunter1r@smh.com.au,'248.168.205.32',1991-01-09 06:40:24 -green,65,Laura,lvasquez1s@wiley.com,'128.129.115.152',1997-10-23 12:04:56 -green,66,Juan,jdunn1t@state.gov,'44.228.124.51',2004-11-10 05:07:35 -green,67,Judith,jholmes1u@wiley.com,'40.227.179.115',1977-08-02 17:01:45 -green,68,Beverly,bbaker1v@wufoo.com,'208.34.84.59',2016-03-06 20:07:23 -green,69,Lawrence,lcarr1w@flickr.com,'59.158.212.223',1988-09-13 06:07:21 -green,70,Gloria,gwilliams1x@mtv.com,'245.231.88.33',1995-03-18 22:32:46 -green,71,Steven,ssims1y@cbslocal.com,'104.50.58.255',2001-08-05 21:26:20 -green,72,Betty,bmills1z@arstechnica.com,'103.177.214.220',1981-12-14 21:26:54 -green,73,Mildred,mfuller20@prnewswire.com,'151.158.8.130',2000-04-19 10:13:55 -green,74,Donald,dday21@icq.com,'9.178.102.255',1972-12-03 00:58:24 -green,75,Eric,ethomas22@addtoany.com,'85.2.241.227',1992-11-01 05:59:30 -green,76,Joyce,jarmstrong23@sitemeter.com,'169.224.20.36',1985-10-24 06:50:01 -green,77,Maria,mmartinez24@amazonaws.com,'143.189.167.135',2005-10-05 05:17:42 -green,78,Harry,hburton25@youtube.com,'156.47.176.237',1978-03-26 05:53:33 -green,79,Kevin,klawrence26@hao123.com,'79.136.183.83',1994-10-12 04:38:52 -green,80,David,dhall27@prweb.com,'133.149.172.153',1976-12-15 16:24:24 -green,81,Kathy,kperry28@twitter.com,'229.242.72.228',1979-03-04 02:58:56 -green,82,Adam,aprice29@elegantthemes.com,'13.145.21.10',1982-11-07 11:46:59 -green,83,Brandon,bgriffin2a@va.gov,'73.249.128.212',2013-10-30 05:30:36 -green,84,Henry,hnguyen2b@discovery.com,'211.36.214.242',1985-01-09 06:37:27 -green,85,Eric,esanchez2c@edublogs.org,'191.166.188.251',2004-05-01 23:21:42 -green,86,Jason,jlee2d@jimdo.com,'193.92.16.182',1973-01-08 09:05:39 -green,87,Diana,drichards2e@istockphoto.com,'19.130.175.245',1994-10-05 22:50:49 -green,88,Andrea,awelch2f@abc.net.au,'94.155.233.96',2002-04-26 08:41:44 -green,89,Louis,lwagner2g@miitbeian.gov.cn,'26.217.34.111',2003-08-25 07:56:39 -green,90,Jane,jsims2h@seesaa.net,'43.4.220.135',1987-03-20 20:39:04 -green,91,Larry,lgrant2i@si.edu,'97.126.79.34',2000-09-07 20:26:19 -green,92,Louis,ldean2j@prnewswire.com,'37.148.40.127',2011-09-16 20:12:14 -green,93,Jennifer,jcampbell2k@xing.com,'38.106.254.142',1988-07-15 05:06:49 -green,94,Wayne,wcunningham2l@google.com.hk,'223.28.26.187',2009-12-15 06:16:54 -green,95,Lori,lstevens2m@icq.com,'181.250.181.58',1984-10-28 03:29:19 -green,96,Judy,jsimpson2n@marriott.com,'180.121.239.219',1986-02-07 15:18:10 -green,97,Phillip,phoward2o@usa.gov,'255.247.0.175',2002-12-26 08:44:45 -green,98,Gloria,gwalker2p@usa.gov,'156.140.7.128',1997-10-04 07:58:58 -green,99,Paul,pjohnson2q@umn.edu,'183.59.198.197',1991-11-14 12:33:55 -green,100,Frank,fgreene2r@blogspot.com,'150.143.68.121',2010-06-12 23:55:39 -""" - -models__schema_yml = """ -version: 2 -sources: - - name: my_source - overrides: localdep - schema: "{{ target.schema }}" - database: "{{ target.database }}" - freshness: - error_after: {count: 3, period: day} - tables: - - name: my_table - freshness: null - identifier: my_real_seed - # on the override, the "color" column is only unique, it can be null! - columns: - - name: id - data_tests: - - not_null - - unique - - name: color - data_tests: - - unique - - name: my_other_table - freshness: null - identifier: my_real_other_seed - - name: snapshot_freshness - identifier: snapshot_freshness_base - loaded_at_field: updated_at - freshness: - error_after: {count: 1, period: day} - -""" - -seeds__expected_result_csv = """letter,color -c,cyan -m,magenta -y,yellow -k,key -""" - -seeds__my_real_other_seed_csv = """id,letter -1,c -2,m -3,y -4,k -""" - -seeds__my_real_seed_csv = """id,color -1,cyan -2,magenta -3,yellow -4,key -5,NULL -""" - - -@pytest.fixture(scope="class") -def local_dependency(): - return { - "dbt_project.yml": local_dependency__dbt_project_yml, - "models": { - "schema.yml": local_dependency__models__schema_yml, - "my_model.sql": local_dependency__models__my_model_sql, - }, - "seeds": { - "my_other_seed.csv": local_dependency__seeds__my_other_seed_csv, - "my_seed.csv": local_dependency__seeds__my_seed_csv, - "keep": { - "never_fresh.csv": local_dependency__seeds__keep__never_fresh_csv, - "snapshot_freshness_base.csv": local_dependency__seeds__keep__snapshot_freshness_base_csv, - }, - }, - } diff --git a/tests/functional/source_overrides/test_simple_source_override.py b/tests/functional/source_overrides/test_simple_source_override.py deleted file mode 100644 index d1cd3352..00000000 --- a/tests/functional/source_overrides/test_simple_source_override.py +++ /dev/null @@ -1,146 +0,0 @@ -from datetime import datetime, timedelta - -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import check_relations_equal, run_dbt, update_config_file -import pytest - -from tests.functional.source_overrides.fixtures import ( - local_dependency, - models__schema_yml, - seeds__expected_result_csv, - seeds__my_real_other_seed_csv, - seeds__my_real_seed_csv, -) - - -class TestSourceOverride: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, local_dependency): # noqa: F811 - write_project_files(project_root, "local_dependency", local_dependency) - - @pytest.fixture(scope="class") - def models(self): - return {"schema.yml": models__schema_yml} - - @pytest.fixture(scope="class") - def seeds(self): - return { - "expected_result.csv": seeds__expected_result_csv, - "my_real_other_seed.csv": seeds__my_real_other_seed_csv, - "my_real_seed.csv": seeds__my_real_seed_csv, - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "local_dependency", - }, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "localdep": { - "enabled": False, - "keep": { - "enabled": True, - }, - }, - "quote_columns": False, - }, - "sources": { - "localdep": { - "my_other_source": { - "enabled": False, - } - } - }, - } - - def _set_updated_at_to(self, insert_id, delta, project): - insert_time = datetime.utcnow() + delta - timestr = insert_time.strftime("%Y-%m-%d %H:%M:%S") - # favorite_color,id,first_name,email,ip_address,updated_at - - quoted_columns = ",".join( - project.adapter.quote(c) - for c in ("favorite_color", "id", "first_name", "email", "ip_address", "updated_at") - ) - - kwargs = { - "schema": project.test_schema, - "time": timestr, - "id": insert_id, - "source": project.adapter.quote("snapshot_freshness_base"), - "quoted_columns": quoted_columns, - } - - raw_code = """INSERT INTO {schema}.{source} - ({quoted_columns}) - VALUES ( - 'blue',{id},'Jake','abc@example.com','192.168.1.1','{time}' - )""".format( - **kwargs - ) - - project.run_sql(raw_code) - - return insert_id + 1 - - def test_source_overrides(self, project): - insert_id = 101 - - run_dbt(["deps"]) - - seed_results = run_dbt(["seed"]) - assert len(seed_results) == 5 - - # There should be 7, as we disabled 1 test of the original 8 - test_results = run_dbt(["test"]) - assert len(test_results) == 7 - - results = run_dbt(["run"]) - assert len(results) == 1 - - check_relations_equal(project.adapter, ["expected_result", "my_model"]) - - # set the updated_at field of this seed to last week - insert_id = self._set_updated_at_to(insert_id, timedelta(days=-7), project) - # if snapshot-freshness fails, freshness just didn't happen! - results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) - # we disabled my_other_source, so we only run the one freshness check - # in - assert len(results) == 1 - # If snapshot-freshness passes, that means error_after was - # applied from the source override but not the source table override - insert_id = self._set_updated_at_to(insert_id, timedelta(days=-2), project) - results = run_dbt( - ["source", "snapshot-freshness"], - expect_pass=False, - ) - assert len(results) == 1 - - insert_id = self._set_updated_at_to(insert_id, timedelta(hours=-12), project) - results = run_dbt(["source", "snapshot-freshness"], expect_pass=True) - assert len(results) == 1 - - # update source to be enabled - new_source_config = { - "sources": { - "localdep": { - "my_other_source": { - "enabled": True, - } - } - } - } - update_config_file(new_source_config, project.project_root, "dbt_project.yml") - - # enable my_other_source, snapshot freshness should fail due to the new - # not-fresh source - results = run_dbt(["source", "snapshot-freshness"], expect_pass=False) - assert len(results) == 2 diff --git a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py b/tests/functional/source_overrides/test_source_overrides_duplicate_model.py deleted file mode 100644 index 0a9ab0d8..00000000 --- a/tests/functional/source_overrides/test_source_overrides_duplicate_model.py +++ /dev/null @@ -1,68 +0,0 @@ -import os - -from dbt.tests.fixtures.project import write_project_files -from dbt.tests.util import run_dbt -from dbt_common.exceptions import CompilationError -import pytest - -from tests.functional.source_overrides.fixtures import ( - dupe_models__schema1_yml, - dupe_models__schema2_yml, - local_dependency, -) - - -class TestSourceOverrideDuplicates: - @pytest.fixture(scope="class", autouse=True) - def setUp(self, project_root, local_dependency): # noqa: F811 - write_project_files(project_root, "local_dependency", local_dependency) - - @pytest.fixture(scope="class") - def models(self): - return { - "schema2.yml": dupe_models__schema2_yml, - "schema1.yml": dupe_models__schema1_yml, - } - - @pytest.fixture(scope="class") - def packages(self): - return { - "packages": [ - { - "local": "local_dependency", - }, - ] - } - - @pytest.fixture(scope="class") - def project_config_update(self): - return { - "seeds": { - "localdep": { - "enabled": False, - "keep": { - "enabled": True, - }, - }, - "quote_columns": False, - }, - "sources": { - "localdep": { - "my_other_source": { - "enabled": False, - } - } - }, - } - - def test_source_duplicate_overrides(self, project): - run_dbt(["deps"]) - with pytest.raises(CompilationError) as exc: - run_dbt(["compile"]) - - assert "dbt found two schema.yml entries for the same source named" in str(exc.value) - assert "one of these files" in str(exc.value) - schema1_path = os.path.join("models", "schema1.yml") - schema2_path = os.path.join("models", "schema2.yml") - assert schema1_path in str(exc.value) - assert schema2_path in str(exc.value) diff --git a/tests/functional/test_connection_manager.py b/tests/functional/test_connection_manager.py index 778b2fbb..b44181ff 100644 --- a/tests/functional/test_connection_manager.py +++ b/tests/functional/test_connection_manager.py @@ -1,6 +1,7 @@ from unittest import TestCase, mock -from dbt.adapters.contracts.connection import Connection +from dbt.adapters.contracts.connection import Connection, Identifier +from dbt_common.helper_types import Port import psycopg2 from dbt.adapters.postgres import PostgresCredentials, PostgresConnectionManager @@ -20,13 +21,13 @@ def get_connection(self) -> Connection: credentials = PostgresCredentials( host="localhost", user="test-user", - port=1111, + port=Port(1111), password="test-password", database="test-db", schema="test-schema", retries=2, ) - connection = Connection("postgres", None, credentials) + connection = Connection(Identifier("postgres"), None, credentials) return connection def test_open(self): diff --git a/tests/functional/test_dbt_runner.py b/tests/functional/test_dbt_runner.py index c1e05f0f..d3db2d20 100644 --- a/tests/functional/test_dbt_runner.py +++ b/tests/functional/test_dbt_runner.py @@ -13,21 +13,21 @@ def dbt(self) -> dbtRunner: def test_group_invalid_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["--invalid-option"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_command_invalid_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["deps", "--invalid-option"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_command_mutually_exclusive_option(self, dbt: dbtRunner) -> None: res = dbt.invoke(["--warn-error", "--warn-error-options", '{"include": "all"}', "deps"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) res = dbt.invoke(["deps", "--warn-error", "--warn-error-options", '{"include": "all"}']) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_invalid_command(self, dbt: dbtRunner) -> None: res = dbt.invoke(["invalid-command"]) - assert type(res.exception) == DbtUsageException + assert isinstance(res.exception, DbtUsageException) def test_invoke_version(self, dbt: dbtRunner) -> None: dbt.invoke(["--version"]) @@ -57,14 +57,14 @@ def test_invoke_kwargs(self, project, dbt): def test_invoke_kwargs_project_dir(self, project, dbt): res = dbt.invoke(["run"], project_dir="some_random_project_dir") - assert type(res.exception) == DbtProjectError + assert isinstance(res.exception, DbtProjectError) msg = "No dbt_project.yml found at expected path some_random_project_dir" assert msg in res.exception.msg def test_invoke_kwargs_profiles_dir(self, project, dbt): res = dbt.invoke(["run"], profiles_dir="some_random_profiles_dir") - assert type(res.exception) == DbtProjectError + assert isinstance(res.exception, DbtProjectError) msg = "Could not find profile named 'test'" assert msg in res.exception.msg diff --git a/tests/functional/test_init.py b/tests/functional/test_init.py index 1c3133c8..1c8202b7 100644 --- a/tests/functional/test_init.py +++ b/tests/functional/test_init.py @@ -6,7 +6,6 @@ from dbt_common.exceptions import DbtRuntimeError from dbt.tests.util import run_dbt import pytest -import yaml class TestInitProjectWithExistingProfilesYml: diff --git a/tests/unit/test_connection.py b/tests/unit/test_connection.py index 0f46d285..d98c1e5f 100644 --- a/tests/unit/test_connection.py +++ b/tests/unit/test_connection.py @@ -2,7 +2,7 @@ from unittest import TestCase, mock import pytest -from dbt.context.manifest import generate_query_header_context +from dbt.context.query_header import generate_query_header_context from dbt.context.providers import generate_runtime_macro_context from dbt.contracts.files import FileHash from dbt.contracts.graph.manifest import ManifestStateCheck @@ -167,7 +167,7 @@ def test_quoting_on_rename(self): @pytest.mark.skip( """ We moved from __version__ to __about__ when establishing `hatch` as our build tool. - However, `adapters.factory.register_adapter` assumes __version__ when determining + However, `adapters.factory.register_adapter` assumes __version__ when determining the adapter version. This test causes an import error """ ) @@ -183,7 +183,7 @@ def test_debug_connection_fail_nopass(self): @pytest.mark.skip( """ We moved from __version__ to __about__ when establishing `hatch` as our build tool. - However, `adapters.factory.register_adapter` assumes __version__ when determining + However, `adapters.factory.register_adapter` assumes __version__ when determining the adapter version. This test causes an import error """ ) diff --git a/tests/unit/test_renamed_relations.py b/tests/unit/test_renamed_relations.py index 49900d8e..29bbabf2 100644 --- a/tests/unit/test_renamed_relations.py +++ b/tests/unit/test_renamed_relations.py @@ -9,8 +9,10 @@ def test_renameable_relation(): identifier="my_table", type=RelationType.Table, ) - assert relation.renameable_relations == frozenset({ - RelationType.View, - RelationType.Table, - RelationType.MaterializedView, - }) + assert relation.renameable_relations == frozenset( + { + RelationType.View, + RelationType.Table, + RelationType.MaterializedView, + } + )