diff --git a/.github/workflows/coverage.yaml b/.github/workflows/coverage.yaml index aedd423..79b8f0b 100644 --- a/.github/workflows/coverage.yaml +++ b/.github/workflows/coverage.yaml @@ -24,6 +24,7 @@ jobs: run: | poetry config virtualenvs.create true --local poetry config virtualenvs.in-project true --local + poetry lock - uses: actions/cache@v3 name: Define a cache for the virtual environment based on the dependencies lock file with: diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 90cdc39..26fcba4 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -20,6 +20,7 @@ jobs: run: | poetry config virtualenvs.create true --local poetry config virtualenvs.in-project true --local + poetry lock - uses: actions/cache@v3 name: Define a cache for the virtual environment based on the dependencies lock file with: diff --git a/.github/workflows/unittest.yaml b/.github/workflows/unittest.yaml index 1eecd5e..9ecdab4 100644 --- a/.github/workflows/unittest.yaml +++ b/.github/workflows/unittest.yaml @@ -17,7 +17,7 @@ jobs: strategy: matrix: - python-version: [ "3.8", "3.9", "3.10", "3.11", "3.12" ] + python-version: [ "3.8", "3.9", "3.10", "3.11" ] steps: - uses: actions/checkout@v3 diff --git a/focus_validator/config_objects/common.py b/focus_validator/config_objects/common.py index 6b65705..b856256 100644 --- a/focus_validator/config_objects/common.py +++ b/focus_validator/config_objects/common.py @@ -1,7 +1,8 @@ from enum import Enum from typing import List, Literal -from pydantic import BaseModel +import sqlglot +from pydantic import BaseModel, field_validator class AllowNullsCheck(BaseModel): @@ -12,6 +13,22 @@ class ValueInCheck(BaseModel): value_in: List[str] +class SQLQueryCheck(BaseModel): + sql_query: str + + @field_validator("sql_query") + def check_sql_query(cls, sql_query): + returned_columns = [ + column.alias + for column in sqlglot.parse_one(sql_query).find_all(sqlglot.exp.Alias) + ] + + assert returned_columns == [ + "check_output" + ], "SQL query must only return a column called 'check_output'" + return sql_query + + SIMPLE_CHECKS = Literal["check_unique", "column_required"] @@ -20,6 +37,7 @@ class DataTypes(Enum): DECIMAL = "decimal" DATETIME = "datetime" CURRENCY_CODE = "currency-code" + STRINGIFIED_JSON_OBJECT = "stringified-json-object" class DataTypeCheck(BaseModel): diff --git a/focus_validator/config_objects/focus_to_pandera_schema_converter.py b/focus_validator/config_objects/focus_to_pandera_schema_converter.py index 826836c..a27306f 100644 --- a/focus_validator/config_objects/focus_to_pandera_schema_converter.py +++ b/focus_validator/config_objects/focus_to_pandera_schema_converter.py @@ -1,7 +1,10 @@ +import os from itertools import groupby from typing import Dict, List, Optional, Set, Union +import pandas as pd import pandera as pa +import sqlglot from pandera.api.pandas.types import PandasDtypeInputTypes from focus_validator.config_objects import ChecklistObject, InvalidRule, Rule @@ -10,11 +13,25 @@ ChecklistObjectStatus, DataTypeCheck, DataTypes, + SQLQueryCheck, ValueInCheck, ) from focus_validator.config_objects.override import Override from focus_validator.exceptions import FocusNotImplementedError +# group index column adds a column to the dataframe which is used to group the dataframe, otherwise the default +# groupby function does not carry forward all rows in the dataframe causing it to not have row numbers +GROUP_INDEX_COLUMN = "group_index_column" + + +def __groupby_fnc__(df: pd.DataFrame, column_alias: List[str]): + """ + Custom groupby function to be used with pandera check_sql_query, allowing null values + Default groupby function does not allow null values + """ + df[GROUP_INDEX_COLUMN] = range(0, len(df)) + return df.groupby(column_alias + [GROUP_INDEX_COLUMN], dropna=False) + class FocusToPanderaSchemaConverter: @staticmethod @@ -40,6 +57,19 @@ def __generate_pandera_check__(rule: Rule, check_id): return pa.Check.check_value_in( allowed_values=check.value_in, error=error_string ) + elif isinstance(check, SQLQueryCheck): + column_alias = [ + column.alias_or_name + for column in sqlglot.parse_one(check.sql_query).find_all( + sqlglot.exp.Column + ) + ] + return pa.Check.check_sql_query( + sql_query=check.sql_query, + error=error_string, + column_alias=column_alias, + groupby=lambda df: __groupby_fnc__(df=df, column_alias=column_alias), + ) elif isinstance(check, AllowNullsCheck): return pa.Check.check_not_null( error=error_string, ignore_na=False, allow_nulls=check.allow_nulls @@ -77,6 +107,14 @@ def __generate_column_definition__( error=f"{rule.check_id}:::Ensures that column is of {data_type.value} type.", ) ) + elif data_type == DataTypes.STRINGIFIED_JSON_OBJECT: + pandera_type = None + column_checks.append( + pa.Check.check_stringified_json_object_dtype( + ignore_na=True, + error=f"{rule.check_id}:::Ensures that column is of {data_type.value} type.", + ) + ) else: pandera_type = pa.String @@ -151,7 +189,7 @@ def generate_pandera_schema( for rule in rules: if isinstance(rule, InvalidRule): checklist[rule.rule_path] = ChecklistObject( - check_name=rule.rule_path, + check_name=os.path.splitext(os.path.basename(rule.rule_path))[0], column_id="Unknown", error=f"{rule.error_type}: {rule.error}", status=ChecklistObjectStatus.ERRORED, @@ -180,4 +218,7 @@ def generate_pandera_schema( overrides=overrides, schema_dict=schema_dict, ) - return pa.DataFrameSchema(schema_dict, strict=False), checklist + return ( + pa.DataFrameSchema(schema_dict, strict=False), + checklist, + ) diff --git a/focus_validator/config_objects/rule.py b/focus_validator/config_objects/rule.py index ddf71f7..511ca8b 100644 --- a/focus_validator/config_objects/rule.py +++ b/focus_validator/config_objects/rule.py @@ -1,3 +1,4 @@ +import os from typing import Optional, Union import yaml @@ -8,6 +9,7 @@ AllowNullsCheck, ChecklistObjectStatus, DataTypeCheck, + SQLQueryCheck, ValueInCheck, generate_check_friendly_name, ) @@ -27,7 +29,9 @@ class Rule(BaseModel): check_id: str column_id: str - check: Union[SIMPLE_CHECKS, AllowNullsCheck, ValueInCheck, DataTypeCheck] + check: Union[ + SIMPLE_CHECKS, AllowNullsCheck, ValueInCheck, DataTypeCheck, SQLQueryCheck + ] check_friendly_name: Optional[ str @@ -46,6 +50,8 @@ def root_val(cls, values): """ Root validator that checks for all options passed in the config and generate missing options. """ + if values is None: + values = {} check = values.get("check") check_friendly_name = values.get("check_friendly_name") @@ -59,10 +65,10 @@ def root_val(cls, values): check_type_friendly_name = check.__class__.__name__ values["check_type_friendly_name"] = check_type_friendly_name - if check_friendly_name is None and column_id is not None: - values["check_friendly_name"] = generate_check_friendly_name( - check=check, column_id=column_id - ) + if check_friendly_name is None and column_id is not None: + values["check_friendly_name"] = generate_check_friendly_name( + check=check, column_id=column_id + ) return values @@ -70,6 +76,8 @@ def root_val(cls, values): def load_yaml( rule_path, column_namespace: Optional[str] = None ) -> Union["Rule", InvalidRule]: + rule_path_basename = os.path.splitext(os.path.basename(rule_path))[0] + try: with open(rule_path, "r") as f: rule_obj = yaml.safe_load(f) @@ -81,10 +89,15 @@ def load_yaml( ): rule_obj["column"] = f"{column_namespace}:{rule_obj['column']}" + if isinstance(rule_obj, dict) and "check_id" not in rule_obj: + rule_obj["check_id"] = rule_path_basename + return Rule.model_validate(rule_obj) except Exception as e: return InvalidRule( - rule_path=rule_path, error=str(e), error_type=e.__class__.__name__ + rule_path=rule_path_basename, + error=str(e), + error_type=e.__class__.__name__, ) diff --git a/focus_validator/data_loaders/data_loader.py b/focus_validator/data_loaders/data_loader.py index 5232192..b00d0b3 100644 --- a/focus_validator/data_loaders/data_loader.py +++ b/focus_validator/data_loaders/data_loader.py @@ -19,13 +19,13 @@ def __init__(self, data_filename): def find_data_loader(self): file_mime_type = get_file_mime_type(self.data_filename) - if file_mime_type in ["ASCII text", "CSV text"]: + if file_mime_type in ["ASCII text", "CSV text", "CSV ASCII text"]: return CSVDataLoader elif file_mime_type == "Apache Parquet": return ParquetDataLoader else: raise FocusNotImplementedError( - msg=f"Validator for file_type {file_mime_type} not implemented yet." + msg=f"Validator for file_type '{file_mime_type}' not implemented yet." ) def load(self): diff --git a/focus_validator/main.py b/focus_validator/main.py index 0377313..6c1db66 100644 --- a/focus_validator/main.py +++ b/focus_validator/main.py @@ -33,7 +33,7 @@ def main(): help="Allow transitional rules in validation", ) parser.add_argument( - "--validate-version", default="0.5", help="Version of FOCUS to validate against" + "--validate-version", default="1.0", help="Version of FOCUS to validate against" ) parser.add_argument( "--rule-set-path", diff --git a/focus_validator/outputter/outputter_console.py b/focus_validator/outputter/outputter_console.py index 9c6260d..a0b4471 100644 --- a/focus_validator/outputter/outputter_console.py +++ b/focus_validator/outputter/outputter_console.py @@ -1,6 +1,6 @@ import math + import pandas as pd -from tabulate import tabulate from focus_validator.config_objects import Rule from focus_validator.rules.spec_rules import ValidationResult @@ -55,41 +55,51 @@ def __restructure_check_list__(result_set: ValidationResult): def write(self, result_set: ValidationResult): self.result_set = result_set - - checklist = self.__restructure_check_list__(result_set) - print("Checklist:") - print(tabulate(checklist, headers="keys", tablefmt="psql")) - if result_set.failure_cases is not None: - aggregated_failures = result_set.failure_cases.groupby(by=['Check Name', 'Column', 'Description'], as_index=False).aggregate(lambda x: maybe_collapse_range(x.unique().tolist())) + aggregated_failures = result_set.failure_cases.groupby( + by=["Check Name", "Description"], as_index=False + ).aggregate(lambda x: collapse_occurrence_range(x.unique().tolist())) - print("Checks summary:") - print( - tabulate( - tabular_data=aggregated_failures, # type: ignore - headers="keys", - tablefmt="psql", + print("Errors encountered:") + for _, fail in aggregated_failures.iterrows(): + print( + f'{fail["Check Name"]} failed:\n\tDescription: {fail["Description"]}\n\tRows: {fail["Row #"] if fail["Row #"] else "(whole file)"}\n\tExample values: {fail["Values"] if fail["Values"] else "(none)"}\n' ) - ) + print("Validation failed!") + else: + print("Validation succeeded.") -def maybe_collapse_range(l): + +def collapse_occurrence_range(occurrence_range: list): start = None i = None collapsed = [] - for n in sorted(l): + + # Edge case + if len(occurrence_range) == 1: + if isinstance(occurrence_range[0], float) and math.isnan(occurrence_range[0]): + return "" + if occurrence_range[0] is None: + return "" + + for n in sorted(occurrence_range): if not isinstance(n, int) and not (isinstance(n, float) and not math.isnan(n)): - return l + return ",".join([str(x) for x in occurrence_range]) elif i is None: - start = i = n + start = i = int(n) elif n == i + 1: - i = n + i = int(n) elif i: - if i == start: collapsed.append(f'{int(start)}') - else: collapsed.append(f'{int(start)}-{int(i)}') - start = i = n + if i == start: + collapsed.append(f"{start}") + else: + collapsed.append(f"{start}-{i}") + start = i = int(n) if start is not None: - if i == start: collapsed.append(int(start)) - else: collapsed.append(f'{int(start)}-{int(i)}') + if i == start: + collapsed.append(f"{start}") + else: + collapsed.append(f"{start}-{i}") - return collapsed + return ",".join(collapsed) diff --git a/focus_validator/rules/base_rule_definitions/FV-M002-0001.yaml b/focus_validator/rules/base_rule_definitions/AmortizedCost_IsDecimal.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-M002-0001.yaml rename to focus_validator/rules/base_rule_definitions/AmortizedCost_IsDecimal.yaml index b05e342..0d70c5c 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M002-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/AmortizedCost_IsDecimal.yaml @@ -1,4 +1,3 @@ -check_id: FV-M002-0001 column_id: AmortizedCost check: data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/FV-M002-0002.yaml b/focus_validator/rules/base_rule_definitions/AmortizedCost_NotNull.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-M002-0002.yaml rename to focus_validator/rules/base_rule_definitions/AmortizedCost_NotNull.yaml index 274fe38..dc45435 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M002-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/AmortizedCost_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-M002-0002 column_id: AmortizedCost check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-M002-0003.yaml b/focus_validator/rules/base_rule_definitions/AmortizedCost_Required.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-M002-0003.yaml rename to focus_validator/rules/base_rule_definitions/AmortizedCost_Required.yaml index 6eadde2..1225820 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M002-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/AmortizedCost_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-M002-0003 column_id: AmortizedCost check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D014-0001.yaml b/focus_validator/rules/base_rule_definitions/AvailabilityZone_IsString.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D014-0001.yaml rename to focus_validator/rules/base_rule_definitions/AvailabilityZone_IsString.yaml index 571abbe..05a737a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D014-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/AvailabilityZone_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D014-0001 column_id: AvailabilityZone check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D014-0002.yaml b/focus_validator/rules/base_rule_definitions/AvailabilityZone_Nullable.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D014-0002.yaml rename to focus_validator/rules/base_rule_definitions/AvailabilityZone_Nullable.yaml index 5c3522c..326126e 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D014-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/AvailabilityZone_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D014-0002 column_id: AvailabilityZone check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/FV-M001-0001.yaml b/focus_validator/rules/base_rule_definitions/BilledCost_IsDecimal.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-M001-0001.yaml rename to focus_validator/rules/base_rule_definitions/BilledCost_IsDecimal.yaml index 554cac5..ff3039a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M001-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCost_IsDecimal.yaml @@ -1,4 +1,3 @@ -check_id: FV-M001-0001 column_id: BilledCost check: data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/FV-M001-0002.yaml b/focus_validator/rules/base_rule_definitions/BilledCost_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-M001-0002.yaml rename to focus_validator/rules/base_rule_definitions/BilledCost_NotNull.yaml index fcac9d7..d18817f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M001-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCost_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-M001-0002 column_id: BilledCost check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-M001-0003.yaml b/focus_validator/rules/base_rule_definitions/BilledCost_Required.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-M001-0003.yaml rename to focus_validator/rules/base_rule_definitions/BilledCost_Required.yaml index 35a9787..7407c46 100644 --- a/focus_validator/rules/base_rule_definitions/FV-M001-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCost_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-M001-0003 column_id: BilledCost check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D010-0001.yaml b/focus_validator/rules/base_rule_definitions/BilledCurrency_IsCurrencyCode.yaml similarity index 72% rename from focus_validator/rules/base_rule_definitions/FV-D010-0001.yaml rename to focus_validator/rules/base_rule_definitions/BilledCurrency_IsCurrencyCode.yaml index 49dca29..5a26704 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D010-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCurrency_IsCurrencyCode.yaml @@ -1,4 +1,3 @@ -check_id: FV-D010-0001 column_id: BilledCurrency check: data_type: currency-code diff --git a/focus_validator/rules/base_rule_definitions/FV-D010-0002.yaml b/focus_validator/rules/base_rule_definitions/BilledCurrency_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D010-0002.yaml rename to focus_validator/rules/base_rule_definitions/BilledCurrency_NotNull.yaml index ddae15e..1094457 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D010-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCurrency_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D010-0002 column_id: BilledCurrency check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D010-0003.yaml b/focus_validator/rules/base_rule_definitions/BilledCurrency_Required.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D010-0003.yaml rename to focus_validator/rules/base_rule_definitions/BilledCurrency_Required.yaml index 9dcd117..76a1828 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D010-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BilledCurrency_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D010-0003 column_id: BilledCurrency check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D006-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountId_IsString.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D006-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountId_IsString.yaml index d8effe8..ce3124a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D006-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountId_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D006-0001 column_id: BillingAccountId check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D006-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountId_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D006-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountId_NotNull.yaml index f8eebe0..2fbb907 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D006-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountId_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D006-0002 column_id: BillingAccountId check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D006-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountId_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D006-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountId_Required.yaml index d08c2e0..3e61188 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D006-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountId_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D006-0003 column_id: BillingAccountId check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D005-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountName_IsString.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D005-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountName_IsString.yaml index 46e7872..8408869 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D005-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D005-0001 column_id: BillingAccountName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D005-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountName_Nullable.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D005-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountName_Nullable.yaml index ac2c851..9115bbf 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D005-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountName_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D005-0002 column_id: BillingAccountName check: allow_nulls: True diff --git a/focus_validator/rules/base_rule_definitions/FV-D005-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingAccountName_Required.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D005-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingAccountName_Required.yaml index 8d091b3..672716f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D005-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingAccountName_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D005-0003 column_id: BillingAccountName check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D011-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_IsDateTime.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D011-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodEnd_IsDateTime.yaml index d97835b..dbe136a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D011-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D011-0001 column_id: BillingPeriodEnd check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D011-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D011-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodEnd_NotNull.yaml index 1210ee0..f8a50cf 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D011-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D011-0002 column_id: BillingPeriodEnd check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D011-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D011-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodEnd_Required.yaml index 75224a1..b864b76 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D011-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodEnd_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D011-0003 column_id: BillingPeriodEnd check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D012-0001.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_IsDateTime.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D012-0001.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodStart_IsDateTime.yaml index f6d695f..58e567e 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D012-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D012-0001 column_id: BillingPeriodStart check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D012-0002.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_NotNull.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D012-0002.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodStart_NotNull.yaml index b7d93bf..7c9b272 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D012-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D012-0002 column_id: BillingPeriodStart check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D012-0003.yaml b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_Required.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D012-0003.yaml rename to focus_validator/rules/base_rule_definitions/BillingPeriodStart_Required.yaml index 049e524..d2ae0c2 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D012-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/BillingPeriodStart_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D012-0003 column_id: BillingPeriodStart check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml new file mode 100644 index 0000000..920a124 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml new file mode 100644 index 0000000..01dd7c9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml new file mode 100644 index 0000000..f08f045 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeDescription +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml new file mode 100644 index 0000000..021af67 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml @@ -0,0 +1,6 @@ +column_id: ChargeFrequency +check: + value_in: + - "One-Time" + - "Recurring" + - "Usage-Based" \ No newline at end of file diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml new file mode 100644 index 0000000..ea4a54a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeFrequency +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml new file mode 100644 index 0000000..49eaedf --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeFrequency +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml new file mode 100644 index 0000000..97ad0df --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeFrequency +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D017-0001.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_IsDateTime.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D017-0001.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodEnd_IsDateTime.yaml index 78bcef0..2e5f049 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D017-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D017-0001 column_id: ChargePeriodEnd check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D017-0002.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D017-0002.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodEnd_NotNull.yaml index 15b43c4..5157bdd 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D017-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D017-0002 column_id: ChargePeriodEnd check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D017-0003.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D017-0003.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodEnd_Required.yaml index 08e058e..b91ee77 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D017-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodEnd_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D017-0003 column_id: ChargePeriodEnd check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D016-0001.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_IsDateTime.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D016-0001.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodStart_IsDateTime.yaml index 4ea40e2..4b9ad12 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D016-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_IsDateTime.yaml @@ -1,4 +1,3 @@ -check_id: FV-D016-0001 column_id: ChargePeriodStart check: data_type: datetime diff --git a/focus_validator/rules/base_rule_definitions/FV-D016-0002.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_NotNull.yaml similarity index 71% rename from focus_validator/rules/base_rule_definitions/FV-D016-0002.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodStart_NotNull.yaml index bed8364..d40abd4 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D016-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D016-0002 column_id: ChargePeriodStart check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D016-0003.yaml b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_Required.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D016-0003.yaml rename to focus_validator/rules/base_rule_definitions/ChargePeriodStart_Required.yaml index d62ca89..aca0c01 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D016-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargePeriodStart_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D016-0003 column_id: ChargePeriodStart check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml new file mode 100644 index 0000000..6dec960 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml @@ -0,0 +1,13 @@ +column_id: ChargeSubcategory +check: + value_in: + # Allowed when ChargeType=Usage + - "On-Demand" + - "Used Commitment" + - "Unused Commitment" + - "Usage" + # Allowed when ChargeType=Adjustment + - "Refund" + - "Credit" + - "Rounding Error" + - "General Adjustment" diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml new file mode 100644 index 0000000..cd7545e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeSubcategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml new file mode 100644 index 0000000..9ca2483 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeSubcategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml new file mode 100644 index 0000000..1deb197 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeSubcategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0003.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_Enum.yaml similarity index 81% rename from focus_validator/rules/base_rule_definitions/FV-D001-0003.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_Enum.yaml index 825ad29..694456f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_Enum.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0003 column_id: ChargeType check: value_in: diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0001.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D001-0001.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_IsString.yaml index 48277c2..8369e69 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0001 column_id: ChargeType check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0004.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D001-0004.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_NotNull.yaml index 0a37e9e..e3a8e80 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0004.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0004 column_id: ChargeType check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D001-0002.yaml b/focus_validator/rules/base_rule_definitions/ChargeType_Required.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D001-0002.yaml rename to focus_validator/rules/base_rule_definitions/ChargeType_Required.yaml index 794bb8b..de31444 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D001-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ChargeType_Required.yaml @@ -1,3 +1,2 @@ -check_id: FV-D001-0002 column_id: ChargeType check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml new file mode 100644 index 0000000..2ad823d --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml @@ -0,0 +1,5 @@ +column_id: CommitmentDiscountCategory +check: + value_in: + - "Spend" + - "Usage" diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml new file mode 100644 index 0000000..afe06ed --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml new file mode 100644 index 0000000..4b4d889 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml new file mode 100644 index 0000000..49b7f23 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml new file mode 100644 index 0000000..650305b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml new file mode 100644 index 0000000..1b7e48c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml new file mode 100644 index 0000000..212e16b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml new file mode 100644 index 0000000..ef740d6 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountName +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml new file mode 100644 index 0000000..99c567a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountName +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml new file mode 100644 index 0000000..d279fb9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml @@ -0,0 +1,2 @@ +column_id: CommitmentDiscountName +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml new file mode 100644 index 0000000..9daa993 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountType +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml new file mode 100644 index 0000000..2197a0b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountType +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/EffectiveCost_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/EffectiveCost_IsDecimal.yaml new file mode 100644 index 0000000..55adf42 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/EffectiveCost_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: EffectiveCost +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/EffectiveCost_NotNull.yaml b/focus_validator/rules/base_rule_definitions/EffectiveCost_NotNull.yaml new file mode 100644 index 0000000..6120158 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/EffectiveCost_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: EffectiveCost +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/EffectiveCost_Required.yaml b/focus_validator/rules/base_rule_definitions/EffectiveCost_Required.yaml new file mode 100644 index 0000000..a78d71c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/EffectiveCost_Required.yaml @@ -0,0 +1,3 @@ +column_id: EffectiveCost +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D018-0003.yaml b/focus_validator/rules/base_rule_definitions/FV-D018-0003.yaml deleted file mode 100644 index 25ab012..0000000 --- a/focus_validator/rules/base_rule_definitions/FV-D018-0003.yaml +++ /dev/null @@ -1,4 +0,0 @@ -check_id: FV-D018-0003 -column_id: SubAccountName -check: - column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D003-0001.yaml b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_IsString.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D003-0001.yaml rename to focus_validator/rules/base_rule_definitions/InvoiceIssuer_IsString.yaml index 15848b9..a98271b 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D003-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D003-0001 column_id: InvoiceIssuer check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D003-0002.yaml b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_NotNull.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D003-0002.yaml rename to focus_validator/rules/base_rule_definitions/InvoiceIssuer_NotNull.yaml index 5347a76..5e4060a 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D003-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/InvoiceIssuer_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D003-0002 column_id: InvoiceIssuer check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ListCost_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ListCost_IsDecimal.yaml new file mode 100644 index 0000000..02b7f93 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListCost_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ListCost +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ListCost_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ListCost_NotNull.yaml new file mode 100644 index 0000000..3259446 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListCost_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ListCost +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ListCost_Required.yaml b/focus_validator/rules/base_rule_definitions/ListCost_Required.yaml new file mode 100644 index 0000000..f0749c4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListCost_Required.yaml @@ -0,0 +1,3 @@ +column_id: ListCost +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/ListUnitPrice_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/ListUnitPrice_IsDecimal.yaml new file mode 100644 index 0000000..6bbd392 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListUnitPrice_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: ListUnitPrice +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/ListUnitPrice_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Nullable.yaml new file mode 100644 index 0000000..e93fb24 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ListUnitPrice +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ListUnitPrice_Required.yaml b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Required.yaml new file mode 100644 index 0000000..f0beb93 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ListUnitPrice_Required.yaml @@ -0,0 +1,3 @@ +column_id: ListUnitPrice +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml new file mode 100644 index 0000000..2e42fa9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml @@ -0,0 +1,7 @@ +column_id: PricingCategory +check: + value_in: + - "On-Demand" + - "Dynamic" + - "Commitment-Based" + - "Other" diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml new file mode 100644 index 0000000..2065111 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: PricingCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml new file mode 100644 index 0000000..49c257f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingCategory +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml new file mode 100644 index 0000000..4904b04 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: PricingCategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingQuantity_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/PricingQuantity_IsDecimal.yaml new file mode 100644 index 0000000..c2f22d2 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingQuantity_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: PricingQuantity +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/PricingQuantity_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingQuantity_Nullable.yaml new file mode 100644 index 0000000..2cda91c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingQuantity_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingQuantity +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingQuantity_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingQuantity_Required.yaml new file mode 100644 index 0000000..6dfdc20 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingQuantity_Required.yaml @@ -0,0 +1,3 @@ +column_id: PricingQuantity +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml new file mode 100644 index 0000000..f138fa4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: PricingUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml new file mode 100644 index 0000000..34e841a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml new file mode 100644 index 0000000..e2c935e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: PricingUnit +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D004-0001.yaml b/focus_validator/rules/base_rule_definitions/Provider_IsString.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D004-0001.yaml rename to focus_validator/rules/base_rule_definitions/Provider_IsString.yaml index 5b79316..6b5e61f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D004-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/Provider_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D004-0001 column_id: Provider check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D004-0002.yaml b/focus_validator/rules/base_rule_definitions/Provider_NotNull.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D004-0002.yaml rename to focus_validator/rules/base_rule_definitions/Provider_NotNull.yaml index 52e2f9e..dfdea5f 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D004-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/Provider_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D004-0002 column_id: Provider check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D004-0003.yaml b/focus_validator/rules/base_rule_definitions/Provider_Required.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D004-0003.yaml rename to focus_validator/rules/base_rule_definitions/Provider_Required.yaml index 17e18d1..88906a7 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D004-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/Provider_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D004-0003 column_id: Provider check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D007-0001.yaml b/focus_validator/rules/base_rule_definitions/Publisher_IsString.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D007-0001.yaml rename to focus_validator/rules/base_rule_definitions/Publisher_IsString.yaml index 4bba250..8502a38 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D007-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/Publisher_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D007-0001 column_id: Publisher check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D007-0002.yaml b/focus_validator/rules/base_rule_definitions/Publisher_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D007-0002.yaml rename to focus_validator/rules/base_rule_definitions/Publisher_NotNull.yaml index 4f82b78..122fa87 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D007-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/Publisher_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D007-0002 column_id: Publisher check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D007-0003.yaml b/focus_validator/rules/base_rule_definitions/Publisher_Required.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D007-0003.yaml rename to focus_validator/rules/base_rule_definitions/Publisher_Required.yaml index 94b52d4..9d4dc1e 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D007-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/Publisher_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D007-0003 column_id: Publisher check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D013-0001.yaml b/focus_validator/rules/base_rule_definitions/Region_IsString.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D013-0001.yaml rename to focus_validator/rules/base_rule_definitions/Region_IsString.yaml index fc77602..8733163 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D013-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/Region_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D013-0001 column_id: Region check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D013-0002.yaml b/focus_validator/rules/base_rule_definitions/Region_NotNull.yaml similarity index 66% rename from focus_validator/rules/base_rule_definitions/FV-D013-0002.yaml rename to focus_validator/rules/base_rule_definitions/Region_NotNull.yaml index 19ba00c..fcc15a5 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D013-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/Region_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D013-0002 column_id: Region check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D013-0003.yaml b/focus_validator/rules/base_rule_definitions/Region_Required.yaml similarity index 65% rename from focus_validator/rules/base_rule_definitions/FV-D013-0003.yaml rename to focus_validator/rules/base_rule_definitions/Region_Required.yaml index e27245d..60968e1 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D013-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/Region_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D013-0003 column_id: Region check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D002-0001.yaml b/focus_validator/rules/base_rule_definitions/ResourceID_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D002-0001.yaml rename to focus_validator/rules/base_rule_definitions/ResourceID_IsString.yaml index fa8f571..42913bb 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D002-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceID_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D002-0001 column_id: ResourceID check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D002-0002.yaml b/focus_validator/rules/base_rule_definitions/ResourceID_Nullable.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D002-0002.yaml rename to focus_validator/rules/base_rule_definitions/ResourceID_Nullable.yaml index 862411d..f855723 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D002-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceID_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D002-0002 column_id: ResourceID check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/FV-D008-0001.yaml b/focus_validator/rules/base_rule_definitions/ResourceName_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D008-0001.yaml rename to focus_validator/rules/base_rule_definitions/ResourceName_IsString.yaml index 6aafb9c..06bc202 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D008-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D008-0001 column_id: ResourceName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D008-0002.yaml b/focus_validator/rules/base_rule_definitions/ResourceName_Nullable.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D008-0002.yaml rename to focus_validator/rules/base_rule_definitions/ResourceName_Nullable.yaml index fd9878e..bcd02f8 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D008-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceName_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D008-0002 column_id: ResourceName check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/FV-D008-0003.yaml b/focus_validator/rules/base_rule_definitions/ResourceName_Required.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D008-0003.yaml rename to focus_validator/rules/base_rule_definitions/ResourceName_Required.yaml index 1821b6b..b713ea3 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D008-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ResourceName_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D008-0003 column_id: ResourceName check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml new file mode 100644 index 0000000..68a5230 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ResourceType +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml new file mode 100644 index 0000000..99cf430 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ResourceType +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml new file mode 100644 index 0000000..5df814f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml @@ -0,0 +1,2 @@ +column_id: ResourceType +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0004.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_Enum.yaml similarity index 95% rename from focus_validator/rules/base_rule_definitions/FV-D015-0004.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_Enum.yaml index fb68aae..85f90f3 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0004.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_Enum.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0004 column_id: ServiceCategory check_friendly_name: "ServiceCategory must have a value defined in spec." check: diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0001.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_IsString.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D015-0001.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_IsString.yaml index 629223b..f359331 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0001 column_id: ServiceCategory check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0002.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_NotNull.yaml similarity index 70% rename from focus_validator/rules/base_rule_definitions/FV-D015-0002.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_NotNull.yaml index a5f600b..e2825f6 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0002 column_id: ServiceCategory check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D015-0003.yaml b/focus_validator/rules/base_rule_definitions/ServiceCategory_Required.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D015-0003.yaml rename to focus_validator/rules/base_rule_definitions/ServiceCategory_Required.yaml index 7290024..ea364b3 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D015-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceCategory_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D015-0003 column_id: ServiceCategory check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D009-0001.yaml b/focus_validator/rules/base_rule_definitions/ServiceName_IsString.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D009-0001.yaml rename to focus_validator/rules/base_rule_definitions/ServiceName_IsString.yaml index b5956f1..7429b14 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D009-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D009-0001 column_id: ServiceName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D009-0002.yaml b/focus_validator/rules/base_rule_definitions/ServiceName_NotNull.yaml similarity index 68% rename from focus_validator/rules/base_rule_definitions/FV-D009-0002.yaml rename to focus_validator/rules/base_rule_definitions/ServiceName_NotNull.yaml index 8d0b787..2c62718 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D009-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceName_NotNull.yaml @@ -1,4 +1,3 @@ -check_id: FV-D009-0002 column_id: ServiceName check: allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/FV-D009-0003.yaml b/focus_validator/rules/base_rule_definitions/ServiceName_Required.yaml similarity index 67% rename from focus_validator/rules/base_rule_definitions/FV-D009-0003.yaml rename to focus_validator/rules/base_rule_definitions/ServiceName_Required.yaml index fdede46..5c07502 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D009-0003.yaml +++ b/focus_validator/rules/base_rule_definitions/ServiceName_Required.yaml @@ -1,4 +1,3 @@ -check_id: FV-D009-0003 column_id: ServiceName check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml new file mode 100644 index 0000000..4b43197 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SkuId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml new file mode 100644 index 0000000..d8656e9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SkuId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml b/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml new file mode 100644 index 0000000..3fdb39d --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SkuId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml new file mode 100644 index 0000000..76460dd --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SkuPriceId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml new file mode 100644 index 0000000..76d2fc1 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml @@ -0,0 +1,9 @@ +column_id: SkuPriceId +check_friendly_name: SkuPriceId must be set for certain values of ChargeType +check: + sql_query: | + SELECT CASE + WHEN ChargeType IN ('Purchase', 'Usage', 'Refund') AND SkuPriceId IS NULL THEN FALSE + ELSE TRUE + END AS check_output + FROM df; diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml new file mode 100644 index 0000000..5352d19 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SkuPriceId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml new file mode 100644 index 0000000..aa49628 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml new file mode 100644 index 0000000..03f41eb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml new file mode 100644 index 0000000..d492ff7 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SubAccountId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/FV-D018-0001.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_IsString.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D018-0001.yaml rename to focus_validator/rules/base_rule_definitions/SubAccountName_IsString.yaml index 904895a..5847d66 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D018-0001.yaml +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_IsString.yaml @@ -1,4 +1,3 @@ -check_id: FV-D018-0001 column_id: SubAccountName check: data_type: string diff --git a/focus_validator/rules/base_rule_definitions/FV-D018-0002.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_Nullable.yaml similarity index 69% rename from focus_validator/rules/base_rule_definitions/FV-D018-0002.yaml rename to focus_validator/rules/base_rule_definitions/SubAccountName_Nullable.yaml index 8520687..d861749 100644 --- a/focus_validator/rules/base_rule_definitions/FV-D018-0002.yaml +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_Nullable.yaml @@ -1,4 +1,3 @@ -check_id: FV-D018-0002 column_id: SubAccountName check: allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml new file mode 100644 index 0000000..635ab28 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml @@ -0,0 +1,2 @@ +column_id: SubAccountName +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml b/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml new file mode 100644 index 0000000..2832cea --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + data_type: stringified-json-object diff --git a/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml b/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml new file mode 100644 index 0000000..b06ee4c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/Tags_Required.yaml b/focus_validator/rules/base_rule_definitions/Tags_Required.yaml new file mode 100644 index 0000000..54ae7c2 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_Required.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml new file mode 100644 index 0000000..e7071b4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml new file mode 100644 index 0000000..05d4bfb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml new file mode 100644 index 0000000..3a21150 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml new file mode 100644 index 0000000..89fa9e0 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: UsageUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml new file mode 100644 index 0000000..b5fb4b5 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: UsageUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml new file mode 100644 index 0000000..a23616c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: UsageUnit +check: column_required diff --git a/focus_validator/rules/checks.py b/focus_validator/rules/checks.py index 27d7bd4..58a9497 100644 --- a/focus_validator/rules/checks.py +++ b/focus_validator/rules/checks.py @@ -1,9 +1,13 @@ +import json from datetime import datetime from typing import Union import numpy as np import pandas as pd +import pandasql +import pandera as pa from pandera import extensions +from pandera.errors import SchemaError from focus_validator.utils.download_currency_codes import get_currency_codes @@ -35,6 +39,48 @@ def check_value_in(pandas_obj: pd.Series, allowed_values): return pandas_obj.isin(allowed_values) +@extensions.register_check_method(check_type="groupby") +def check_sql_query(df_groups, sql_query, column_alias): + grouped_elements = [] + for values in list(df_groups): + row_obj = {} + for column_name, value in zip(column_alias + ["index"], values): + row_obj[column_name] = value + grouped_elements.append(row_obj) + + df = pd.DataFrame(grouped_elements) + check_output = pandasql.sqldf(sql_query, locals())["check_output"] + + # Getting the index of rows where the series values are False + false_indexes = [i for i, val in enumerate(check_output) if not val] + if false_indexes: + # Extracting those rows from the dataframe + extracted_rows = df.loc[false_indexes].to_dict("records")[:3] + false_indexes = [i for i, val in enumerate(check_output) if not val] + + # for the given indexes in false_indexes list, we are extracting the rows from the dataframe and + # add column_alias value to failure_case column and index to index column + failure_cases = df[df.index.isin(false_indexes)] + failure_cases["failure_case"] = df.apply( + lambda row: {column: row[column] for column in column_alias}, axis=1 + ) + failure_cases["failure_case"] = df.apply( + lambda row: ",".join( + [f"{column}:{row[column]}" for column in column_alias] + ), + axis=1, + ) + + raise SchemaError( + schema=pa.DataFrameSchema(), + data=None, + message="", + failure_cases=failure_cases, + ) + + return True + + @extensions.register_check_method() def check_datetime_dtype(pandas_obj: pd.Series): def __validate_date_obj__(value: Union[str, datetime]): @@ -63,3 +109,15 @@ def check_currency_code_dtype(pandas_obj: pd.Series): return pd.Series( map(lambda v: isinstance(v, str) and v in currency_codes, pandas_obj.values) ) + + +@extensions.register_check_method() +def check_stringified_json_object_dtype(pandas_obj: pd.Series): + def __validate_stringified_json_object__(value: str): + try: + parsed = json.loads(value) + return isinstance(parsed, dict) + except Exception: + return False + + return pd.Series(map(__validate_stringified_json_object__, pandas_obj.values)) diff --git a/focus_validator/rules/version_sets.yaml b/focus_validator/rules/version_sets.yaml index 86a15d6..b2163a4 100644 --- a/focus_validator/rules/version_sets.yaml +++ b/focus_validator/rules/version_sets.yaml @@ -1,60 +1,188 @@ +'1.0': +- ChargeDescription_IsString.yaml +- ChargeDescription_Required.yaml +- ChargeDescription_NotNull.yaml +- ChargeFrequency_Enum.yaml +- ChargeFrequency_IsString.yaml +- ChargeFrequency_NotNull.yaml +- ChargeFrequency_Required.yaml +- ChargeSubcategory_Enum.yaml +- ChargeSubcategory_IsString.yaml +- ChargeSubcategory_NotNull.yaml +- ChargeSubcategory_Required.yaml +- CommitmentDiscountCategory_Enum.yaml +- CommitmentDiscountCategory_IsString.yaml +- CommitmentDiscountCategory_NotNull.yaml +- CommitmentDiscountCategory_Required.yaml +- CommitmentDiscountId_IsString.yaml +- CommitmentDiscountId_Nullable.yaml +- CommitmentDiscountId_Required.yaml +- CommitmentDiscountName_IsString.yaml +- CommitmentDiscountName_Nullable.yaml +- CommitmentDiscountName_Required.yaml +- CommitmentDiscountType_IsString.yaml +- CommitmentDiscountType_Nullable.yaml +- EffectiveCost_IsDecimal.yaml +- EffectiveCost_NotNull.yaml +- EffectiveCost_Required.yaml +- ListCost_IsDecimal.yaml +- ListCost_NotNull.yaml +- ListCost_Required.yaml +- ListUnitPrice_IsDecimal.yaml +- ListUnitPrice_Nullable.yaml +- ListUnitPrice_Required.yaml +- PricingCategory_Enum.yaml +- PricingCategory_IsString.yaml +- PricingCategory_Nullable.yaml +- PricingCategory_Required.yaml +- PricingQuantity_IsDecimal.yaml +- PricingQuantity_Nullable.yaml +- PricingQuantity_Required.yaml +- PricingUnit_IsString.yaml +- PricingUnit_Nullable.yaml +- PricingUnit_Required.yaml +- ResourceType_IsString.yaml +- ResourceType_Nullable.yaml +- ResourceType_Required.yaml +- SkuId_IsString.yaml +- SkuId_Nullable.yaml +- SkuId_Required.yaml +- SkuPriceId_IsString.yaml +- SkuPriceId_Nullable.yaml +- SkuPriceId_Required.yaml +- Tags_IsJSONObject.yaml +- Tags_Nullable.yaml +- Tags_Required.yaml +- UsageQuantity_IsDecimal.yaml +- UsageQuantity_Nullable.yaml +- UsageQuantity_Required.yaml +- UsageUnit_IsString.yaml +- UsageUnit_Nullable.yaml +- UsageUnit_Required.yaml + +### Removed 0.5 content is left here in commented form for reference +# - AmortizedCost_IsDecimal.yaml +# - AmortizedCost_NotNull.yaml +# - AmortizedCost_Required.yaml + +### Unmodified 0.5 content follows +- AvailabilityZone_IsString.yaml +- AvailabilityZone_Nullable.yaml +- BilledCost_IsDecimal.yaml +- BilledCost_NotNull.yaml +- BilledCost_Required.yaml +- BilledCurrency_IsCurrencyCode.yaml +- BilledCurrency_NotNull.yaml +- BilledCurrency_Required.yaml +- BillingAccountId_IsString.yaml +- BillingAccountId_NotNull.yaml +- BillingAccountId_Required.yaml +- BillingAccountName_IsString.yaml +- BillingAccountName_Nullable.yaml +- BillingAccountName_Required.yaml +- BillingPeriodEnd_IsDateTime.yaml +- BillingPeriodEnd_NotNull.yaml +- BillingPeriodEnd_Required.yaml +- BillingPeriodStart_IsDateTime.yaml +- BillingPeriodStart_NotNull.yaml +- BillingPeriodStart_Required.yaml +- ChargePeriodEnd_IsDateTime.yaml +- ChargePeriodEnd_NotNull.yaml +- ChargePeriodEnd_Required.yaml +- ChargePeriodStart_IsDateTime.yaml +- ChargePeriodStart_NotNull.yaml +- ChargePeriodStart_Required.yaml +- ChargeType_Enum.yaml +- ChargeType_IsString.yaml +- ChargeType_NotNull.yaml +- ChargeType_Required.yaml +- InvoiceIssuer_IsString.yaml +- InvoiceIssuer_NotNull.yaml +- Provider_IsString.yaml +- Provider_NotNull.yaml +- Provider_Required.yaml +- Publisher_IsString.yaml +- Publisher_NotNull.yaml +- Publisher_Required.yaml +- Region_IsString.yaml +- Region_NotNull.yaml +- Region_Required.yaml +- ResourceID_IsString.yaml +- ResourceID_Nullable.yaml +- ResourceName_IsString.yaml +- ResourceName_Nullable.yaml +- ResourceName_Required.yaml +- ServiceCategory_Enum.yaml +- ServiceCategory_IsString.yaml +- ServiceCategory_NotNull.yaml +- ServiceCategory_Required.yaml +- ServiceName_IsString.yaml +- ServiceName_NotNull.yaml +- ServiceName_Required.yaml +- SubAccountName_IsString.yaml +- SubAccountName_Nullable.yaml +- SubAccountName_Required.yaml + '0.5': -- FV-D001-0001.yaml -- FV-D001-0002.yaml -- FV-D001-0003.yaml -- FV-D001-0004.yaml -- FV-D002-0001.yaml -- FV-D002-0002.yaml -- FV-D003-0001.yaml -- FV-D003-0002.yaml -- FV-D004-0001.yaml -- FV-D004-0002.yaml -- FV-D004-0003.yaml -- FV-D005-0001.yaml -- FV-D005-0002.yaml -- FV-D005-0003.yaml -- FV-D006-0001.yaml -- FV-D006-0002.yaml -- FV-D006-0003.yaml -- FV-D007-0001.yaml -- FV-D007-0002.yaml -- FV-D007-0003.yaml -- FV-D008-0001.yaml -- FV-D008-0002.yaml -- FV-D008-0003.yaml -- FV-D009-0001.yaml -- FV-D009-0002.yaml -- FV-D009-0003.yaml -- FV-D010-0001.yaml -- FV-D010-0002.yaml -- FV-D010-0003.yaml -- FV-D011-0001.yaml -- FV-D011-0002.yaml -- FV-D011-0003.yaml -- FV-D012-0001.yaml -- FV-D012-0002.yaml -- FV-D012-0003.yaml -- FV-D013-0001.yaml -- FV-D013-0002.yaml -- FV-D013-0003.yaml -- FV-D014-0001.yaml -- FV-D014-0002.yaml -- FV-D015-0001.yaml -- FV-D015-0002.yaml -- FV-D015-0003.yaml -- FV-D015-0004.yaml -- FV-D016-0001.yaml -- FV-D016-0002.yaml -- FV-D016-0003.yaml -- FV-D017-0001.yaml -- FV-D017-0002.yaml -- FV-D017-0003.yaml -- FV-D018-0001.yaml -- FV-D018-0002.yaml -- FV-D018-0003.yaml -- FV-M001-0001.yaml -- FV-M001-0002.yaml -- FV-M001-0003.yaml -- FV-M002-0001.yaml -- FV-M002-0002.yaml -- FV-M002-0003.yaml +- AmortizedCost_IsDecimal.yaml +- AmortizedCost_NotNull.yaml +- AmortizedCost_Required.yaml +- AvailabilityZone_IsString.yaml +- AvailabilityZone_Nullable.yaml +- BilledCost_IsDecimal.yaml +- BilledCost_NotNull.yaml +- BilledCost_Required.yaml +- BilledCurrency_IsCurrencyCode.yaml +- BilledCurrency_NotNull.yaml +- BilledCurrency_Required.yaml +- BillingAccountId_IsString.yaml +- BillingAccountId_NotNull.yaml +- BillingAccountId_Required.yaml +- BillingAccountName_IsString.yaml +- BillingAccountName_Nullable.yaml +- BillingAccountName_Required.yaml +- BillingPeriodEnd_IsDateTime.yaml +- BillingPeriodEnd_NotNull.yaml +- BillingPeriodEnd_Required.yaml +- BillingPeriodStart_IsDateTime.yaml +- BillingPeriodStart_NotNull.yaml +- BillingPeriodStart_Required.yaml +- ChargePeriodEnd_IsDateTime.yaml +- ChargePeriodEnd_NotNull.yaml +- ChargePeriodEnd_Required.yaml +- ChargePeriodStart_IsDateTime.yaml +- ChargePeriodStart_NotNull.yaml +- ChargePeriodStart_Required.yaml +- ChargeType_Enum.yaml +- ChargeType_IsString.yaml +- ChargeType_NotNull.yaml +- ChargeType_Required.yaml +- InvoiceIssuer_IsString.yaml +- InvoiceIssuer_NotNull.yaml +- Provider_IsString.yaml +- Provider_NotNull.yaml +- Provider_Required.yaml +- Publisher_IsString.yaml +- Publisher_NotNull.yaml +- Publisher_Required.yaml +- Region_IsString.yaml +- Region_NotNull.yaml +- Region_Required.yaml +- ResourceID_IsString.yaml +- ResourceID_Nullable.yaml +- ResourceName_IsString.yaml +- ResourceName_Nullable.yaml +- ResourceName_Required.yaml +- ServiceCategory_Enum.yaml +- ServiceCategory_IsString.yaml +- ServiceCategory_NotNull.yaml +- ServiceCategory_Required.yaml +- ServiceName_IsString.yaml +- ServiceName_NotNull.yaml +- ServiceName_Required.yaml +- SubAccountId_IsString.yaml +- SubAccountId_Nullable.yaml +- SubAccountId_Required.yaml +- SubAccountName_IsString.yaml +- SubAccountName_Nullable.yaml +- SubAccountName_Required.yaml diff --git a/focus_validator/utils/download_currency_codes.py b/focus_validator/utils/download_currency_codes.py index a748556..53e0d08 100644 --- a/focus_validator/utils/download_currency_codes.py +++ b/focus_validator/utils/download_currency_codes.py @@ -7,7 +7,7 @@ CURRENCY_CODE_CSV_PATH = "focus_validator/utils/currency_codes.csv" -def download_currency_codes(): +def download_currency_codes(): # pragma: no cover r = requests.get(DATAHUB_URL) root = ET.fromstring(r.content.decode()) @@ -25,5 +25,5 @@ def get_currency_codes(): return set(df["currency_codes"].values) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover download_currency_codes() diff --git a/pyproject.toml b/pyproject.toml index e56dc06..13956c5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -6,7 +6,8 @@ authors = [] readme = "README.md" packages = [{ include = "focus_validator" }] include = [ - { path = "focus_validator/rules/version_sets/0.5/*" } + { path = "focus_validator/rules/version_sets/0.5/*" }, + { path = "focus_validator/rules/version_sets/1.0/*" } ] # TODO: For some reason, this doesn't exclude anything exclude = [ @@ -27,6 +28,7 @@ python-magic = "*" pyyaml = "*" requests = "*" pandera = { version = "^0.17.2" } +sqlglot = "^18.7.0" # for Python 3.12, force higher version of numpy numpy = [ @@ -34,6 +36,7 @@ numpy = [ { version = "~1.26", python = "~3.12" } ] pytz = "^2023.3.post1" +pandasql = "^0.7.3" [tool.poetry.group.dev.dependencies] black = { extras = ["d"], version = "^23.7.0" } diff --git a/setup.cfg b/setup.cfg index 9d6fa07..146ca93 100644 --- a/setup.cfg +++ b/setup.cfg @@ -13,3 +13,9 @@ line_length=88 [mypy] plugins = pandera.mypy + +[coverage:run] +omit = focus_validator/utils/*.py + +[mypy-pandasql.*] +ignore_missing_imports = True diff --git a/tests/attributes/test_attribute_json.py b/tests/attributes/test_attribute_json.py new file mode 100644 index 0000000..e1ed79b --- /dev/null +++ b/tests/attributes/test_attribute_json.py @@ -0,0 +1,78 @@ +from unittest import TestCase +from uuid import uuid4 + +import pandas as pd +from pandera.errors import SchemaErrors + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import ( + ChecklistObjectStatus, + DataTypeCheck, + DataTypes, +) +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + + +# noinspection DuplicatedCode +class TestAttributeJSONObject(TestCase): + def __eval_function__(self, sample_value, should_fail): + random_column_id = str(uuid4()) + random_check_id = str(uuid4()) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=[ + Rule( + check_id=random_check_id, + column_id=random_column_id, + check=DataTypeCheck(data_type=DataTypes.STRINGIFIED_JSON_OBJECT), + ) + ] + ) + + sample_data = pd.DataFrame([{random_column_id: sample_value}]) + + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases = e.failure_cases + + validation_result = ValidationResult( + failure_cases=failure_cases, checklist=checklist + ) + validation_result.process_result() + + if should_fail: + self.assertIsNotNone(validation_result.failure_cases) + records = validation_result.failure_cases.to_dict(orient="records") + self.assertEqual(len(records), 1) + collected_values = [record["Values"] for record in records] + self.assertEqual(collected_values, [sample_value]) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.FAILED, + ) + else: + self.assertIsNone(validation_result.failure_cases) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.PASSED, + ) + + def test_valid_json(self): + self.__eval_function__('{"my-cool-tag": "focus", "my-cool-tag-2": "whahoo"}', False) + + def test_valid_json_empty(self): + self.__eval_function__('{}', False) + + def test_valid_json_bad_data_type(self): + self.__eval_function__(0, True) + + def test_valid_json_null_value(self): + self.__eval_function__(None, False) + + def test_valid_json_empty_string(self): + self.__eval_function__("", True) diff --git a/tests/checks/test_sql_query_check.py b/tests/checks/test_sql_query_check.py new file mode 100644 index 0000000..48a6339 --- /dev/null +++ b/tests/checks/test_sql_query_check.py @@ -0,0 +1,142 @@ +import json +from unittest import TestCase + +import pandas as pd +from pandera.errors import SchemaErrors +from pydantic import ValidationError + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import ( + SQLQueryCheck, + DataTypes, + DataTypeCheck, +) +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + + +# noinspection SqlNoDataSourceInspection,SqlDialectInspection +class TestSQLQueryCheck(TestCase): + @staticmethod + def __generate_sample_rule_type_string__(allow_nulls: bool, data_type: DataTypes): + return [ + Rule( + check_id="sql_check_for_multiple_columns", + column_id="test_dimension", + check=SQLQueryCheck( + sql_query=""" + SELECT + test_dimension, + CASE WHEN test_dimension = 'some-value' THEN true ELSE false END AS check_output + FROM df; + """ + ), + ), + Rule( + check_id="test_dimension", + column_id="test_dimension", + check=DataTypeCheck(data_type=data_type), + ), + ] + + @staticmethod + def __validate_helper__(schema, checklist, sample_data): + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases: pd.DataFrame = e.failure_cases + + validation_result = ValidationResult( + checklist=checklist, failure_cases=failure_cases + ) + validation_result.process_result() + return validation_result + + def test_sql_check_for_multiple_columns(self): + test_sql_query = "SELECT * FROM table" + + with self.assertRaises(ValidationError) as cm: + SQLQueryCheck(sql_query=test_sql_query) + self.assertIn( + "Assertion failed, SQL query must only return a column called 'check_output'", + str(cm.exception), + ) + + def test_sql_check_with_invalid_sql(self): + """ + Check for sql query that do not return column called check + """ + + # noinspection SqlDialectInspection,SqlNoDataSourceInspection + test_sql_query = """SELECT + product_id, + (CASE + WHEN product_id = 'a' THEN TRUE + ELSE FALSE + END) AS check_output + FROM Products;""" + + # this query should be valid + SQLQueryCheck(sql_query=test_sql_query) + + def test_null_value_allowed_valid_case(self): + rules = self.__generate_sample_rule_type_string__( + allow_nulls=True, data_type=DataTypes.STRING + ) + sample_data = pd.DataFrame( + [ + {"test_dimension": "NULL", "column_2": "some-value"}, + {"test_dimension": "some-value", "column_2": "some-value"}, + {"test_dimension": "some-value", "column_2": "some-value"}, + {"test_dimension": "NULL", "column_2": "some-value"}, + ] + ) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=rules, override_config=None + ) + validation_result = self.__validate_helper__( + schema=schema, checklist=checklist, sample_data=sample_data + ) + + failure_cases_dict = validation_result.failure_cases.to_dict(orient="records") + + self.assertEqual(len(failure_cases_dict), 2) + self.assertEqual( + failure_cases_dict, + [ + { + "Column": "test_dimension", + "Check Name": "sql_check_for_multiple_columns", + "Description": " None", + "Values": "test_dimension:NULL,test_dimension:NULL", + "Row #": 1, + }, + { + "Column": "test_dimension", + "Check Name": "sql_check_for_multiple_columns", + "Description": " None", + "Values": "test_dimension:NULL,test_dimension:NULL", + "Row #": 4, + }, + ], + ) + + def test_pass_case(self): + rules = self.__generate_sample_rule_type_string__( + allow_nulls=True, data_type=DataTypes.STRING + ) + sample_data = pd.DataFrame( + [{"test_dimension": "some-value"}, {"test_dimension": "some-value"}] + ) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=rules, override_config=None + ) + validation_result = self.__validate_helper__( + schema=schema, checklist=checklist, sample_data=sample_data + ) + self.assertIsNone(validation_result.failure_cases) diff --git a/tests/checks/test_sql_query_check_from_config.py b/tests/checks/test_sql_query_check_from_config.py new file mode 100644 index 0000000..19dc3d2 --- /dev/null +++ b/tests/checks/test_sql_query_check_from_config.py @@ -0,0 +1,115 @@ +import tempfile +from unittest import TestCase + +import pandas as pd +from pandera.errors import SchemaErrors + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import DataTypeCheck, DataTypes +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + +YAML_CONFIG = """ +check_id: SkuPriceId +column_id: SkuPriceId +check_friendly_name: SkuPriceId must be set for certain values of ChargeType +check: + sql_query: | + SELECT CASE + WHEN ChargeType IN ('Purchase', 'Usage', 'Refund') AND SkuPriceId IS NULL THEN FALSE + ELSE TRUE + END AS check_output + FROM df; +""" + + +class TestSQLQueryCheckConfig(TestCase): + def test_config_from_yaml(self): + with tempfile.NamedTemporaryFile() as f: + f.write(YAML_CONFIG.encode()) + f.seek(0) + rule = Rule.load_yaml(f.name) + + dimension_checks = [ + Rule( + check_id="SkuPriceId", + column_id="SkuPriceId", + check=DataTypeCheck(data_type=DataTypes.STRING), + ), + Rule( + check_id="test_dimension", + column_id="test_dimension", + check=DataTypeCheck(data_type=DataTypes.STRING), + ), + Rule( + check_id="ChargeType", + column_id="ChargeType", + check=DataTypeCheck(data_type=DataTypes.STRING), + ), + ] + + sample_data = pd.DataFrame( + [ + { + "test_dimension": "some-value", + "SkuPriceId": "some-value", + "ChargeType": "Purchase", + }, + { + "test_dimension": "some-value", + "SkuPriceId": None, + "ChargeType": "Purchase", + }, + { + "test_dimension": "some-value", + "SkuPriceId": "random-value", + "ChargeType": "Purchase", + }, + { + "test_dimension": "some-value", + "SkuPriceId": None, + "ChargeType": "Purchase", + }, + ] + ) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=dimension_checks + [rule], override_config=None + ) + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases = e.failure_cases + + validation_result = ValidationResult( + checklist=checklist, failure_cases=failure_cases + ) + validation_result.process_result() + + failure_cases_dict = validation_result.failure_cases.to_dict(orient="records") + self.assertEqual(len(failure_cases_dict), 2) + + # row # does not match nan, need to fix thsi + # failure_cases_dict[0] + self.assertEqual( + failure_cases_dict, + [ + { + "Column": "SkuPriceId", + "Check Name": "SkuPriceId", + "Description": " SkuPriceId must be set for certain values of ChargeType", + "Values": "ChargeType:Purchase,SkuPriceId:nan", + "Row #": 2, + }, + { + "Column": "SkuPriceId", + "Check Name": "SkuPriceId", + "Description": " SkuPriceId must be set for certain values of ChargeType", + "Values": "ChargeType:Purchase,SkuPriceId:nan", + "Row #": 4, + }, + ], + ) diff --git a/tests/config_objects/test_check_friendly_name.py b/tests/config_objects/test_check_friendly_name.py index 1208753..c526b67 100644 --- a/tests/config_objects/test_check_friendly_name.py +++ b/tests/config_objects/test_check_friendly_name.py @@ -2,12 +2,14 @@ from uuid import uuid4 from polyfactory.factories.pydantic_factory import ModelFactory +from pydantic import ValidationError from focus_validator.config_objects import Rule from focus_validator.config_objects.common import ( AllowNullsCheck, DataTypeCheck, ValueInCheck, + SQLQueryCheck, ) @@ -20,7 +22,15 @@ def test_default_friendly_name_is_generated(self): ) for _ in range(1000): # there is no way to generate all values for a field type - random_model = model_factory.build() + try: + random_model = model_factory.build() + except ValidationError as e: + if "SQLQueryCheck" in str(e): + # SQLQueryCheck is not supported by ModelFactory + continue + else: + raise e + if random_model.check == "column_required": self.assertEqual( random_model.check_friendly_name, @@ -53,6 +63,8 @@ def test_default_friendly_name_is_generated(self): random_model.check_friendly_name, f"{random_column_name} must have a value from the list: {options}.", ) + elif isinstance(random_model.check, SQLQueryCheck): + pass else: raise NotImplementedError( f"check_type: {random_model.check} not implemented" diff --git a/tests/config_objects/test_check_type_friendly_name.py b/tests/config_objects/test_check_type_friendly_name.py index 61e6a53..a5fb610 100644 --- a/tests/config_objects/test_check_type_friendly_name.py +++ b/tests/config_objects/test_check_type_friendly_name.py @@ -18,7 +18,15 @@ def test_generate_name_for_check_types(self): model_factory = ModelFactory.create_factory(model=Rule) for _ in range(1000): # there is no way to generate all values for a field type - random_model = model_factory.build() + try: + random_model = model_factory.build() + except ValidationError as e: + if "SQLQueryCheck" in str(e): + # SQLQueryCheck is not supported by ModelFactory + continue + else: + raise e + self.assertIn( random_model.check_type_friendly_name, [ @@ -51,7 +59,15 @@ def test_data_type_config(self): def test_check_type_config_deny_update(self): model_factory = ModelFactory.create_factory(model=Rule) - sample_data_type = model_factory.build() + try: + sample_data_type = model_factory.build() + except ValidationError as e: + if "SQLQueryCheck" in str(e): + # SQLQueryCheck is not supported by ModelFactory + return + else: + raise e + with self.assertRaises(ValidationError) as cm: sample_data_type.check_type_friendly_name = "new_value" self.assertIn( @@ -69,6 +85,6 @@ def test_assign_bad_type(self): ) self.assertEqual(len(cm.exception.errors()), 1) self.assertIn( - "Input should be 'string', 'decimal', 'datetime' or 'currency-code'", + "Input should be 'string', 'decimal', 'datetime', 'currency-code' or 'stringified-json-object'", str(cm.exception), ) diff --git a/tests/config_objects/test_load_bad_rule_config_file.py b/tests/config_objects/test_load_bad_rule_config_file.py index 66b5f19..a8c10d3 100644 --- a/tests/config_objects/test_load_bad_rule_config_file.py +++ b/tests/config_objects/test_load_bad_rule_config_file.py @@ -48,30 +48,29 @@ def test_load_schema(self): _, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( rules=rules, override_config=None ) + self.assertEqual( - checklist["FV-D001-0001"].status, ChecklistObjectStatus.PENDING + checklist["bad_rule_config_empty_file"].status, ChecklistObjectStatus.ERRORED ) - self.assertIsNone(checklist["FV-D001-0001"].error) - self.assertIsNotNone(checklist["FV-D001-0001"].friendly_name) - self.assertEqual(checklist["FV-D001"].column_id, "ChargeType") - self.assertEqual(checklist["FV-D001"].status, ChecklistObjectStatus.PENDING) - self.assertIsNone(checklist["FV-D001"].error) - self.assertIsNotNone(checklist["FV-D001"].friendly_name) + self.assertEqual(checklist["valid_rule_config_column_metadata"].column_id, "ChargeType") + self.assertEqual(checklist["valid_rule_config_column_metadata"].status, ChecklistObjectStatus.PENDING) + self.assertIsNone(checklist["valid_rule_config_column_metadata"].error) + self.assertIsNotNone(checklist["valid_rule_config_column_metadata"].friendly_name) self.assertEqual( - checklist["FV-D001"].friendly_name, "Ensures that column is of string type." + checklist["valid_rule_config_column_metadata"].friendly_name, "Ensures that column is of string type." ) - for errored_file_paths in [ - "tests/samples/rule_configs/bad_rule_config_empty_file.yaml", - "tests/samples/rule_configs/bad_rule_config_missing_check.yaml", + for errored_checks in [ + 'bad_rule_config_empty_file', + 'bad_rule_config_missing_check' ]: self.assertEqual( - checklist[errored_file_paths].status, ChecklistObjectStatus.ERRORED + checklist[errored_checks].status, ChecklistObjectStatus.ERRORED ) - self.assertIsNotNone(checklist[errored_file_paths].error) - self.assertIsNone(checklist[errored_file_paths].friendly_name) - self.assertEqual(checklist[errored_file_paths].column_id, "Unknown") + self.assertIsNotNone(checklist[errored_checks].error) + self.assertIsNone(checklist[errored_checks].friendly_name) + self.assertEqual(checklist[errored_checks].column_id, "Unknown") def test_load_schema_without_valid_column_metadata(self): rules = [ @@ -88,11 +87,11 @@ def test_load_schema_without_valid_column_metadata(self): rules=rules, override_config=None ) self.assertEqual( - checklist["FV-D001-0001"].status, ChecklistObjectStatus.ERRORED + checklist["bad_rule_config_missing_check"].status, ChecklistObjectStatus.ERRORED ) - self.assertEqual( - checklist["FV-D001-0001"].error, - "ConfigurationError: No configuration found for column.", + self.assertRegex( + checklist["bad_rule_config_missing_check"].error, + "ValidationError:.*", ) - self.assertIsNotNone(checklist["FV-D001-0001"].friendly_name) - self.assertEqual(checklist["FV-D001-0001"].column_id, "ChargeType") + self.assertIsNotNone(checklist["valid_rule_config"].friendly_name) + self.assertEqual(checklist["valid_rule_config"].column_id, "ChargeType") diff --git a/tests/outputter/test_outputter_console.py b/tests/outputter/test_outputter_console.py index a947a39..bef733d 100644 --- a/tests/outputter/test_outputter_console.py +++ b/tests/outputter/test_outputter_console.py @@ -4,7 +4,7 @@ from focus_validator.config_objects.focus_to_pandera_schema_converter import ( FocusToPanderaSchemaConverter, ) -from focus_validator.outputter.outputter_console import ConsoleOutputter +from focus_validator.outputter.outputter_console import ConsoleOutputter, collapse_occurrence_range from focus_validator.rules.spec_rules import ValidationResult from focus_validator.validator import Validator @@ -34,6 +34,16 @@ def test_failure_output(self): ], ) + def test_collapse_range(self): + self.assertEqual( + collapse_occurrence_range([1, 5, 6, 7, 23.0]), + '1,5-7,23' + ) + self.assertEqual( + collapse_occurrence_range(['category', 'category2', 'category3']), + 'category,category2,category3' + ) + def test_output_with_bad_configs_loaded(self): schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( rules=[ @@ -50,6 +60,7 @@ def test_output_with_bad_configs_loaded(self): outputter = ConsoleOutputter(output_destination=None) checklist = outputter.__restructure_check_list__(result_set=validation_result) + outputter.write(validation_result) self.assertEqual( checklist.to_dict(orient="records"), [ diff --git a/tests/samples/rule_configs/bad_rule_config_empty_file.yaml b/tests/samples/rule_configs/bad_rule_config_empty_file.yaml index 8179223..e69de29 100644 --- a/tests/samples/rule_configs/bad_rule_config_empty_file.yaml +++ b/tests/samples/rule_configs/bad_rule_config_empty_file.yaml @@ -1,4 +0,0 @@ -check_id: FV-D001-0001 -column: ChargeType -validation_config: - check_friendly_name: "ChargeType must have a value from the list: {values}." diff --git a/tests/samples/rule_configs/bad_rule_config_missing_check.yaml b/tests/samples/rule_configs/bad_rule_config_missing_check.yaml index e69de29..8242360 100644 --- a/tests/samples/rule_configs/bad_rule_config_missing_check.yaml +++ b/tests/samples/rule_configs/bad_rule_config_missing_check.yaml @@ -0,0 +1,3 @@ +column_id: ChargeType +validation_config: + check_friendly_name: "ChargeType must have a value from the list: {values}." diff --git a/tests/samples/rule_configs/valid_rule_config.yaml b/tests/samples/rule_configs/valid_rule_config.yaml index 7d0c2bd..0c20356 100644 --- a/tests/samples/rule_configs/valid_rule_config.yaml +++ b/tests/samples/rule_configs/valid_rule_config.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001-0001 column_id: ChargeType check_friendly_name: "ChargeType must have a value from the list: {values}." check: diff --git a/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml b/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml index a89f1e8..8369e69 100644 --- a/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml +++ b/tests/samples/rule_configs/valid_rule_config_column_metadata.yaml @@ -1,4 +1,3 @@ -check_id: FV-D001 column_id: ChargeType check: data_type: string diff --git a/tests/test_validate_default_configs.py b/tests/test_validate_default_configs.py index 17d79d8..c994ab7 100644 --- a/tests/test_validate_default_configs.py +++ b/tests/test_validate_default_configs.py @@ -29,59 +29,3 @@ def test_version_sets_have_valid_config(self): self.assertIsNot( result.checklist[check_id].status, ChecklistObjectStatus.ERRORED ) - - def test_default_rules_with_sample_data(self): - check_id_pattern = re.compile(r"FV-[D,M]\d{3}-\d{4}$") - - for root, dirs, files in os.walk( - "focus_validator/rules/version_sets", topdown=False - ): - column_test_suites = [] - for file_path in files: - rule_path = os.path.join(root, file_path) - rule = Rule.load_yaml(rule_path=rule_path) - self.assertIsInstance(rule, Rule) - - column_id = rule.column_id - self.assertIsNotNone(re.match(check_id_pattern, rule.check_id)) - - check_column_id = rule.check_id.split("-")[1] - local_check_id = rule.check_id.split("-")[2] - column_test_suites.append((column_id, check_column_id, local_check_id)) - - # sort column test suites to allow grouping by column - column_test_suites = sorted(column_test_suites, key=lambda item: item[0]) - for _, test_suites in groupby(column_test_suites, key=lambda item: item[0]): - test_suites = list(test_suites) - self.assertEqual( - len(set([test_suite[1] for test_suite in test_suites])), 1 - ) - local_check_ids = [int(test_suite[2]) for test_suite in test_suites] - # check all ids are in order - self.assertEqual( - sorted(local_check_ids), list(range(1, len(local_check_ids) + 1)) - ) - - def test_metric_file_format_metric_vs_dimension(self): - metric_check_id_pattern = re.compile(r"FV-M\d{3}-\d{4}$") - dimension_check_id_pattern = re.compile(r"FV-D\d{3}-\d{4}$") - - for root, dirs, files in os.walk( - "focus_validator/rules/version_sets", topdown=False - ): - for file_path in files: - rule_path = os.path.join(root, file_path) - rule = Rule.load_yaml(rule_path=rule_path) - self.assertIsInstance(rule, Rule) - - if isinstance(rule.check, DataTypeCheck): - if rule.check.data_type == DataTypes.DECIMAL: - self.assertIsNotNone( - re.match(metric_check_id_pattern, rule.check_id), - "For metric column type check_id format should be FV-MYYY-YYYY", - ) - else: - self.assertIsNotNone( - re.match(dimension_check_id_pattern, rule.check_id), - "For metric column type check_id format should be FV-DYYY-YYYY", - )