From 9d0bc112576092444ad1c59b0e5c5a7531baf632 Mon Sep 17 00:00:00 2001 From: Joshua Kwan Date: Mon, 13 Nov 2023 15:34:40 -0500 Subject: [PATCH] First pass of validation rules for FOCUS 1.0 Signed-off-by: Joshua Kwan --- focus_validator/config_objects/common.py | 1 + .../focus_to_pandera_schema_converter.py | 8 ++ focus_validator/main.py | 2 +- .../ChargeDescription_IsString.yaml | 3 + .../ChargeDescription_NotNull.yaml | 3 + .../ChargeDescription_Required.yaml | 2 + .../ChargeFrequency_Enum.yaml | 6 ++ .../ChargeFrequency_IsString.yaml | 3 + .../ChargeFrequency_NotNull.yaml | 3 + .../ChargeFrequency_Required.yaml | 2 + .../ChargeSubcategory_Enum.yaml | 13 ++++ .../ChargeSubcategory_IsString.yaml | 3 + .../ChargeSubcategory_NotNull.yaml | 3 + .../ChargeSubcategory_Required.yaml | 2 + .../CommitmentDiscountCategory_Enum.yaml | 5 ++ .../CommitmentDiscountCategory_IsString.yaml | 3 + .../CommitmentDiscountCategory_NotNull.yaml | 3 + .../CommitmentDiscountCategory_Required.yaml | 3 + .../CommitmentDiscountId_IsString.yaml | 3 + .../CommitmentDiscountId_Nullable.yaml | 3 + .../CommitmentDiscountId_Required.yaml | 3 + .../CommitmentDiscountName_IsString.yaml | 3 + .../CommitmentDiscountName_Nullable.yaml | 3 + .../CommitmentDiscountName_Required.yaml | 2 + .../CommitmentDiscountType_IsString.yaml | 3 + .../CommitmentDiscountType_Nullable.yaml | 3 + .../PricingCategory_Enum.yaml | 7 ++ .../PricingCategory_IsString.yaml | 3 + .../PricingCategory_Nullable.yaml | 3 + .../PricingCategory_Required.yaml | 2 + .../PricingUnit_IsString.yaml | 3 + .../PricingUnit_Nullable.yaml | 3 + .../PricingUnit_Required.yaml | 2 + .../ResourceType_IsString.yaml | 3 + .../ResourceType_Nullable.yaml | 3 + .../ResourceType_Required.yaml | 2 + .../base_rule_definitions/SkuId_IsString.yaml | 3 + .../base_rule_definitions/SkuId_Nullable.yaml | 3 + .../base_rule_definitions/SkuId_Required.yaml | 2 + .../SkuPriceId_IsString.yaml | 3 + .../SkuPriceId_Nullable.yaml | 3 + .../SkuPriceId_Required.yaml | 2 + .../SubAccountId_IsString.yaml | 3 + .../SubAccountId_Nullable.yaml | 3 + .../SubAccountId_Required.yaml | 2 + .../SubAccountName_Required.yaml | 3 +- .../Tags_IsJSONObject.yaml | 3 + .../base_rule_definitions/Tags_Nullable.yaml | 3 + .../base_rule_definitions/Tags_Required.yaml | 3 + .../UsageQuantity_IsDecimal.yaml | 3 + .../UsageQuantity_Nullable.yaml | 3 + .../UsageQuantity_Required.yaml | 3 + .../UsageUnit_IsString.yaml | 3 + .../UsageUnit_Nullable.yaml | 3 + .../UsageUnit_Required.yaml | 2 + focus_validator/rules/checks.py | 14 ++++ focus_validator/rules/version_sets.yaml | 66 ++++++++++++++++ .../utils/download_currency_codes.py | 4 +- tests/attributes/test_attribute_json.py | 78 +++++++++++++++++++ .../test_check_type_friendly_name.py | 2 +- 60 files changed, 333 insertions(+), 6 deletions(-) create mode 100644 focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SkuId_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml create mode 100644 focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/Tags_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml create mode 100644 focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml create mode 100644 focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml create mode 100644 focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml create mode 100644 focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml create mode 100644 tests/attributes/test_attribute_json.py diff --git a/focus_validator/config_objects/common.py b/focus_validator/config_objects/common.py index 6b65705..2cb1562 100644 --- a/focus_validator/config_objects/common.py +++ b/focus_validator/config_objects/common.py @@ -20,6 +20,7 @@ class DataTypes(Enum): DECIMAL = "decimal" DATETIME = "datetime" CURRENCY_CODE = "currency-code" + STRINGIFIED_JSON_OBJECT = "stringified-json-object" class DataTypeCheck(BaseModel): diff --git a/focus_validator/config_objects/focus_to_pandera_schema_converter.py b/focus_validator/config_objects/focus_to_pandera_schema_converter.py index 410fc51..a45de09 100644 --- a/focus_validator/config_objects/focus_to_pandera_schema_converter.py +++ b/focus_validator/config_objects/focus_to_pandera_schema_converter.py @@ -78,6 +78,14 @@ def __generate_column_definition__( error=f"{rule.check_id}:::Ensures that column is of {data_type.value} type.", ) ) + elif data_type == DataTypes.STRINGIFIED_JSON_OBJECT: + pandera_type = None + column_checks.append( + pa.Check.check_stringified_json_object_dtype( + ignore_na=True, + error=f"{rule.check_id}:::Ensures that column is of {data_type.value} type.", + ) + ) else: pandera_type = pa.String diff --git a/focus_validator/main.py b/focus_validator/main.py index 0377313..6c1db66 100644 --- a/focus_validator/main.py +++ b/focus_validator/main.py @@ -33,7 +33,7 @@ def main(): help="Allow transitional rules in validation", ) parser.add_argument( - "--validate-version", default="0.5", help="Version of FOCUS to validate against" + "--validate-version", default="1.0", help="Version of FOCUS to validate against" ) parser.add_argument( "--rule-set-path", diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml new file mode 100644 index 0000000..920a124 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml new file mode 100644 index 0000000..01dd7c9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeDescription +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml new file mode 100644 index 0000000..f08f045 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeDescription_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeDescription +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml new file mode 100644 index 0000000..021af67 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Enum.yaml @@ -0,0 +1,6 @@ +column_id: ChargeFrequency +check: + value_in: + - "One-Time" + - "Recurring" + - "Usage-Based" \ No newline at end of file diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml new file mode 100644 index 0000000..ea4a54a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeFrequency +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml new file mode 100644 index 0000000..49eaedf --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeFrequency +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml new file mode 100644 index 0000000..97ad0df --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeFrequency_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeFrequency +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml new file mode 100644 index 0000000..6dec960 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Enum.yaml @@ -0,0 +1,13 @@ +column_id: ChargeSubcategory +check: + value_in: + # Allowed when ChargeType=Usage + - "On-Demand" + - "Used Commitment" + - "Unused Commitment" + - "Usage" + # Allowed when ChargeType=Adjustment + - "Refund" + - "Credit" + - "Rounding Error" + - "General Adjustment" diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml new file mode 100644 index 0000000..cd7545e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ChargeSubcategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml new file mode 100644 index 0000000..9ca2483 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: ChargeSubcategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml new file mode 100644 index 0000000..1deb197 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ChargeSubcategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: ChargeSubcategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml new file mode 100644 index 0000000..2ad823d --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Enum.yaml @@ -0,0 +1,5 @@ +column_id: CommitmentDiscountCategory +check: + value_in: + - "Spend" + - "Usage" diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml new file mode 100644 index 0000000..afe06ed --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml new file mode 100644 index 0000000..4b4d889 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_NotNull.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + allow_nulls: false diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml new file mode 100644 index 0000000..49b7f23 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountCategory_Required.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountCategory +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml new file mode 100644 index 0000000..650305b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml new file mode 100644 index 0000000..1b7e48c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml new file mode 100644 index 0000000..212e16b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountId_Required.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountId +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml new file mode 100644 index 0000000..ef740d6 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountName +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml new file mode 100644 index 0000000..99c567a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountName +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml new file mode 100644 index 0000000..d279fb9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountName_Required.yaml @@ -0,0 +1,2 @@ +column_id: CommitmentDiscountName +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml new file mode 100644 index 0000000..9daa993 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_IsString.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountType +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml new file mode 100644 index 0000000..2197a0b --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/CommitmentDiscountType_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: CommitmentDiscountType +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml new file mode 100644 index 0000000..2e42fa9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Enum.yaml @@ -0,0 +1,7 @@ +column_id: PricingCategory +check: + value_in: + - "On-Demand" + - "Dynamic" + - "Commitment-Based" + - "Other" diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml new file mode 100644 index 0000000..2065111 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_IsString.yaml @@ -0,0 +1,3 @@ +column_id: PricingCategory +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml new file mode 100644 index 0000000..49c257f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingCategory +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml new file mode 100644 index 0000000..4904b04 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingCategory_Required.yaml @@ -0,0 +1,2 @@ +column_id: PricingCategory +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml new file mode 100644 index 0000000..f138fa4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: PricingUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml new file mode 100644 index 0000000..34e841a --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: PricingUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml new file mode 100644 index 0000000..e2c935e --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/PricingUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: PricingUnit +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml new file mode 100644 index 0000000..68a5230 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_IsString.yaml @@ -0,0 +1,3 @@ +column_id: ResourceType +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml new file mode 100644 index 0000000..99cf430 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: ResourceType +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml b/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml new file mode 100644 index 0000000..5df814f --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/ResourceType_Required.yaml @@ -0,0 +1,2 @@ +column_id: ResourceType +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml new file mode 100644 index 0000000..4b43197 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SkuId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml new file mode 100644 index 0000000..d8656e9 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SkuId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml b/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml new file mode 100644 index 0000000..3fdb39d --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SkuId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml new file mode 100644 index 0000000..76460dd --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SkuPriceId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml new file mode 100644 index 0000000..a32dbcd --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SkuPriceId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml b/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml new file mode 100644 index 0000000..5352d19 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SkuPriceId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SkuPriceId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml new file mode 100644 index 0000000..aa49628 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_IsString.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountId +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml new file mode 100644 index 0000000..03f41eb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: SubAccountId +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml b/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml new file mode 100644 index 0000000..d492ff7 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/SubAccountId_Required.yaml @@ -0,0 +1,2 @@ +column_id: SubAccountId +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml b/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml index f7a9d68..635ab28 100644 --- a/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml +++ b/focus_validator/rules/base_rule_definitions/SubAccountName_Required.yaml @@ -1,3 +1,2 @@ column_id: SubAccountName -check: - column_required +check: column_required diff --git a/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml b/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml new file mode 100644 index 0000000..2832cea --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_IsJSONObject.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + data_type: stringified-json-object diff --git a/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml b/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml new file mode 100644 index 0000000..b06ee4c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/Tags_Required.yaml b/focus_validator/rules/base_rule_definitions/Tags_Required.yaml new file mode 100644 index 0000000..54ae7c2 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/Tags_Required.yaml @@ -0,0 +1,3 @@ +column_id: Tags +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml new file mode 100644 index 0000000..e7071b4 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_IsDecimal.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + data_type: decimal diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml new file mode 100644 index 0000000..05d4bfb --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml b/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml new file mode 100644 index 0000000..3a21150 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageQuantity_Required.yaml @@ -0,0 +1,3 @@ +column_id: UsageQuantity +check: + column_required diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml new file mode 100644 index 0000000..89fa9e0 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_IsString.yaml @@ -0,0 +1,3 @@ +column_id: UsageUnit +check: + data_type: string diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml new file mode 100644 index 0000000..b5fb4b5 --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_Nullable.yaml @@ -0,0 +1,3 @@ +column_id: UsageUnit +check: + allow_nulls: true diff --git a/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml b/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml new file mode 100644 index 0000000..a23616c --- /dev/null +++ b/focus_validator/rules/base_rule_definitions/UsageUnit_Required.yaml @@ -0,0 +1,2 @@ +column_id: UsageUnit +check: column_required diff --git a/focus_validator/rules/checks.py b/focus_validator/rules/checks.py index 27d7bd4..65b3275 100644 --- a/focus_validator/rules/checks.py +++ b/focus_validator/rules/checks.py @@ -1,6 +1,7 @@ from datetime import datetime from typing import Union +import json import numpy as np import pandas as pd from pandera import extensions @@ -63,3 +64,16 @@ def check_currency_code_dtype(pandas_obj: pd.Series): return pd.Series( map(lambda v: isinstance(v, str) and v in currency_codes, pandas_obj.values) ) + +@extensions.register_check_method() +def check_stringified_json_object_dtype(pandas_obj: pd.Series): + def __validate_stringified_json_object__(value: str): + try: + parsed = json.loads(value) + return isinstance(parsed, dict) + except Exception as e: + return False + + return pd.Series( + map(__validate_stringified_json_object__, pandas_obj.values) + ) diff --git a/focus_validator/rules/version_sets.yaml b/focus_validator/rules/version_sets.yaml index 2a601a2..4fbfcb3 100644 --- a/focus_validator/rules/version_sets.yaml +++ b/focus_validator/rules/version_sets.yaml @@ -1,3 +1,69 @@ +'1.0': +- ChargeDescription_IsString.yaml +- ChargeDescription_Required.yaml +- ChargeDescription_NotNull.yaml + +### Unmodified 0.5 content follows +- AmortizedCost_IsDecimal.yaml +- AmortizedCost_NotNull.yaml +- AmortizedCost_Required.yaml +- AvailabilityZone_IsString.yaml +- AvailabilityZone_Nullable.yaml +- BilledCost_IsDecimal.yaml +- BilledCost_NotNull.yaml +- BilledCost_Required.yaml +- BilledCurrency_IsCurrencyCode.yaml +- BilledCurrency_NotNull.yaml +- BilledCurrency_Required.yaml +- BillingAccountId_IsString.yaml +- BillingAccountId_NotNull.yaml +- BillingAccountId_Required.yaml +- BillingAccountName_IsString.yaml +- BillingAccountName_Nullable.yaml +- BillingAccountName_Required.yaml +- BillingPeriodEnd_IsDateTime.yaml +- BillingPeriodEnd_NotNull.yaml +- BillingPeriodEnd_Required.yaml +- BillingPeriodStart_IsDateTime.yaml +- BillingPeriodStart_NotNull.yaml +- BillingPeriodStart_Required.yaml +- ChargePeriodEnd_IsDateTime.yaml +- ChargePeriodEnd_NotNull.yaml +- ChargePeriodEnd_Required.yaml +- ChargePeriodStart_IsDateTime.yaml +- ChargePeriodStart_NotNull.yaml +- ChargePeriodStart_Required.yaml +- ChargeType_Enum.yaml +- ChargeType_IsString.yaml +- ChargeType_NotNull.yaml +- ChargeType_Required.yaml +- InvoiceIssuer_IsString.yaml +- InvoiceIssuer_NotNull.yaml +- Provider_IsString.yaml +- Provider_NotNull.yaml +- Provider_Required.yaml +- Publisher_IsString.yaml +- Publisher_NotNull.yaml +- Publisher_Required.yaml +- Region_IsString.yaml +- Region_NotNull.yaml +- Region_Required.yaml +- ResourceID_IsString.yaml +- ResourceID_Nullable.yaml +- ResourceName_IsString.yaml +- ResourceName_Nullable.yaml +- ResourceName_Required.yaml +- ServiceCategory_Enum.yaml +- ServiceCategory_IsString.yaml +- ServiceCategory_NotNull.yaml +- ServiceCategory_Required.yaml +- ServiceName_IsString.yaml +- ServiceName_NotNull.yaml +- ServiceName_Required.yaml +- SubAccountName_IsString.yaml +- SubAccountName_Nullable.yaml +- SubAccountName_Required.yaml + '0.5': - AmortizedCost_IsDecimal.yaml - AmortizedCost_NotNull.yaml diff --git a/focus_validator/utils/download_currency_codes.py b/focus_validator/utils/download_currency_codes.py index a748556..fb78773 100644 --- a/focus_validator/utils/download_currency_codes.py +++ b/focus_validator/utils/download_currency_codes.py @@ -7,7 +7,7 @@ CURRENCY_CODE_CSV_PATH = "focus_validator/utils/currency_codes.csv" -def download_currency_codes(): +def download_currency_codes(): # pragma: no cover r = requests.get(DATAHUB_URL) root = ET.fromstring(r.content.decode()) @@ -25,5 +25,5 @@ def get_currency_codes(): return set(df["currency_codes"].values) -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover download_currency_codes() diff --git a/tests/attributes/test_attribute_json.py b/tests/attributes/test_attribute_json.py new file mode 100644 index 0000000..e1ed79b --- /dev/null +++ b/tests/attributes/test_attribute_json.py @@ -0,0 +1,78 @@ +from unittest import TestCase +from uuid import uuid4 + +import pandas as pd +from pandera.errors import SchemaErrors + +from focus_validator.config_objects import Rule +from focus_validator.config_objects.common import ( + ChecklistObjectStatus, + DataTypeCheck, + DataTypes, +) +from focus_validator.config_objects.focus_to_pandera_schema_converter import ( + FocusToPanderaSchemaConverter, +) +from focus_validator.rules.spec_rules import ValidationResult + + +# noinspection DuplicatedCode +class TestAttributeJSONObject(TestCase): + def __eval_function__(self, sample_value, should_fail): + random_column_id = str(uuid4()) + random_check_id = str(uuid4()) + + schema, checklist = FocusToPanderaSchemaConverter.generate_pandera_schema( + rules=[ + Rule( + check_id=random_check_id, + column_id=random_column_id, + check=DataTypeCheck(data_type=DataTypes.STRINGIFIED_JSON_OBJECT), + ) + ] + ) + + sample_data = pd.DataFrame([{random_column_id: sample_value}]) + + try: + schema.validate(sample_data, lazy=True) + failure_cases = None + except SchemaErrors as e: + failure_cases = e.failure_cases + + validation_result = ValidationResult( + failure_cases=failure_cases, checklist=checklist + ) + validation_result.process_result() + + if should_fail: + self.assertIsNotNone(validation_result.failure_cases) + records = validation_result.failure_cases.to_dict(orient="records") + self.assertEqual(len(records), 1) + collected_values = [record["Values"] for record in records] + self.assertEqual(collected_values, [sample_value]) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.FAILED, + ) + else: + self.assertIsNone(validation_result.failure_cases) + self.assertEqual( + validation_result.checklist[random_check_id].status, + ChecklistObjectStatus.PASSED, + ) + + def test_valid_json(self): + self.__eval_function__('{"my-cool-tag": "focus", "my-cool-tag-2": "whahoo"}', False) + + def test_valid_json_empty(self): + self.__eval_function__('{}', False) + + def test_valid_json_bad_data_type(self): + self.__eval_function__(0, True) + + def test_valid_json_null_value(self): + self.__eval_function__(None, False) + + def test_valid_json_empty_string(self): + self.__eval_function__("", True) diff --git a/tests/config_objects/test_check_type_friendly_name.py b/tests/config_objects/test_check_type_friendly_name.py index 61e6a53..3010960 100644 --- a/tests/config_objects/test_check_type_friendly_name.py +++ b/tests/config_objects/test_check_type_friendly_name.py @@ -69,6 +69,6 @@ def test_assign_bad_type(self): ) self.assertEqual(len(cm.exception.errors()), 1) self.assertIn( - "Input should be 'string', 'decimal', 'datetime' or 'currency-code'", + "Input should be 'string', 'decimal', 'datetime', 'currency-code' or 'stringified-json-object'", str(cm.exception), )