From 8ea7922f39f2e0527e3dce7fc1aee993c7a49c8e Mon Sep 17 00:00:00 2001 From: Rostislav Solomasov Date: Mon, 18 Dec 2023 11:24:56 +0200 Subject: [PATCH] Update comparer.py - added support for schema selection Adding the ability to select which schemas are being compared --- sqlalchemydiff/comparer.py | 343 ++++++++++++++++++++++--------------- 1 file changed, 207 insertions(+), 136 deletions(-) diff --git a/sqlalchemydiff/comparer.py b/sqlalchemydiff/comparer.py index d7d29fd..21de48f 100644 --- a/sqlalchemydiff/comparer.py +++ b/sqlalchemydiff/comparer.py @@ -2,12 +2,23 @@ from copy import deepcopy from .util import ( - TablesInfo, DiffResult, InspectorFactory, CompareResult, IgnoreManager + TablesInfo, + DiffResult, + InspectorFactory, + CompareResult, + IgnoreManager, ) -def compare(left_uri, right_uri, ignores=None, ignores_sep=None): - """Compare two databases, given two URIs. +def compare( + left_uri, + right_uri, + ignores=None, + ignores_sep=None, + left_schema="public", + right_schema="public", +): + """Compare two databases, given two URIs and the corresponding schemas. Compare two databases, ignoring whatever is specified in `ignores`. @@ -15,8 +26,8 @@ def compare(left_uri, right_uri, ignores=None, ignores_sep=None): info = { 'uris': { - 'left': 'left_uri', - 'right': 'right_uri', + 'left': 'left_uri | schema: left_schema', + 'right': 'right_uri | schema: right_schema' , }, 'tables': { 'left': 'tables_left', @@ -76,6 +87,8 @@ def compare(left_uri, right_uri, ignores=None, ignores_sep=None): to be excluded from the comparison. :param string ignores_sep: Separator to be used to spilt the `ignores` clauses. + :param string left_schema: The schema for the first (left) database. + :param string right_schema: The schema for the second (right) database. :return: A :class:`~.util.CompareResult` object with ``info`` and ``errors`` dicts populated with the comparison result. @@ -85,18 +98,30 @@ def compare(left_uri, right_uri, ignores=None, ignores_sep=None): left_inspector, right_inspector = _get_inspectors(left_uri, right_uri) tables_info = _get_tables_info( - left_inspector, right_inspector, ignore_manager.ignore_tables) + left_inspector, + right_inspector, + ignore_manager.ignore_tables, + left_schema=left_schema, + right_schema=right_schema, + ) - info = _get_info_dict(left_uri, right_uri, tables_info) + info = _get_info_dict(left_uri, right_uri, tables_info, left_schema, right_schema) - info['tables_data'] = _get_tables_data( - tables_info.common, left_inspector, right_inspector, ignore_manager + info["tables_data"] = _get_tables_data( + tables_info.common, + left_inspector, + right_inspector, + ignore_manager, + left_schema, + right_schema, ) - info['enums'] = _get_enums_info( + info["enums"] = _get_enums_info( left_inspector, right_inspector, - ignore_manager.get('*', 'enum'), + ignore_manager.get("*", "enum"), + left_schema, + right_schema, ) errors = _compile_errors(info) @@ -111,36 +136,44 @@ def _get_inspectors(left_uri, right_uri): return left_inspector, right_inspector -def _get_tables_info(left_inspector, right_inspector, ignore_tables): - """Get information about the differences at the table level. """ +def _get_tables_info( + left_inspector, right_inspector, ignore_tables, left_schema, right_schema +): + """Get information about the differences at the table level.""" tables_left, tables_right = _get_tables( - left_inspector, right_inspector, ignore_tables) + left_inspector, right_inspector, ignore_tables, left_schema, right_schema + ) - tables_left_only, tables_right_only = _get_tables_diff( - tables_left, tables_right) + tables_left_only, tables_right_only = _get_tables_diff(tables_left, tables_right) tables_common = _get_common_tables(tables_left, tables_right) return TablesInfo( - left=tables_left, right=tables_right, left_only=tables_left_only, - right_only=tables_right_only, common=tables_common) + left=tables_left, + right=tables_right, + left_only=tables_left_only, + right_only=tables_right_only, + common=tables_common, + ) -def _get_tables(left_inspector, right_inspector, ignore_tables): - """Get table names for both databases. ``ignore_tables`` are removed. """ - tables_left = _get_tables_names(left_inspector, ignore_tables) - tables_right = _get_tables_names(right_inspector, ignore_tables) +def _get_tables( + left_inspector, right_inspector, ignore_tables, left_schema, right_schema +): + """Get table names for both databases. ``ignore_tables`` are removed.""" + tables_left = _get_tables_names(left_inspector, ignore_tables, left_schema) + tables_right = _get_tables_names(right_inspector, ignore_tables, right_schema) return tables_left, tables_right -def _get_tables_names(inspector, ignore_tables): - return sorted(set(inspector.get_table_names()) - ignore_tables) +def _get_tables_names(inspector, ignore_tables, schema): + return sorted(set(inspector.get_table_names(schema=schema)) - ignore_tables) def _get_tables_diff(tables_left, tables_right): return ( _diff_table_lists(tables_left, tables_right), - _diff_table_lists(tables_right, tables_left) + _diff_table_lists(tables_right, tables_left), ) @@ -152,35 +185,45 @@ def _get_common_tables(tables_left, tables_right): return sorted(set(tables_left) & set(tables_right)) -def _get_info_dict(left_uri, right_uri, tables_info): - """Create an empty stub for the `info` dict. """ +def _get_info_dict(left_uri, right_uri, tables_info, left_schema, right_schema): + """Create an empty stub for the `info` dict.""" info = { - 'uris': { - 'left': left_uri, - 'right': right_uri, + "uris": { + "left": f"{left_uri} | schema: {left_schema}", + "right": f"{right_uri} | schema: {right_schema}", }, - 'tables': { - 'left': tables_info.left, - 'left_only': tables_info.left_only, - 'right': tables_info.right, - 'right_only': tables_info.right_only, - 'common': tables_info.common, + "tables": { + "left": tables_info.left, + "left_only": tables_info.left_only, + "right": tables_info.right, + "right_only": tables_info.right_only, + "common": tables_info.common, }, - 'tables_data': {}, - 'enums': {}, + "tables_data": {}, + "enums": {}, } return info def _get_tables_data( - tables_common, left_inspector, right_inspector, ignore_manager + tables_common, + left_inspector, + right_inspector, + ignore_manager, + left_schema, + right_schema, ): tables_data = {} for table_name in tables_common: table_data = _get_table_data( - left_inspector, right_inspector, table_name, ignore_manager + left_inspector, + right_inspector, + table_name, + ignore_manager, + left_schema, + right_schema, ) tables_data[table_name] = table_data @@ -188,44 +231,59 @@ def _get_tables_data( def _get_table_data( - left_inspector, right_inspector, table_name, ignore_manager + left_inspector, + right_inspector, + table_name, + ignore_manager, + left_schema, + right_schema, ): table_data = {} # foreign keys - table_data['foreign_keys'] = _get_foreign_keys_info( + table_data["foreign_keys"] = _get_foreign_keys_info( left_inspector, right_inspector, table_name, - ignore_manager.get(table_name, 'fk') + ignore_manager.get(table_name, "fk"), + left_schema=left_schema, + right_schema=right_schema, ) - table_data['primary_keys'] = _get_primary_keys_info( + table_data["primary_keys"] = _get_primary_keys_info( left_inspector, right_inspector, table_name, - ignore_manager.get(table_name, 'pk') + ignore_manager.get(table_name, "pk"), + left_schema=left_schema, + right_schema=right_schema, ) - table_data['indexes'] = _get_indexes_info( + table_data["indexes"] = _get_indexes_info( left_inspector, right_inspector, table_name, - ignore_manager.get(table_name, 'idx') + ignore_manager.get(table_name, "idx"), + left_schema=left_schema, + right_schema=right_schema, ) - table_data['columns'] = _get_columns_info( + table_data["columns"] = _get_columns_info( left_inspector, right_inspector, table_name, - ignore_manager.get(table_name, 'col') + ignore_manager.get(table_name, "col"), + left_schema=left_schema, + right_schema=right_schema, ) - table_data['constraints'] = _get_constraints_info( + table_data["constraints"] = _get_constraints_info( left_inspector, right_inspector, table_name, - ignore_manager.get(table_name, 'cons') + ignore_manager.get(table_name, "cons"), + left_schema=left_schema, + right_schema=right_schema, ) return table_data @@ -258,11 +316,13 @@ def _diff_dicts(left, right): if left[key] == right[key]: common.append(left[key]) else: - diff.append({ - 'key': key, - 'left': left[key], - 'right': right[key], - }) + diff.append( + { + "key": key, + "left": left[key], + "right": right[key], + } + ) return DiffResult( left_only=left_only, right_only=right_only, common=common, diff=diff @@ -270,44 +330,49 @@ def _diff_dicts(left, right): def _get_foreign_keys_info( - left_inspector, right_inspector, table_name, ignores + left_inspector, right_inspector, table_name, ignores, left_schema, right_schema ): - left_fk_list = _get_foreign_keys(left_inspector, table_name) - right_fk_list = _get_foreign_keys(right_inspector, table_name) + left_fk_list = _get_foreign_keys(left_inspector, table_name, left_schema) + right_fk_list = _get_foreign_keys(right_inspector, table_name, right_schema) left_fk_list = _discard_ignores_by_name(left_fk_list, ignores) right_fk_list = _discard_ignores_by_name(right_fk_list, ignores) # process into dict - left_fk = dict((elem['name'], elem) for elem in left_fk_list) - right_fk = dict((elem['name'], elem) for elem in right_fk_list) + left_fk = dict((elem["name"], elem) for elem in left_fk_list) + right_fk = dict((elem["name"], elem) for elem in right_fk_list) return _diff_dicts(left_fk, right_fk) -def _get_foreign_keys(inspector, table_name): - return inspector.get_foreign_keys(table_name) +def _get_foreign_keys(inspector, table_name, schema): + return inspector.get_foreign_keys(table_name=table_name, schema=schema) def _get_primary_keys_info( - left_inspector, right_inspector, table_name, ignores + left_inspector, right_inspector, table_name, ignores, left_schema, right_schema ): - left_pk_constraint = _get_primary_keys(left_inspector, table_name) - right_pk_constraint = _get_primary_keys(right_inspector, table_name) + left_pk_constraint = _get_primary_keys(left_inspector, table_name, left_schema) + right_pk_constraint = _get_primary_keys(right_inspector, table_name, right_schema) - pk_constraint_has_name = ('name' in left_pk_constraint and - left_pk_constraint['name'] is not None) + pk_constraint_has_name = ( + "name" in left_pk_constraint and left_pk_constraint["name"] is not None + ) if pk_constraint_has_name: - left_pk = ({left_pk_constraint['name']: left_pk_constraint} - if _discard_ignores_by_name([left_pk_constraint], ignores) - else {}) - right_pk = ({right_pk_constraint['name']: right_pk_constraint} - if _discard_ignores_by_name([right_pk_constraint], ignores) - else {}) + left_pk = ( + {left_pk_constraint["name"]: left_pk_constraint} + if _discard_ignores_by_name([left_pk_constraint], ignores) + else {} + ) + right_pk = ( + {right_pk_constraint["name"]: right_pk_constraint} + if _discard_ignores_by_name([right_pk_constraint], ignores) + else {} + ) else: - left_pk_list = left_pk_constraint['constrained_columns'] - right_pk_list = right_pk_constraint['constrained_columns'] + left_pk_list = left_pk_constraint["constrained_columns"] + right_pk_list = right_pk_constraint["constrained_columns"] left_pk_list = _discard_ignores(left_pk_list, ignores) right_pk_list = _discard_ignores(right_pk_list, ignores) @@ -319,38 +384,42 @@ def _get_primary_keys_info( return _diff_dicts(left_pk, right_pk) -def _get_primary_keys(inspector, table_name): - return inspector.get_pk_constraint(table_name) +def _get_primary_keys(inspector, table_name, schema): + return inspector.get_pk_constraint(table_name=table_name, schema=schema) -def _get_indexes_info(left_inspector, right_inspector, table_name, ignores): - left_index_list = _get_indexes(left_inspector, table_name) - right_index_list = _get_indexes(right_inspector, table_name) +def _get_indexes_info( + left_inspector, right_inspector, table_name, ignores, left_schema, right_schema +): + left_index_list = _get_indexes(left_inspector, table_name, left_schema) + right_index_list = _get_indexes(right_inspector, table_name, right_schema) left_index_list = _discard_ignores_by_name(left_index_list, ignores) right_index_list = _discard_ignores_by_name(right_index_list, ignores) # process into dict - left_index = dict((elem['name'], elem) for elem in left_index_list) - right_index = dict((elem['name'], elem) for elem in right_index_list) + left_index = dict((elem["name"], elem) for elem in left_index_list) + right_index = dict((elem["name"], elem) for elem in right_index_list) return _diff_dicts(left_index, right_index) -def _get_indexes(inspector, table_name): - return inspector.get_indexes(table_name) +def _get_indexes(inspector, table_name, schema): + return inspector.get_indexes(table_name=table_name, schema=schema) -def _get_columns_info(left_inspector, right_inspector, table_name, ignores): - left_columns_list = _get_columns(left_inspector, table_name) - right_columns_list = _get_columns(right_inspector, table_name) +def _get_columns_info( + left_inspector, right_inspector, table_name, ignores, left_schema, right_schema +): + left_columns_list = _get_columns(left_inspector, table_name, left_schema) + right_columns_list = _get_columns(right_inspector, table_name, right_schema) left_columns_list = _discard_ignores_by_name(left_columns_list, ignores) right_columns_list = _discard_ignores_by_name(right_columns_list, ignores) # process into dict - left_columns = dict((elem['name'], elem) for elem in left_columns_list) - right_columns = dict((elem['name'], elem) for elem in right_columns_list) + left_columns = dict((elem["name"], elem) for elem in left_columns_list) + right_columns = dict((elem["name"], elem) for elem in right_columns_list) # process `type` fields _process_types(left_columns) @@ -359,62 +428,65 @@ def _get_columns_info(left_inspector, right_inspector, table_name, ignores): return _diff_dicts(left_columns, right_columns) -def _get_columns(inspector, table_name): - return inspector.get_columns(table_name) +def _get_columns(inspector, table_name, schema): + return inspector.get_columns(table_name=table_name, schema=schema) -def _get_constraints_info(left_inspector, right_inspector, - table_name, ignores): - left_constraints_list = _get_constraints_data(left_inspector, table_name) - right_constraints_list = _get_constraints_data(right_inspector, table_name) +def _get_constraints_info( + left_inspector, right_inspector, table_name, ignores, left_schema, right_schema +): + left_constraints_list = _get_constraints_data( + left_inspector, table_name, left_schema + ) + right_constraints_list = _get_constraints_data( + right_inspector, table_name, right_schema + ) - left_constraints_list = _discard_ignores_by_name(left_constraints_list, - ignores) - right_constraints_list = _discard_ignores_by_name(right_constraints_list, - ignores) + left_constraints_list = _discard_ignores_by_name(left_constraints_list, ignores) + right_constraints_list = _discard_ignores_by_name(right_constraints_list, ignores) # process into dict - left_constraints = dict((elem['name'], elem) - for elem in left_constraints_list) - right_constraints = dict((elem['name'], elem) - for elem in right_constraints_list) + left_constraints = dict((elem["name"], elem) for elem in left_constraints_list) + right_constraints = dict((elem["name"], elem) for elem in right_constraints_list) return _diff_dicts(left_constraints, right_constraints) -def _get_constraints_data(inspector, table_name): +def _get_constraints_data(inspector, table_name, schema): try: - return inspector.get_check_constraints(table_name) + return inspector.get_check_constraints(table_name=table_name, schema=schema) except (AttributeError, NotImplementedError): # pragma: no cover # sqlalchemy < 1.1.0 # or a dialect that doesn't implement get_check_constraints return [] -def _get_enums_info(left_inspector, right_inspector, ignores): - left_enums_list = _get_enums_data(left_inspector) - right_enums_list = _get_enums_data(right_inspector) +def _get_enums_info( + left_inspector, right_inspector, ignores, left_schema, right_schema +): + left_enums_list = _get_enums_data(left_inspector, left_schema) + right_enums_list = _get_enums_data(right_inspector, right_schema) left_enums_list = _discard_ignores_by_name(left_enums_list, ignores) right_enums_list = _discard_ignores_by_name(right_enums_list, ignores) # process into dict - left_enums = dict((elem['name'], elem) for elem in left_enums_list) - right_enums = dict((elem['name'], elem) for elem in right_enums_list) + left_enums = dict((elem["name"], elem) for elem in left_enums_list) + right_enums = dict((elem["name"], elem) for elem in right_enums_list) return _diff_dicts(left_enums, right_enums) -def _get_enums_data(inspector): +def _get_enums_data(inspector, schema): try: # as of 1.2.0, PostgreSQL dialect only; see PGInspector - return inspector.get_enums() + return inspector.get_enums(schema=schema) except AttributeError: return [] def _discard_ignores_by_name(items, ignores): - return [item for item in items if item['name'] not in ignores] + return [item for item in items if item["name"] not in ignores] def _discard_ignores(items, ignores): @@ -423,8 +495,7 @@ def _discard_ignores(items, ignores): def _process_types(column_dict): for column in column_dict: - column_dict[column]['type'] = _process_type( - column_dict[column]['type']) + column_dict[column]["type"] = _process_type(column_dict[column]["type"]) def _process_type(type_): @@ -438,44 +509,44 @@ def _process_type(type_): def _compile_errors(info): - """Create ``errors`` dict from ``info`` dict. """ + """Create ``errors`` dict from ``info`` dict.""" errors_template = { - 'tables': {}, - 'tables_data': {}, - 'enums': {}, + "tables": {}, + "tables_data": {}, + "enums": {}, } errors = deepcopy(errors_template) # first check if tables aren't a match - if info['tables']['left_only']: - errors['tables']['left_only'] = info['tables']['left_only'] + if info["tables"]["left_only"]: + errors["tables"]["left_only"] = info["tables"]["left_only"] - if info['tables']['right_only']: - errors['tables']['right_only'] = info['tables']['right_only'] + if info["tables"]["right_only"]: + errors["tables"]["right_only"] = info["tables"]["right_only"] # then check if there is a discrepancy in the data for each table - keys = ['foreign_keys', 'primary_keys', 'indexes', 'columns', - 'constraints'] - subkeys = ['left_only', 'right_only', 'diff'] + keys = ["foreign_keys", "primary_keys", "indexes", "columns", "constraints"] + subkeys = ["left_only", "right_only", "diff"] - for table_name in info['tables_data']: + for table_name in info["tables_data"]: for key in keys: for subkey in subkeys: - if info['tables_data'][table_name][key][subkey]: - table_d = errors['tables_data'].setdefault(table_name, {}) - table_d.setdefault(key, {})[subkey] = info[ - 'tables_data'][table_name][key][subkey] + if info["tables_data"][table_name][key][subkey]: + table_d = errors["tables_data"].setdefault(table_name, {}) + table_d.setdefault(key, {})[subkey] = info["tables_data"][ + table_name + ][key][subkey] for subkey in subkeys: - if info['enums'][subkey]: - errors['enums'][subkey] = info['enums'][subkey] + if info["enums"][subkey]: + errors["enums"][subkey] = info["enums"][subkey] if errors != errors_template: - errors['uris'] = info['uris'] + errors["uris"] = info["uris"] return errors return {} def _make_result(info, errors): - """Create a :class:`~.util.CompareResult` object. """ + """Create a :class:`~.util.CompareResult` object.""" return CompareResult(info, errors)