From d7a2a288d7a676380c528818f4af0433fa6dd0ea Mon Sep 17 00:00:00 2001 From: Bruno Agutoli Date: Mon, 15 Nov 2021 20:13:45 +1100 Subject: [PATCH 1/2] fixing query and dashboard importation when data is out-to-date --- redash_toolbelt/client.py | 4 ++- redash_toolbelt/examples/migrate.py | 40 ++++++++++++++++++++++++----- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/redash_toolbelt/client.py b/redash_toolbelt/client.py index 2a07d48..6084b0b 100644 --- a/redash_toolbelt/client.py +++ b/redash_toolbelt/client.py @@ -80,8 +80,10 @@ def create_data_source(self, name, _type, options): return self._post("api/data_sources", json=payload) - def dashboard(self, slug): + def dashboard(self, slug, legacy=False): """GET api/dashboards/{slug}""" + if legacy: + return self._get("api/dashboards/{}?legacy=1".format(slug)).json() return self._get("api/dashboards/{}".format(slug)).json() def create_query(self, query_json): diff --git a/redash_toolbelt/examples/migrate.py b/redash_toolbelt/examples/migrate.py index 93ebd75..7aa596a 100644 --- a/redash_toolbelt/examples/migrate.py +++ b/redash_toolbelt/examples/migrate.py @@ -9,6 +9,7 @@ import sys, os, json, logging, textwrap import click from redash_toolbelt import Redash +from dateutil import parser logging.basicConfig(stream=sys.stdout, level=logging.ERROR) logging.getLogger("requests").setLevel("ERROR") @@ -388,9 +389,19 @@ def import_query_subroutine(query): data_source_id = DATA_SOURCES.get(query["data_source_id"]) + should_update_query = False if origin_id in meta["queries"] or str(origin_id) in meta["queries"]: - print("Query {} - SKIP - was already imported".format(origin_id)) - return + destination_id = meta["queries"][query["id"]] + dest_query = dest_client.get_query(destination_id) + origin_updated_at = parser.parse(query["updated_at"]) + dest_updated_at = parser.parse(dest_query["updated_at"]) + + if origin_updated_at > dest_updated_at: + should_update_query = True + print("Origin: %s > Desct: %s" % (query["id"], destination_id)) + else: + print("Query {} - SKIP - was already imported".format(origin_id)) + return if data_source_id is None: print( @@ -424,7 +435,11 @@ def import_query_subroutine(query): user_client = Redash(DESTINATION, user_api_key) try: - response = user_client.create_query(data) + if should_update_query: + print("Updating query %s - %s OK" % (origin_id, destination_id)) + response = user_client.update_query(destination_id, data) + else: + response = user_client.create_query(data) except Exception as e: print("Query {} - FAIL - {}".format(origin_id, e)) return @@ -552,9 +567,19 @@ def import_dashboards(orig_client, dest_client): dashboards = sorted(dashboards, key=lambda x: x.get("created_at", 0)) for dashboard in dashboards: + should_update_query = False if dashboard["slug"] in meta["dashboards"]: - print("Dashboard `{}` - SKIP - Already imported".format(dashboard["slug"])) - continue + # print(json.dumps(dashboard, indent=2)) + dashboard_slug = meta["dashboards"][dashboard["slug"]] + dest_dashboard = dest_client.dashboard(dashboard_slug, True) + origin_updated_at = parser.parse(dashboard["updated_at"]) + dest_updated_at = parser.parse(dest_dashboard["updated_at"]) + if origin_updated_at > dest_updated_at: + should_update_query = True + print("Updating: %s" % dashboard_slug) + else: + print("Dashboard `{}` - SKIP - Already imported".format(dashboard["slug"])) + continue print(" importing: {}".format(dashboard["slug"])) @@ -573,7 +598,10 @@ def import_dashboards(orig_client, dest_client): user_client = Redash(DESTINATION, user_api_key) - new_dashboard = user_client.create_dashboard(d["name"]) + if should_update_query: + new_dashboard = dest_dashboard + else: + new_dashboard = user_client.create_dashboard(d["name"]) new_dash_id = new_dashboard["id"] From fd4cc6000414d20b73de4312fb51f8fba7ae1b54 Mon Sep 17 00:00:00 2001 From: Bruno Agutoli Date: Tue, 16 Nov 2021 08:49:23 +1100 Subject: [PATCH 2/2] removing comments + improving improving logs --- pyproject.toml | 1 + redash_toolbelt/examples/migrate.py | 5 ++--- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ebb332f..b16ff92 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,6 +14,7 @@ packages = [ [tool.poetry.dependencies] python = "^3.6" +python-dateutil = "2.8.1" requests = "^2.22.0" click = "^7.0" diff --git a/redash_toolbelt/examples/migrate.py b/redash_toolbelt/examples/migrate.py index 7aa596a..678402e 100644 --- a/redash_toolbelt/examples/migrate.py +++ b/redash_toolbelt/examples/migrate.py @@ -398,7 +398,7 @@ def import_query_subroutine(query): if origin_updated_at > dest_updated_at: should_update_query = True - print("Origin: %s > Desct: %s" % (query["id"], destination_id)) + print("Query {} - Re-importing - Destination is out-to-date.".format(origin_id)) else: print("Query {} - SKIP - was already imported".format(origin_id)) return @@ -569,14 +569,13 @@ def import_dashboards(orig_client, dest_client): for dashboard in dashboards: should_update_query = False if dashboard["slug"] in meta["dashboards"]: - # print(json.dumps(dashboard, indent=2)) dashboard_slug = meta["dashboards"][dashboard["slug"]] dest_dashboard = dest_client.dashboard(dashboard_slug, True) origin_updated_at = parser.parse(dashboard["updated_at"]) dest_updated_at = parser.parse(dest_dashboard["updated_at"]) if origin_updated_at > dest_updated_at: should_update_query = True - print("Updating: %s" % dashboard_slug) + print("Dashboard `{}` - Re-importing origin".format(dashboard_slug)) else: print("Dashboard `{}` - SKIP - Already imported".format(dashboard["slug"])) continue