From 0931b07c406d5e9001894bd7bf508178575b1f44 Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 10 May 2022 20:40:42 -0700 Subject: [PATCH 001/107] stub convert45 --- CHANGELOG.md | 11 +++-------- archive-v45/cqml45.py | 33 +++++++++++++++++++++++++++++++++ archive-v45/cqml45_convert.py | 17 +++++++++++++++++ archive-v45/cqml45_test.py | 8 ++++++++ 4 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 archive-v45/cqml45.py create mode 100644 archive-v45/cqml45_convert.py create mode 100644 archive-v45/cqml45_test.py diff --git a/CHANGELOG.md b/CHANGELOG.md index b0f8ef1..56a5fa0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,21 +1,16 @@ -0.4.6 (unreleased) +0.5.0 (unreleased) ------------------ -- Nothing changed yet. +- Use extrinsic folder names for project/package +- do: schedule for top-level ordering and scheduling 0.4.5 (2022-05-10) ------------------ - Fix Manifest - - -0.4.4 (unreleased) ------------------- - - YMM 0.6.1 support: open direct URL for test runs - 0.4.3 (2022-04-18) ------------------ diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py new file mode 100644 index 0000000..c5780a1 --- /dev/null +++ b/archive-v45/cqml45.py @@ -0,0 +1,33 @@ +from .keys import * + +kDoc_OLD="doc" + +def is_v02(yaml_data): return isinstance(yaml_data, dict) + +def convert_action(action): + if kDoc_OLD in action: + doc = action.pop(kDoc_OLD) + action[kDoc] = doc + if kCols in action: + cols = action.pop(kCols) + if cols != "all": + colmap = {key: "tbd" for key in cols} + action[kCols] = colmap + return action + +def upgrade(cqml1): + actions = {} + for action in cqml1: + print(action) + id = action["id"] + action.pop("id") + new_action = convert_action(action) + actions[id] = action + cqml2 = { + "cqml": 0.2, + "actions": actions + } + return cqml2 + +def ensure_v02(yaml_data): + return yaml_data if is_v02(yaml_data) else upgrade(yaml_data) diff --git a/archive-v45/cqml45_convert.py b/archive-v45/cqml45_convert.py new file mode 100644 index 0000000..7c5af23 --- /dev/null +++ b/archive-v45/cqml45_convert.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import pytest, sys +from .context import cqml + +if len(sys.argv) > 1: + files = sys.argv[1:] + for file in files: + print(file) + upgrade_file(file) + +def upgrade_file(yaml_file): + print("Upgrading "+yaml_file) + with open(yaml_file) as data: + raw_yaml = yaml.full_load(data) + # insert converter here + with open(yaml_file, 'w') as file: + yaml.dump(raw_yaml, file, sort_keys=False) diff --git a/archive-v45/cqml45_test.py b/archive-v45/cqml45_test.py new file mode 100644 index 0000000..f880fde --- /dev/null +++ b/archive-v45/cqml45_test.py @@ -0,0 +1,8 @@ +#!/usr/bin/env python3 +import pytest +from .context import cqml, TEST_YAML + +def test_from_file(): + cvm = cqml.from_file(TEST_YAML, None) + assert(cvm) + assert(cvm.yaml) From 1cb50d5a95b20de9d33f910a9b229bbb294d1448 Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 10 May 2022 20:45:41 -0700 Subject: [PATCH 002/107] rename --- archive-v45/cqml45.py | 52 ++++++++++++++++------------------- archive-v45/cqml45_convert.py | 17 ------------ archive-v45/cqml45_trans.py | 33 ++++++++++++++++++++++ 3 files changed, 56 insertions(+), 46 deletions(-) delete mode 100644 archive-v45/cqml45_convert.py create mode 100644 archive-v45/cqml45_trans.py diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py index c5780a1..84f55e0 100644 --- a/archive-v45/cqml45.py +++ b/archive-v45/cqml45.py @@ -1,33 +1,27 @@ -from .keys import * +#!/usr/bin/env python3 +import pytest, sys -kDoc_OLD="doc" +# +# Take two paths: root4 root5 +# Copy files in each subfolder +# Strip out meta +# Strip leading numbers +# -def is_v02(yaml_data): return isinstance(yaml_data, dict) +def convert(root4, root5): + for folder in os.listdir(root4): + print(file) + upgrade_file(file) -def convert_action(action): - if kDoc_OLD in action: - doc = action.pop(kDoc_OLD) - action[kDoc] = doc - if kCols in action: - cols = action.pop(kCols) - if cols != "all": - colmap = {key: "tbd" for key in cols} - action[kCols] = colmap - return action +def upgrade_file(yaml_file): + print("Upgrading "+yaml_file) + with open(yaml_file) as data: + raw_yaml = yaml.full_load(data) + # insert converter here + with open(yaml_file, 'w') as file: + yaml.dump(raw_yaml, file, sort_keys=False) -def upgrade(cqml1): - actions = {} - for action in cqml1: - print(action) - id = action["id"] - action.pop("id") - new_action = convert_action(action) - actions[id] = action - cqml2 = { - "cqml": 0.2, - "actions": actions - } - return cqml2 - -def ensure_v02(yaml_data): - return yaml_data if is_v02(yaml_data) else upgrade(yaml_data) +if len(sys.argv) > 2: + convert(sys.argv[1], sys.argv[2]) +else: + print("cqml45 root4 root5 (requires 2 arguments)") diff --git a/archive-v45/cqml45_convert.py b/archive-v45/cqml45_convert.py deleted file mode 100644 index 7c5af23..0000000 --- a/archive-v45/cqml45_convert.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -import pytest, sys -from .context import cqml - -if len(sys.argv) > 1: - files = sys.argv[1:] - for file in files: - print(file) - upgrade_file(file) - -def upgrade_file(yaml_file): - print("Upgrading "+yaml_file) - with open(yaml_file) as data: - raw_yaml = yaml.full_load(data) - # insert converter here - with open(yaml_file, 'w') as file: - yaml.dump(raw_yaml, file, sort_keys=False) diff --git a/archive-v45/cqml45_trans.py b/archive-v45/cqml45_trans.py new file mode 100644 index 0000000..c5780a1 --- /dev/null +++ b/archive-v45/cqml45_trans.py @@ -0,0 +1,33 @@ +from .keys import * + +kDoc_OLD="doc" + +def is_v02(yaml_data): return isinstance(yaml_data, dict) + +def convert_action(action): + if kDoc_OLD in action: + doc = action.pop(kDoc_OLD) + action[kDoc] = doc + if kCols in action: + cols = action.pop(kCols) + if cols != "all": + colmap = {key: "tbd" for key in cols} + action[kCols] = colmap + return action + +def upgrade(cqml1): + actions = {} + for action in cqml1: + print(action) + id = action["id"] + action.pop("id") + new_action = convert_action(action) + actions[id] = action + cqml2 = { + "cqml": 0.2, + "actions": actions + } + return cqml2 + +def ensure_v02(yaml_data): + return yaml_data if is_v02(yaml_data) else upgrade(yaml_data) From 6ca596b66639203b9073532e3e8de3daa010cdb7 Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 10 May 2022 21:15:51 -0700 Subject: [PATCH 003/107] extracted --- archive-v45/cqml45.py | 37 +++++++++++++++++++++++++------------ 1 file changed, 25 insertions(+), 12 deletions(-) mode change 100644 => 100755 archive-v45/cqml45.py diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py old mode 100644 new mode 100755 index 84f55e0..ded02b2 --- a/archive-v45/cqml45.py +++ b/archive-v45/cqml45.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import pytest, sys +import os, sys, yaml # # Take two paths: root4 root5 @@ -8,20 +8,33 @@ # Strip leading numbers # -def convert(root4, root5): - for folder in os.listdir(root4): - print(file) - upgrade_file(file) +ROOT="/Users/nauto/Developer" +R4=f"{ROOT}/it/databricks" +R5=f"{ROOT}/dbt/cqml" -def upgrade_file(yaml_file): - print("Upgrading "+yaml_file) +def read_yaml(yaml_file): with open(yaml_file) as data: raw_yaml = yaml.full_load(data) - # insert converter here + return raw_yaml + +def extract(root): + tree = [] + for folder in os.scandir(root): + if folder.is_dir(): + print(folder.name) + for file in os.scandir(folder.path): + if file.name.endswith(".yml"): + print(file.name) + yml = read_yaml(file.path) + node = {"folder":folder.name, "file": file.name, "yml": yml} + tree.append(node) + return tree + + +def write_yaml(yaml_file, raw_yaml): with open(yaml_file, 'w') as file: yaml.dump(raw_yaml, file, sort_keys=False) -if len(sys.argv) > 2: - convert(sys.argv[1], sys.argv[2]) -else: - print("cqml45 root4 root5 (requires 2 arguments)") +t = extract(R4) +print("\nextracted\n") +[print(n["file"]) for n in t] From 5bb6c7be3fe4fc798b3d10b1c0eafd9ea3e91218 Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 10 May 2022 21:21:38 -0700 Subject: [PATCH 004/107] path5 --- archive-v45/cqml45.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py index ded02b2..150e85e 100755 --- a/archive-v45/cqml45.py +++ b/archive-v45/cqml45.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -import os, sys, yaml +import os, yaml # # Take two paths: root4 root5 @@ -24,9 +24,10 @@ def extract(root): print(folder.name) for file in os.scandir(folder.path): if file.name.endswith(".yml"): - print(file.name) yml = read_yaml(file.path) - node = {"folder":folder.name, "file": file.name, "yml": yml} + file5 = ''.join([c for c in file.name if not c.isdigit()]) + path5 = os.path.join(folder.name, file5) + node = {"path":path5, "yml": yml} tree.append(node) return tree @@ -36,5 +37,6 @@ def write_yaml(yaml_file, raw_yaml): yaml.dump(raw_yaml, file, sort_keys=False) t = extract(R4) -print("\nextracted\n") -[print(n["file"]) for n in t] +print(f"\nExtracted: {len(t)} files\n") +for n in t: + print(n["path"]) From 4fe50d8de531df98f22e09cd4976fea6a5ec7a9a Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 10 May 2022 21:32:03 -0700 Subject: [PATCH 005/107] convert(root, node): --- archive-v45/cqml45.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py index 150e85e..5b17a8d 100755 --- a/archive-v45/cqml45.py +++ b/archive-v45/cqml45.py @@ -17,6 +17,10 @@ def read_yaml(yaml_file): raw_yaml = yaml.full_load(data) return raw_yaml +def write_yaml(yaml_file, raw_yaml): + with open(yaml_file, 'w') as file: + yaml.dump(raw_yaml, file, sort_keys=False) + def extract(root): tree = [] for folder in os.scandir(root): @@ -25,18 +29,25 @@ def extract(root): for file in os.scandir(folder.path): if file.name.endswith(".yml"): yml = read_yaml(file.path) - file5 = ''.join([c for c in file.name if not c.isdigit()]) - path5 = os.path.join(folder.name, file5) - node = {"path":path5, "yml": yml} + node = {"file":file.name, "folder":folder.name, "yml": yml} tree.append(node) return tree - -def write_yaml(yaml_file, raw_yaml): - with open(yaml_file, 'w') as file: - yaml.dump(raw_yaml, file, sort_keys=False) +def convert(root, node): + file = ''.join([c for c in node["file"] if not c.isdigit()]) + path = os.path.join(root, node["folder"], file) + yml = node["yml"] + del yml["meta"] + yml["cqml"] = 0.5 + yml["project"] = node["folder"] + yml["package"] = file + write_yaml(path, yml) + return path t = extract(R4) print(f"\nExtracted: {len(t)} files\n") +print(t[0]["yml"]) for n in t: - print(n["path"]) + print(n["file"]) + p = convert(R5, n) + print("\t",p) From 46d55f3dd90bf8cc29c7df8d3c3d6f250acaa0ba Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 10 May 2022 21:40:02 -0700 Subject: [PATCH 006/107] write_yaml --- archive-v45/cqml45.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py index 5b17a8d..6b4536d 100755 --- a/archive-v45/cqml45.py +++ b/archive-v45/cqml45.py @@ -34,19 +34,24 @@ def extract(root): return tree def convert(root, node): + dir = node["folder"] file = ''.join([c for c in node["file"] if not c.isdigit()]) - path = os.path.join(root, node["folder"], file) + prefix = file.split("_")[0] + name = file.split("_")[1] + if prefix == "rnr": dir = prefix + if prefix == "sierra": name = f"{prefix}.yml" + path = os.path.join(root, dir, name) yml = node["yml"] del yml["meta"] yml["cqml"] = 0.5 - yml["project"] = node["folder"] - yml["package"] = file + yml["project"] = dir + yml["package"] = name write_yaml(path, yml) return path t = extract(R4) print(f"\nExtracted: {len(t)} files\n") -print(t[0]["yml"]) +#print(t[0]["yml"]) for n in t: print(n["file"]) p = convert(R5, n) From 60b879d4dbab5e53a7dd0f0e15a84c6002b57ff1 Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:30:24 -0700 Subject: [PATCH 007/107] "0.4.6.dev1_"2022-05-11T22:30:24Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 93ee737..60da6e1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev0 +0.4.6.dev1 diff --git a/version.txt.prev b/version.txt.prev index 8ebd7c0..93ee737 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.4.dev1 +0.4.6.dev0 From a9ffaa4576bf0c9f3f6e79e7ec3a5156b7b4f2c1 Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:30:25 -0700 Subject: [PATCH 008/107] "post-regen-tests/cqml_test_db.py-0.4.6.dev1" --- tests/cqml_test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 48e1dd5..1acfb50 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -8,8 +8,8 @@ !pip install --upgrade pip #!pip install cqml -!pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@ymm-6 -!pip install cqml==0.4.4.dev2 +!pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 +!pip install cqml==0.4.6.dev1 import cqml From 4f9d195adae65b4190eea7c47f71cd85f211b91c Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:37:42 -0700 Subject: [PATCH 009/107] remove unused v45 files --- archive-v45/cqml45_test.py | 8 -------- archive-v45/cqml45_trans.py | 33 --------------------------------- tests/tree_test.py | 14 ++++++++++++++ 3 files changed, 14 insertions(+), 41 deletions(-) delete mode 100644 archive-v45/cqml45_test.py delete mode 100644 archive-v45/cqml45_trans.py create mode 100644 tests/tree_test.py diff --git a/archive-v45/cqml45_test.py b/archive-v45/cqml45_test.py deleted file mode 100644 index f880fde..0000000 --- a/archive-v45/cqml45_test.py +++ /dev/null @@ -1,8 +0,0 @@ -#!/usr/bin/env python3 -import pytest -from .context import cqml, TEST_YAML - -def test_from_file(): - cvm = cqml.from_file(TEST_YAML, None) - assert(cvm) - assert(cvm.yaml) diff --git a/archive-v45/cqml45_trans.py b/archive-v45/cqml45_trans.py deleted file mode 100644 index c5780a1..0000000 --- a/archive-v45/cqml45_trans.py +++ /dev/null @@ -1,33 +0,0 @@ -from .keys import * - -kDoc_OLD="doc" - -def is_v02(yaml_data): return isinstance(yaml_data, dict) - -def convert_action(action): - if kDoc_OLD in action: - doc = action.pop(kDoc_OLD) - action[kDoc] = doc - if kCols in action: - cols = action.pop(kCols) - if cols != "all": - colmap = {key: "tbd" for key in cols} - action[kCols] = colmap - return action - -def upgrade(cqml1): - actions = {} - for action in cqml1: - print(action) - id = action["id"] - action.pop("id") - new_action = convert_action(action) - actions[id] = action - cqml2 = { - "cqml": 0.2, - "actions": actions - } - return cqml2 - -def ensure_v02(yaml_data): - return yaml_data if is_v02(yaml_data) else upgrade(yaml_data) diff --git a/tests/tree_test.py b/tests/tree_test.py new file mode 100644 index 0000000..eed5093 --- /dev/null +++ b/tests/tree_test.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import pytest +from .context import cqml, TEST_YAML +from .db_mock import spark + +def test_pkg(): + dict = cqml.pkg_cqml('cqml_test', spark, 'tests') + assert 'pkg' in dict + assert 'html' in dict + assert 'actions' in dict + +def skip_test_all(): + dict = cqml.pkg_all(spark, 'tests') + assert 'cqml_test' in dict From 2b698afed548c736762b8bba6eeeea40466741b1 Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:41:24 -0700 Subject: [PATCH 010/107] split out yml wrappers --- src/cqml/wrappers.py | 16 +--------------- src/cqml/yml.py | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 15 deletions(-) create mode 100644 src/cqml/yml.py diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py index 71c45cb..8f9bf2b 100644 --- a/src/cqml/wrappers.py +++ b/src/cqml/wrappers.py @@ -3,6 +3,7 @@ from operator import itemgetter from .db2quilt import cvm2pkg, extract_pkg from .cvm import CVM +from .yml import * class CQML(CVM): def __init__(self, yaml_data, spark): @@ -24,14 +25,6 @@ def do_save(self, action): pkg = cvm2pkg(self, False) # do not re-run return pkg -def upgrade_file(yaml_file): - print("Upgrading "+yaml_file) - with open(yaml_file) as data: - raw_yaml = yaml.full_load(data) - # insert converter here - with open(yaml_file, 'w') as file: - yaml.dump(raw_yaml, file, sort_keys=False) - def from_file(yaml_file, spark): print("Loading "+yaml_file) with open(yaml_file) as data: @@ -72,13 +65,6 @@ def pkg_cqml(name, spark, folder="pipes"): 'frames': cvm.df, } -def yml_keys(folder="pipes"): - files = os.listdir(folder) - keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")] - keys.sort() - print(keys) - return keys - def pkg_all(spark, folder="pipes"): keys = yml_keys(folder) pkgs = {key:pkg_cqml(key, spark, folder) for key in keys} diff --git a/src/cqml/yml.py b/src/cqml/yml.py new file mode 100644 index 0000000..ea4d32b --- /dev/null +++ b/src/cqml/yml.py @@ -0,0 +1,20 @@ +import yaml +import os,shutil +from operator import itemgetter +from .db2quilt import cvm2pkg, extract_pkg +from .cvm import CVM + +def upgrade_file(yaml_file): + print("Upgrading "+yaml_file) + with open(yaml_file) as data: + raw_yaml = yaml.full_load(data) + # insert converter here + with open(yaml_file, 'w') as file: + yaml.dump(raw_yaml, file, sort_keys=False) + +def yml_keys(folder="pipes"): + files = os.listdir(folder) + keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")] + keys.sort() + print(keys) + return keys From 97f07e148fe476f2799b06818b260a17b62f938f Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:41:40 -0700 Subject: [PATCH 011/107] run all pkg tests vs tree tests --- tests/pkg_test.py | 2 +- tests/tree_test.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pkg_test.py b/tests/pkg_test.py index eed5093..3e74018 100644 --- a/tests/pkg_test.py +++ b/tests/pkg_test.py @@ -9,6 +9,6 @@ def test_pkg(): assert 'html' in dict assert 'actions' in dict -def skip_test_all(): +def test_all(): dict = cqml.pkg_all(spark, 'tests') assert 'cqml_test' in dict diff --git a/tests/tree_test.py b/tests/tree_test.py index eed5093..fae03aa 100644 --- a/tests/tree_test.py +++ b/tests/tree_test.py @@ -3,7 +3,7 @@ from .context import cqml, TEST_YAML from .db_mock import spark -def test_pkg(): +def skip_test_pkg(): dict = cqml.pkg_cqml('cqml_test', spark, 'tests') assert 'pkg' in dict assert 'html' in dict From 91eca4206b3ad5d03d525be2a8b43d44e004078e Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:49:13 -0700 Subject: [PATCH 012/107] fix invalid key in demo.yml --- src/cqml/boxquilt.py | 2 +- src/cqml/db2quilt.py | 4 ++-- tests/demo.yml | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cqml/boxquilt.py b/src/cqml/boxquilt.py index cb64a17..8b62124 100644 --- a/src/cqml/boxquilt.py +++ b/src/cqml/boxquilt.py @@ -119,7 +119,7 @@ def create_or_update_box(self, skipUpdate=False): dbfs = list(self.rows.keys()) to_create = list(set(dbfs) - set(box)) to_update = list(set(dbfs).intersection(box)) - print(f"create:{len(to_create)} update:{len(to_update)}") + print(f"box_create:{len(to_create)} update:{len(to_update)}") n = 0 for name in to_create: diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index a54233d..a47a9b3 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -112,8 +112,8 @@ def callback(x): grid.restore(filter=[['{KEY}', '==', x]]) grid """ def make_widget(opts): - print('make_widget') - print(opts) + #print('make_widget') + #print(opts) code = [NB_WIDGET.format(KEY=col,WIDGET=w) for col, w in opts.items()] cells = [[True, c] for c in code] return cells diff --git a/tests/demo.yml b/tests/demo.yml index ff7bfdc..3fdb1f6 100644 --- a/tests/demo.yml +++ b/tests/demo.yml @@ -62,7 +62,7 @@ actions: letter: tbd plus_operator: do: call - from: rename_merged + from: grouped operator: + args: - next From e8172e1fa58f1e3448174993ad32115c85021d7a Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:51:58 -0700 Subject: [PATCH 013/107] cqml tree --- pipes/demo/demo.yml | 142 +++++++++++++++++++++++++++ pipes/test/cqml.yml | 227 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 369 insertions(+) create mode 100644 pipes/demo/demo.yml create mode 100644 pipes/test/cqml.yml diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml new file mode 100644 index 0000000..3fdb1f6 --- /dev/null +++ b/pipes/demo/demo.yml @@ -0,0 +1,142 @@ +cqml: 0.4.4 +id: cqml_demo +meta: + org: nauto + project: sangam + s3.bucket: biz-databricks-root-prod-us + catalog: quilt + root: /dbfs/tmp +actions: + /Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data: + do: loadfiles + as_number: devices + tables: {test1: test1.csv, test2: test2.csv} + selected: + +doc: simple select query + do: select + from: test1 + where: + letter: {A: contains, B: contains} + text: {null: is_not} + cols: {num: tbd, dat: tbd} + merged: + +doc: join and save into a datagrid + do: merge + save: grid + into: $id + join: num + from: test2 + join_type: inner + cols: + num: join + letter|better: tbd + dat|dot: tbd + box_export: + +doc: Split by num, upload, store URLs + do: box + from: $id + group: num + box: + root_folder: CQML_Test + root_id: 156760265584 + data_dir: test + file_ext: csv + expiration_date: '2200-02-02' + cols: + letter: sort + dat: sort + widget-report: + do: report + from: test1 + cols: + text: Combobox + grouped: + do: group + from: test1 + agg: + num: sum + dat: max + sort: max_dat + cols: + text: tbd + letter: tbd + plus_operator: + do: call + from: grouped + operator: + + args: + - next + - num + calc_quarters: + do: calc + from: plus_operator + operator: / + round: 3 + else: 0 + args: + - $col + - 4 + cols: + num: qnum + next: qnumb + plus_operator: qdumb + flagged: + do: flag + from: test1 + where: + text: + null: is + union: + do: union + into: test1 + from: test2 + drop: + - note + macro_def: + do: macro|merge + into: "{into}" + join: "{join}" + from: test2 + join_type: inner + cols: + "{join}": tbd + num|numerate: tbd + text|texas: tbd + dat|database: tbd + macro_run: + do: macro.macro_def + into: test1 + join: letter + char: A + rename: + do: assign + from: macro_run + pivoted: + do: pivot + skip: true + from: $id + agg: + sum_num: count + sort: text + pivot: letter + cols: + text: tbd + cqml_test_summarize: + do: summary + save: [grid, parquet] + from: test1 + count: num + as: total + cols: + letter: tbd + text: tbd + aggregates: + do: group + from: $id + agg: + concat_space: count + call_coalesce: sum + sort: n_concat_space + cols: + num: tbd + letter: tbd diff --git a/pipes/test/cqml.yml b/pipes/test/cqml.yml new file mode 100644 index 0000000..e91dbca --- /dev/null +++ b/pipes/test/cqml.yml @@ -0,0 +1,227 @@ +cqml: 0.2 +id: cqml_test +meta: + org: nauto + project: sangam + s3.bucket: biz-databricks-root-prod-us + catalog: quilt + root: /dbfs/tmp +actions: + /Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data: + do: loadfiles + as_number: devices + tables: + test1: test1.csv + test2: test2.csv + today: + do: call + from: test1 + sort: dat + ensure_current: today + function: current_date + args: [] + unique_num: + do: unique + from: test1 + sort: dat + save: grid + ensure_unique: num + cols: + num: tbd + count: [letter] + box_details: + do: box + from: test1 + group: num + +doc: Split by num, upload, store URLs + box: + root_folder: CQML_Test + root_id: 156760265584 + data_dir: test + file_ext: csv + expiration_date: '2200-02-02' + cols: + letter: sort + dat: sort + widget-report: + do: report + from: test1 + cols: + text: Combobox +# num: slider + grouped: + do: group + from: test1 + agg: + num: sum + dat: max + sort: max_dat + cols: + text: tbd + letter: tbd + selected: + do: select + from: test1 + where: + letter: + A: contains + B: contains + text: + null: is_not + cols: + num: tbd + dat: tbd + letter: tbd + text: tbd + merged: + do: merge + #save: grid + into: test1 + join: num + from: test2 + join_type: inner + where: + letter: + A: contains + cols: + num: tbd + letter|better: tbd + dat|dot: tbd + rename_merged: + do: merge + into: test1 + join: num + from: test2 + join_type: inner + where: + letter: + A: contains + cols: + num|next: tbd + letter|better: tbd + dat|dot: tbd + plus_operator: + do: call + from: rename_merged + operator: + + args: + - next + - num + calc_quarters: + do: calc + from: plus_operator + operator: / + round: 3 + else: 0 + args: + - $col + - 4 + cols: + num: qnum + next: qnumb + plus_operator: qdumb + flagged: + do: flag + from: test1 + where: + text: + null: is + count_days: + do: call + from: test1 + function: datediff + args: + - current_date() + - dat + check_count: + do: flag + from: $id + where: + count_days: + 1: lesser + union: + do: union + into: test1 + from: test2 + drop: + - note + cqml_test_inner_dupes: + do: merge + save: table + into: test1 + join: num + join_type: inner + from: test2 + keep_join: left + cols: + num: tbd + letter|character: tbd + macro_def: + do: macro|merge + into: "{into}" + join: "{join}" + from: test2 + join_type: inner + cols: + "{join}": tbd + num|numerate: tbd + text|texas: tbd + dat|database: tbd + macro_run: + do: macro.macro_def + into: test1 + join: letter + char: A + mult_number: + do: assign + from: macro_run + pivoted: + do: pivot + skip: true + from: $id + agg: + sum_num: count + sort: text + pivot: letter + cols: + text: tbd + cqml_test_summarize: + do: summary + save: [grid, parquet] + from: test1 + count: num + as: total + cols: + letter: tbd + text: tbd + call_coalesce: + do: call + from: test1 + function: coalesce + args: + - text + - "'Unassigned'" + concat_space: + do: call + from: $id + operator: '||' + args: + - num + - "' '" + - letter + aggregates: + do: group + from: $id + agg: + concat_space: count + call_coalesce: sum + sort: n_concat_space + cols: + num: tbd + letter: tbd + leftsemi_match: + do: select + from: test1 + matching: + num: + test2: num From 8b54507e5419aec83023601937abd17bb50dd51e Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 15:56:54 -0700 Subject: [PATCH 014/107] skip pgk tests while dev --- tests/{pkg_test.py => pkg_test_skip.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/{pkg_test.py => pkg_test_skip.py} (100%) diff --git a/tests/pkg_test.py b/tests/pkg_test_skip.py similarity index 100% rename from tests/pkg_test.py rename to tests/pkg_test_skip.py From d9a2c7c0f512062ddc38f57c6aee82aaddc9e840 Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 16:01:11 -0700 Subject: [PATCH 015/107] test_yml_keys --- src/cqml/__init__.py | 1 + src/cqml/yml.py | 8 ++++++++ tests/tree_test.py | 9 ++++----- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/cqml/__init__.py b/src/cqml/__init__.py index dc7855f..4df6dec 100644 --- a/src/cqml/__init__.py +++ b/src/cqml/__init__.py @@ -4,4 +4,5 @@ # TODO: https://github.com/LucaCanali/sparkMeasure +from .yml import * from .wrappers import * diff --git a/src/cqml/yml.py b/src/cqml/yml.py index ea4d32b..2394041 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -13,6 +13,14 @@ def upgrade_file(yaml_file): yaml.dump(raw_yaml, file, sort_keys=False) def yml_keys(folder="pipes"): + files = os.listdir(folder) + print(files) + keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")] + keys.sort() + print(keys) + return keys + +def yml_tree(folder="pipes"): files = os.listdir(folder) keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")] keys.sort() diff --git a/tests/tree_test.py b/tests/tree_test.py index fae03aa..0ad55fd 100644 --- a/tests/tree_test.py +++ b/tests/tree_test.py @@ -3,11 +3,10 @@ from .context import cqml, TEST_YAML from .db_mock import spark -def skip_test_pkg(): - dict = cqml.pkg_cqml('cqml_test', spark, 'tests') - assert 'pkg' in dict - assert 'html' in dict - assert 'actions' in dict +def test_yml_keys(): + dict = cqml.yml_keys("pipes/test") + assert dict + assert "cqml" in dict def skip_test_all(): dict = cqml.pkg_all(spark, 'tests') From 2a9ad42898e8fe6ee7c2a4c661bfb699f28dc4c8 Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 16:18:34 -0700 Subject: [PATCH 016/107] extract yml_tree --- src/cqml/yml.py | 26 ++++++++++++++++++++++---- tests/tree_test.py | 13 +++++++------ 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/cqml/yml.py b/src/cqml/yml.py index 2394041..80a18b4 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -21,8 +21,26 @@ def yml_keys(folder="pipes"): return keys def yml_tree(folder="pipes"): - files = os.listdir(folder) - keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")] - keys.sort() - print(keys) + keys = extract(folder) return keys + +def read_yaml(yaml_file): + with open(yaml_file) as data: + raw_yaml = yaml.full_load(data) + return raw_yaml + +def write_yaml(yaml_file, raw_yaml): + with open(yaml_file, 'w') as file: + yaml.dump(raw_yaml, file, sort_keys=False) + +def extract(root): + tree = [] + for folder in os.scandir(root): + if folder.is_dir(): + print(folder.name) + for file in os.scandir(folder.path): + if file.name.endswith(".yml"): + yml = read_yaml(file.path) + node = {"file":file.name, "folder":folder.name, "yml": yml} + tree.append(node) + return tree diff --git a/tests/tree_test.py b/tests/tree_test.py index 0ad55fd..75b7652 100644 --- a/tests/tree_test.py +++ b/tests/tree_test.py @@ -4,10 +4,11 @@ from .db_mock import spark def test_yml_keys(): - dict = cqml.yml_keys("pipes/test") - assert dict - assert "cqml" in dict + keys = cqml.yml_keys("pipes/test") + assert keys + assert "cqml" in keys -def skip_test_all(): - dict = cqml.pkg_all(spark, 'tests') - assert 'cqml_test' in dict +def test_yml_tree(): + keys = cqml.yml_tree("pipes") + assert keys + assert len(keys) == 2 From 8f062827fd21c2d9fa504bf7fc647f2547609eab Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 16:30:35 -0700 Subject: [PATCH 017/107] extract key from tree --- src/cqml/yml.py | 19 ++++++++++++++----- tests/tree_test.py | 9 ++++++--- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/src/cqml/yml.py b/src/cqml/yml.py index 80a18b4..e7c6e72 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -33,14 +33,23 @@ def write_yaml(yaml_file, raw_yaml): with open(yaml_file, 'w') as file: yaml.dump(raw_yaml, file, sort_keys=False) +def yml_folder(folder, tree): + print(folder.name) + for entry in os.scandir(folder.path): + if entry.name.endswith(".yml"): + key = os.path.splitext(entry.name)[0] + #yml = read_yaml(entry.path) + node = {"file":entry.name, "folder":folder.name, "path": entry.path, "key": key} + tree.append(node) + elif entry.is_dir(): + print(folder.name) + yml_folder(folder, tree) + return tree + def extract(root): tree = [] for folder in os.scandir(root): if folder.is_dir(): print(folder.name) - for file in os.scandir(folder.path): - if file.name.endswith(".yml"): - yml = read_yaml(file.path) - node = {"file":file.name, "folder":folder.name, "yml": yml} - tree.append(node) + yml_folder(folder, tree) return tree diff --git a/tests/tree_test.py b/tests/tree_test.py index 75b7652..6b6f975 100644 --- a/tests/tree_test.py +++ b/tests/tree_test.py @@ -9,6 +9,9 @@ def test_yml_keys(): assert "cqml" in keys def test_yml_tree(): - keys = cqml.yml_tree("pipes") - assert keys - assert len(keys) == 2 + rows = cqml.yml_tree("pipes") + assert rows + assert len(rows) == 2 + d0 = rows[0] + assert "key" in d0 + From 3c810c58d11635145fbb9244cbf8d28ef254ee5c Mon Sep 17 00:00:00 2001 From: drernie Date: Wed, 11 May 2022 21:32:10 -0700 Subject: [PATCH 018/107] spec out new cqml parse --- src/cqml/yml.py | 6 ++--- tests/cqml_test_db.py.tmp | 52 ++++++++++++++++++++++++++++----------- 2 files changed, 40 insertions(+), 18 deletions(-) diff --git a/src/cqml/yml.py b/src/cqml/yml.py index e7c6e72..467609d 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -33,18 +33,18 @@ def write_yaml(yaml_file, raw_yaml): with open(yaml_file, 'w') as file: yaml.dump(raw_yaml, file, sort_keys=False) -def yml_folder(folder, tree): +def yml_folder(folder, nodes): print(folder.name) for entry in os.scandir(folder.path): if entry.name.endswith(".yml"): key = os.path.splitext(entry.name)[0] #yml = read_yaml(entry.path) node = {"file":entry.name, "folder":folder.name, "path": entry.path, "key": key} - tree.append(node) + nodes.append(node) elif entry.is_dir(): print(folder.name) yml_folder(folder, tree) - return tree + return nodes def extract(root): tree = [] diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index a78da20..63dca31 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -15,29 +15,51 @@ import cqml # COMMAND ---------- -KEY="cqml_test" -cvm = cqml.load_cqml(KEY,spark, '.') -cvm.debug = True -cvm.run() -print(cvm.sizes) +CQML_ROOT="pipes" +croot = cqml.parse(CQML_ROOT) +dbutils.widgets.dropdown("CONF", tree.default_key(), tree.keys()) +dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) # COMMAND ---------- -K = list(cvm.df.keys()) -print(K) -def d(i): return cvm.df[K[i]] -def view(i): - print(K[i]) - d(i).show() -def values(i, col): return d(i).select(col).distinct().collect() -view(-1) +CONF=getArgument("CONF") +DEBUG=True if getArgument("DEBUG") == "DEBUG" else False +print(f'Parameters[{CONF}]DEBUG={DEBUG} {CF}') +cvm = croot.new(spark, CONF, DEBUG) +print(cvm.status()) # COMMAND ---------- -cvm.do_save({}) -displayHTML(cvm.pkg.html) +if not DEBUG: + cvm.run() + print(cvm.sizes) + dbutils.notebook.exit(0) # COMMAND ---------- + +cvm.init() +steps = cvm.steps() +print(steps) +#dbutils.notebook.exit(0) + + +# COMMAND ---------- + +df = {} +def values(s, col): return df[s].select(col).distinct().collect() + +for step in steps: + print(step) + df[step] = cvm.test(step) + df[step].show() + +# COMMAND ---------- + +cvm.save() +displayHTML(cvm.result()) + +# COMMAND ---------- + dbutils.notebook.exit(0) #spark.sql('create database nauto') From 754cab8a6c27ffd8f9544402c00ff6f9eab661ca Mon Sep 17 00:00:00 2001 From: drernie Date: Thu, 12 May 2022 08:29:20 -0700 Subject: [PATCH 019/107] class CQML_Parser --- src/cqml/parse.py | 27 +++++++++++++++++++++++++++ tests/cqml_test_db.py.tmp | 4 ++-- 2 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 src/cqml/parse.py diff --git a/src/cqml/parse.py b/src/cqml/parse.py new file mode 100644 index 0000000..cd0fc5b --- /dev/null +++ b/src/cqml/parse.py @@ -0,0 +1,27 @@ +import os, yaml + +class CQML_Parser: + def __init__(self, root): + self.nodes = [] + self.scan(root) + + def scan(self, root): + for entry in os.scandir(root): + self.parse(entry, root) + + def parse(self, entry, folder): + if entry.name.endswith(".yml"): + key = os.path.splitext(entry.name)[0] + yml = read_yaml(entry.path) + node = { + "file":entry.name, + "folder":folder, + "path": entry.path, + "key": key, + "fkey": f"{folder}.{key}", + "yml": yml + } + self.nodes.append(node) + elif entry.is_dir(): + print(entry.name) + self.scan(entry.name) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index 63dca31..a1e0a3f 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -16,7 +16,7 @@ import cqml # COMMAND ---------- CQML_ROOT="pipes" -croot = cqml.parse(CQML_ROOT) +cq = cqml.parse(CQML_ROOT) dbutils.widgets.dropdown("CONF", tree.default_key(), tree.keys()) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) @@ -25,7 +25,7 @@ dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG} {CF}') -cvm = croot.new(spark, CONF, DEBUG) +cvm = cq.new(spark, CONF, DEBUG) print(cvm.status()) # COMMAND ---------- From 2c9a69ecaafa8adf21dd4a591e70ea907408de1b Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 13 May 2022 17:42:15 -0700 Subject: [PATCH 020/107] root --- src/cqml/{parse.py => root.py} | 2 +- tests/cqml_test_db.py.tmp | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) rename src/cqml/{parse.py => root.py} (97%) diff --git a/src/cqml/parse.py b/src/cqml/root.py similarity index 97% rename from src/cqml/parse.py rename to src/cqml/root.py index cd0fc5b..28a10d0 100644 --- a/src/cqml/parse.py +++ b/src/cqml/root.py @@ -1,6 +1,6 @@ import os, yaml -class CQML_Parser: +class Root: def __init__(self, root): self.nodes = [] self.scan(root) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index a1e0a3f..dc6f4ef 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -16,8 +16,8 @@ import cqml # COMMAND ---------- CQML_ROOT="pipes" -cq = cqml.parse(CQML_ROOT) -dbutils.widgets.dropdown("CONF", tree.default_key(), tree.keys()) +root = cqml.root(CQML_ROOT) +dbutils.widgets.dropdown("CONF", root.default_key(), root.keys()) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) # COMMAND ---------- @@ -25,7 +25,7 @@ dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG} {CF}') -cvm = cq.new(spark, CONF, DEBUG) +cvm = root.new(spark, CONF, DEBUG) print(cvm.status()) # COMMAND ---------- From cdd40be28421b577adefc0687162159fd1f06aca Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 13 May 2022 20:07:05 -0700 Subject: [PATCH 021/107] Root pipes --- src/cqml/root.py | 22 +++++++++++----------- src/cqml/wrappers.py | 4 ++++ tests/root_test.py | 9 +++++++++ tests/tree_test.py | 17 ----------------- 4 files changed, 24 insertions(+), 28 deletions(-) create mode 100644 tests/root_test.py delete mode 100644 tests/tree_test.py diff --git a/src/cqml/root.py b/src/cqml/root.py index 28a10d0..63322d0 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -2,7 +2,7 @@ class Root: def __init__(self, root): - self.nodes = [] + self.pipes = {} self.scan(root) def scan(self, root): @@ -10,18 +10,18 @@ def scan(self, root): self.parse(entry, root) def parse(self, entry, folder): - if entry.name.endswith(".yml"): - key = os.path.splitext(entry.name)[0] + name = entry.name + if name.endswith(".yml"): + file_key = os.path.splitext(name)[0] + key = f"{folder}/{file_key}" yml = read_yaml(entry.path) - node = { - "file":entry.name, - "folder":folder, + yml["meta"] = { + "file": name, + "file_key": file_key, + "folder": folder, "path": entry.path, - "key": key, - "fkey": f"{folder}.{key}", - "yml": yml } - self.nodes.append(node) + self.nodes[key] = yml elif entry.is_dir(): print(entry.name) - self.scan(entry.name) + self.scan(entry.path) diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py index 8f9bf2b..f9b1887 100644 --- a/src/cqml/wrappers.py +++ b/src/cqml/wrappers.py @@ -4,6 +4,7 @@ from .db2quilt import cvm2pkg, extract_pkg from .cvm import CVM from .yml import * +from .root import Root class CQML(CVM): def __init__(self, yaml_data, spark): @@ -25,6 +26,9 @@ def do_save(self, action): pkg = cvm2pkg(self, False) # do not re-run return pkg +def root(folder="pipes"): + return Root(folder) + def from_file(yaml_file, spark): print("Loading "+yaml_file) with open(yaml_file) as data: diff --git a/tests/root_test.py b/tests/root_test.py new file mode 100644 index 0000000..14f9038 --- /dev/null +++ b/tests/root_test.py @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +import pytest +from .context import cqml, TEST_YAML +from .db_mock import spark + +def test_root(): + root = cqml.root("pipes") + assert root + #assert "cqml" in keys diff --git a/tests/tree_test.py b/tests/tree_test.py deleted file mode 100644 index 6b6f975..0000000 --- a/tests/tree_test.py +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env python3 -import pytest -from .context import cqml, TEST_YAML -from .db_mock import spark - -def test_yml_keys(): - keys = cqml.yml_keys("pipes/test") - assert keys - assert "cqml" in keys - -def test_yml_tree(): - rows = cqml.yml_tree("pipes") - assert rows - assert len(rows) == 2 - d0 = rows[0] - assert "key" in d0 - From 5666c8d652bcea3efa82c774aff70c448e06cea2 Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 13 May 2022 20:12:02 -0700 Subject: [PATCH 022/107] Root.keys --- src/cqml/root.py | 9 ++++++++- src/cqml/yml.py | 9 --------- tests/root_test.py | 2 ++ 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/cqml/root.py b/src/cqml/root.py index 63322d0..969f16b 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -1,10 +1,17 @@ import os, yaml +def read_yaml(yaml_file): + with open(yaml_file) as data: + raw_yaml = yaml.full_load(data) + return raw_yaml + class Root: def __init__(self, root): self.pipes = {} self.scan(root) + def keys(self): return self.pipes.keys() + def scan(self, root): for entry in os.scandir(root): self.parse(entry, root) @@ -21,7 +28,7 @@ def parse(self, entry, folder): "folder": folder, "path": entry.path, } - self.nodes[key] = yml + self.pipes[key] = yml elif entry.is_dir(): print(entry.name) self.scan(entry.path) diff --git a/src/cqml/yml.py b/src/cqml/yml.py index 467609d..8c728dd 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -24,15 +24,6 @@ def yml_tree(folder="pipes"): keys = extract(folder) return keys -def read_yaml(yaml_file): - with open(yaml_file) as data: - raw_yaml = yaml.full_load(data) - return raw_yaml - -def write_yaml(yaml_file, raw_yaml): - with open(yaml_file, 'w') as file: - yaml.dump(raw_yaml, file, sort_keys=False) - def yml_folder(folder, nodes): print(folder.name) for entry in os.scandir(folder.path): diff --git a/tests/root_test.py b/tests/root_test.py index 14f9038..62511ec 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -6,4 +6,6 @@ def test_root(): root = cqml.root("pipes") assert root + keys = root.keys() + assert keys #assert "cqml" in keys From 6c6f9ec43e27cf5fae7ededefc132f720d4c0b75 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 08:37:20 -0700 Subject: [PATCH 023/107] self.env --- src/cqml/root.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/cqml/root.py b/src/cqml/root.py index 969f16b..56b92d9 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -7,7 +7,9 @@ def read_yaml(yaml_file): class Root: def __init__(self, root): + self.root = root self.pipes = {} + self.env = {} self.scan(root) def keys(self): return self.pipes.keys() @@ -22,13 +24,16 @@ def parse(self, entry, folder): file_key = os.path.splitext(name)[0] key = f"{folder}/{file_key}" yml = read_yaml(entry.path) - yml["meta"] = { + yml["source"] = { "file": name, "file_key": file_key, "folder": folder, + "key": key, "path": entry.path, } self.pipes[key] = yml + if name.startswith("cqml"): + self.env[folder] = yml elif entry.is_dir(): print(entry.name) self.scan(entry.path) From 263ac549c1078bb5217af5acf60d66cf812f390b Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 08:42:59 -0700 Subject: [PATCH 024/107] cqml env --- pipes/cqml.yml | 21 +++++++++++++++++++++ src/cqml/root.py | 3 +-- 2 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 pipes/cqml.yml diff --git a/pipes/cqml.yml b/pipes/cqml.yml new file mode 100644 index 0000000..48154b3 --- /dev/null +++ b/pipes/cqml.yml @@ -0,0 +1,21 @@ +cqml: 0.6 +order: 0.0 +id: main +env: + org: nauto + s3.bucket: biz-databricks-root-prod-us + catalog: quilt + path: . +refresh_trips: + do: schedule + start: 0100 + folders: + - refresh + - fleet +core: + do: schedule + start: 0500 + folders: + - nauto + - user + - rnr diff --git a/src/cqml/root.py b/src/cqml/root.py index 56b92d9..bf56b2f 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -32,8 +32,7 @@ def parse(self, entry, folder): "path": entry.path, } self.pipes[key] = yml - if name.startswith("cqml"): - self.env[folder] = yml + if "env" in yml: self.env[folder] = yml["env"] elif entry.is_dir(): print(entry.name) self.scan(entry.path) From f235b0b64ec05ca0b5aa505bded73131bcba4388 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 10:34:47 -0700 Subject: [PATCH 025/107] folder_key --- src/cqml/root.py | 5 +++-- tests/root_test.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/src/cqml/root.py b/src/cqml/root.py index bf56b2f..e394842 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -12,7 +12,7 @@ def __init__(self, root): self.env = {} self.scan(root) - def keys(self): return self.pipes.keys() + def keys(self): return list(self.pipes.keys()) def scan(self, root): for entry in os.scandir(root): @@ -22,7 +22,8 @@ def parse(self, entry, folder): name = entry.name if name.endswith(".yml"): file_key = os.path.splitext(name)[0] - key = f"{folder}/{file_key}" + folder_key = folder.split("/")[-1] + key = f"{folder_key}/{file_key}" yml = read_yaml(entry.path) yml["source"] = { "file": name, diff --git a/tests/root_test.py b/tests/root_test.py index 62511ec..530eb06 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -8,4 +8,4 @@ def test_root(): assert root keys = root.keys() assert keys - #assert "cqml" in keys + assert "test/cqml" in keys From 9653e5a31acdb70af7ab5993ca5436723c897425 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 10:36:32 -0700 Subject: [PATCH 026/107] test_keys(root) --- tests/root_test.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/root_test.py b/tests/root_test.py index 530eb06..e09a473 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -3,9 +3,15 @@ from .context import cqml, TEST_YAML from .db_mock import spark -def test_root(): +@pytest.fixture +def root(): root = cqml.root("pipes") + return root + +def test_root(root): assert root + +def test_keys(root): keys = root.keys() assert keys assert "test/cqml" in keys From 089a7b2366e51b2ae5cd02c1ccf6225f52ad9d28 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 10:41:23 -0700 Subject: [PATCH 027/107] "0.4.6.dev2_"2022-05-14T17:41:23Z"" --- src/cqml/keys.py | 1 + tests/cqml_test_db.py.tmp | 3 ++- tests/root_test.py | 1 + version.txt | 2 +- version.txt.prev | 2 +- 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cqml/keys.py b/src/cqml/keys.py index c689436..3ea9121 100644 --- a/src/cqml/keys.py +++ b/src/cqml/keys.py @@ -9,6 +9,7 @@ TRACE=True cAlias='|' +kALL='_all_' kAbove='above' kAny='any' kArgs='args' diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index dc6f4ef..7a3f47d 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -17,7 +17,8 @@ import cqml CQML_ROOT="pipes" root = cqml.root(CQML_ROOT) -dbutils.widgets.dropdown("CONF", root.default_key(), root.keys()) +keys = root.keys() +dbutils.widgets.dropdown("CONF", keys[0], keys) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) # COMMAND ---------- diff --git a/tests/root_test.py b/tests/root_test.py index e09a473..718d3cf 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -14,4 +14,5 @@ def test_root(root): def test_keys(root): keys = root.keys() assert keys + assert keys[0] assert "test/cqml" in keys diff --git a/version.txt b/version.txt index 60da6e1..8cc13bd 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev1 +0.4.6.dev2 diff --git a/version.txt.prev b/version.txt.prev index 93ee737..60da6e1 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.6.dev0 +0.4.6.dev1 From f4407189262c9b3eda289e51754f17d89c7a6069 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 10:41:24 -0700 Subject: [PATCH 028/107] "post-regen-tests/cqml_test_db.py-0.4.6.dev2" --- tests/cqml_test_db.py | 55 ++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 1acfb50..158a811 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,35 +9,58 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.4.6.dev1 +!pip install cqml==0.4.6.dev2 import cqml # COMMAND ---------- -KEY="cqml_test" -cvm = cqml.load_cqml(KEY,spark, '.') -cvm.debug = True -cvm.run() -print(cvm.sizes) +CQML_ROOT="pipes" +root = cqml.root(CQML_ROOT) +keys = root.keys() +dbutils.widgets.dropdown("CONF", keys[0], keys) +dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) # COMMAND ---------- -K = list(cvm.df.keys()) -print(K) -def d(i): return cvm.df[K[i]] -def view(i): - print(K[i]) - d(i).show() -def values(i, col): return d(i).select(col).distinct().collect() -view(-1) +CONF=getArgument("CONF") +DEBUG=True if getArgument("DEBUG") == "DEBUG" else False +print(f'Parameters[{CONF}]DEBUG={DEBUG} {CF}') +cvm = root.new(spark, CONF, DEBUG) +print(cvm.status()) # COMMAND ---------- -cvm.do_save({}) -displayHTML(cvm.pkg.html) +if not DEBUG: + cvm.run() + print(cvm.sizes) + dbutils.notebook.exit(0) # COMMAND ---------- + +cvm.init() +steps = cvm.steps() +print(steps) +#dbutils.notebook.exit(0) + + +# COMMAND ---------- + +df = {} +def values(s, col): return df[s].select(col).distinct().collect() + +for step in steps: + print(step) + df[step] = cvm.test(step) + df[step].show() + +# COMMAND ---------- + +cvm.save() +displayHTML(cvm.result()) + +# COMMAND ---------- + dbutils.notebook.exit(0) #spark.sql('create database nauto') From 2f8e94ecb21c236496ecd62c369729e96b1e6594 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 10:52:29 -0700 Subject: [PATCH 029/107] test_new(root) --- src/cqml/keys.py | 1 + src/cqml/root.py | 5 ++++- tests/cqml_test_db.py.tmp | 2 +- tests/root_test.py | 7 ++++++- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/src/cqml/keys.py b/src/cqml/keys.py index 3ea9121..962868b 100644 --- a/src/cqml/keys.py +++ b/src/cqml/keys.py @@ -18,6 +18,7 @@ kCount='count' kDoc='+doc' kDrop='drop' +kEnv='env' kExt='file_ext' kFunc='function' kGroup='group' diff --git a/src/cqml/root.py b/src/cqml/root.py index e394842..c5e5db1 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -1,4 +1,5 @@ import os, yaml +from .keys import * def read_yaml(yaml_file): with open(yaml_file) as data: @@ -14,6 +15,8 @@ def __init__(self, root): def keys(self): return list(self.pipes.keys()) + def new(self, spark, key, debug=False): return self + def scan(self, root): for entry in os.scandir(root): self.parse(entry, root) @@ -33,7 +36,7 @@ def parse(self, entry, folder): "path": entry.path, } self.pipes[key] = yml - if "env" in yml: self.env[folder] = yml["env"] + if kEnv in yml: self.env[folder] = yml[kEnv] elif entry.is_dir(): print(entry.name) self.scan(entry.path) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index 7a3f47d..6e4d76f 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -15,7 +15,7 @@ import cqml # COMMAND ---------- -CQML_ROOT="pipes" +CQML_ROOT="../pipes" root = cqml.root(CQML_ROOT) keys = root.keys() dbutils.widgets.dropdown("CONF", keys[0], keys) diff --git a/tests/root_test.py b/tests/root_test.py index 718d3cf..3b2fbd1 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -3,6 +3,8 @@ from .context import cqml, TEST_YAML from .db_mock import spark +T_KEY="test/cqml" + @pytest.fixture def root(): root = cqml.root("pipes") @@ -15,4 +17,7 @@ def test_keys(root): keys = root.keys() assert keys assert keys[0] - assert "test/cqml" in keys + assert T_KEY in keys + +def test_new(root): + cvm = root.new(spark, T_KEY) From 5bf3c1b336344085d389f6ef104164b9213afc09 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 14:13:44 -0700 Subject: [PATCH 030/107] new CVM --- src/cqml/root.py | 7 ++++++- src/cqml/vm.py | 1 + tests/root_test.py | 2 ++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/cqml/root.py b/src/cqml/root.py index c5e5db1..3732584 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -1,5 +1,6 @@ import os, yaml from .keys import * +from .cvm import CVM def read_yaml(yaml_file): with open(yaml_file) as data: @@ -15,7 +16,11 @@ def __init__(self, root): def keys(self): return list(self.pipes.keys()) - def new(self, spark, key, debug=False): return self + def new(self, spark, key, debug=False): + pipe = self.pipes[key] + cvm = CVM(pipe, spark) + if debug: cvm.debug = True + return cvm def scan(self, root): for entry in os.scandir(root): diff --git a/src/cqml/vm.py b/src/cqml/vm.py index f3496fb..77b930f 100644 --- a/src/cqml/vm.py +++ b/src/cqml/vm.py @@ -73,6 +73,7 @@ def log(self, str, name=False): if self.debug: if name: print(name) print(str) + return str def macro(self, todo, action): mdef = todo.split("|") diff --git a/tests/root_test.py b/tests/root_test.py index 3b2fbd1..bc4a757 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -21,3 +21,5 @@ def test_keys(root): def test_new(root): cvm = root.new(spark, T_KEY) + assert cvm + assert "test" == cvm.log("test") From 40e5f8de3433e97ec90e7ae095144f28e1e88fcd Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 14:34:05 -0700 Subject: [PATCH 031/107] Remove superflous dependencies --- src/cqml/yml.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/cqml/yml.py b/src/cqml/yml.py index 8c728dd..bfca1f4 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -1,8 +1,4 @@ -import yaml -import os,shutil -from operator import itemgetter -from .db2quilt import cvm2pkg, extract_pkg -from .cvm import CVM +import os, yaml def upgrade_file(yaml_file): print("Upgrading "+yaml_file) From b46723375dab676dd1c37ec788b0cafaa4537d6f Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 15:35:29 -0700 Subject: [PATCH 032/107] self.set_env(pipe, key) --- src/cqml/db2quilt.py | 8 ++++++++ src/cqml/root.py | 15 ++++++++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index a47a9b3..6960b4b 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -133,6 +133,14 @@ def make_slug(name): return re.sub(r'[^\w-]', '_', name.lower()) Quilt Wrappers """ +DEFAULT_ENV={ + 'catalog': PKG_DIR, + 'root': PYROOT, +} + +def get_env(env, key, default): + return env[key] if key in env else default + class Project: def __init__(self, config): org, bucket, project = itemgetter('org','s3.bucket','project')(config) diff --git a/src/cqml/root.py b/src/cqml/root.py index 3732584..adf0139 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -16,8 +16,21 @@ def __init__(self, root): def keys(self): return list(self.pipes.keys()) + def add_env(self, yml, folder): + if kEnv in yml: self.env[folder] = yml[kEnv] + + def set_env(self, yml, key): + folder = yml["source"]["folder"] + env = {} + if self.root in self.env: env.update(self.env[self.root]) + if folder in self.env: env.update(self.env[folder]) + if kEnv in yml: env.update(yml[kEnv]) + yml[kEnv] = env + return env + def new(self, spark, key, debug=False): pipe = self.pipes[key] + self.set_env(pipe, key) cvm = CVM(pipe, spark) if debug: cvm.debug = True return cvm @@ -41,7 +54,7 @@ def parse(self, entry, folder): "path": entry.path, } self.pipes[key] = yml - if kEnv in yml: self.env[folder] = yml[kEnv] + self.add_env(yml, folder) elif entry.is_dir(): print(entry.name) self.scan(entry.path) From 117c3b6a40b37a8d1880e8df570dae7dd5927f57 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:32:34 -0700 Subject: [PATCH 033/107] bucket config --- pipes/cqml.yml | 2 +- pipes/demo/demo.yml | 2 +- pipes/test/cqml.yml | 2 +- src/cqml/db2quilt.py | 2 +- src/cqml/root.py | 10 ++++++---- tests/cqml_test.yml | 2 +- tests/demo.yml | 2 +- 7 files changed, 12 insertions(+), 10 deletions(-) diff --git a/pipes/cqml.yml b/pipes/cqml.yml index 48154b3..a361399 100644 --- a/pipes/cqml.yml +++ b/pipes/cqml.yml @@ -3,7 +3,7 @@ order: 0.0 id: main env: org: nauto - s3.bucket: biz-databricks-root-prod-us + bucket: biz-databricks-root-prod-us catalog: quilt path: . refresh_trips: diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml index 3fdb1f6..6d61f63 100644 --- a/pipes/demo/demo.yml +++ b/pipes/demo/demo.yml @@ -3,7 +3,7 @@ id: cqml_demo meta: org: nauto project: sangam - s3.bucket: biz-databricks-root-prod-us + bucket: biz-databricks-root-prod-us catalog: quilt root: /dbfs/tmp actions: diff --git a/pipes/test/cqml.yml b/pipes/test/cqml.yml index e91dbca..6fe193e 100644 --- a/pipes/test/cqml.yml +++ b/pipes/test/cqml.yml @@ -3,7 +3,7 @@ id: cqml_test meta: org: nauto project: sangam - s3.bucket: biz-databricks-root-prod-us + bucket: biz-databricks-root-prod-us catalog: quilt root: /dbfs/tmp actions: diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index 6960b4b..e1fc09e 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -143,7 +143,7 @@ def get_env(env, key, default): class Project: def __init__(self, config): - org, bucket, project = itemgetter('org','s3.bucket','project')(config) + org, bucket, project = itemgetter('org','bucket','project')(config) pkg_dir = config['catalog'] if 'catalog' in config else PKG_DIR root = config['root'] if 'root' in config else PYROOT self.repo = "s3://"+bucket diff --git a/src/cqml/root.py b/src/cqml/root.py index adf0139..a54670f 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -16,8 +16,10 @@ def __init__(self, root): def keys(self): return list(self.pipes.keys()) - def add_env(self, yml, folder): - if kEnv in yml: self.env[folder] = yml[kEnv] + def add_env(self, yml, key): + if not kEnv in yml: return {} + self.env[key] = yml[kEnv] + yml[kEnv] def set_env(self, yml, key): folder = yml["source"]["folder"] @@ -42,10 +44,11 @@ def scan(self, root): def parse(self, entry, folder): name = entry.name if name.endswith(".yml"): + yml = read_yaml(entry.path) file_key = os.path.splitext(name)[0] folder_key = folder.split("/")[-1] + env = self.add_env(yml, folder_key) key = f"{folder_key}/{file_key}" - yml = read_yaml(entry.path) yml["source"] = { "file": name, "file_key": file_key, @@ -54,7 +57,6 @@ def parse(self, entry, folder): "path": entry.path, } self.pipes[key] = yml - self.add_env(yml, folder) elif entry.is_dir(): print(entry.name) self.scan(entry.path) diff --git a/tests/cqml_test.yml b/tests/cqml_test.yml index e91dbca..6fe193e 100644 --- a/tests/cqml_test.yml +++ b/tests/cqml_test.yml @@ -3,7 +3,7 @@ id: cqml_test meta: org: nauto project: sangam - s3.bucket: biz-databricks-root-prod-us + bucket: biz-databricks-root-prod-us catalog: quilt root: /dbfs/tmp actions: diff --git a/tests/demo.yml b/tests/demo.yml index 3fdb1f6..6d61f63 100644 --- a/tests/demo.yml +++ b/tests/demo.yml @@ -3,7 +3,7 @@ id: cqml_demo meta: org: nauto project: sangam - s3.bucket: biz-databricks-root-prod-us + bucket: biz-databricks-root-prod-us catalog: quilt root: /dbfs/tmp actions: From bfcbec7437af5db654f42b752d369ebedee09555 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:38:48 -0700 Subject: [PATCH 034/107] env.update(source) --- pipes/cqml.yml | 1 + pipes/demo/demo.yml | 8 +------- pipes/test/cqml.yml | 6 ------ src/cqml/root.py | 10 ++++++---- 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/pipes/cqml.yml b/pipes/cqml.yml index a361399..574f909 100644 --- a/pipes/cqml.yml +++ b/pipes/cqml.yml @@ -5,6 +5,7 @@ env: org: nauto bucket: biz-databricks-root-prod-us catalog: quilt + root: /dbfs/tmp path: . refresh_trips: do: schedule diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml index 6d61f63..4fce33a 100644 --- a/pipes/demo/demo.yml +++ b/pipes/demo/demo.yml @@ -1,11 +1,5 @@ -cqml: 0.4.4 +cqml: 0.5.0 id: cqml_demo -meta: - org: nauto - project: sangam - bucket: biz-databricks-root-prod-us - catalog: quilt - root: /dbfs/tmp actions: /Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data: do: loadfiles diff --git a/pipes/test/cqml.yml b/pipes/test/cqml.yml index 6fe193e..fb41d9b 100644 --- a/pipes/test/cqml.yml +++ b/pipes/test/cqml.yml @@ -1,11 +1,5 @@ cqml: 0.2 id: cqml_test -meta: - org: nauto - project: sangam - bucket: biz-databricks-root-prod-us - catalog: quilt - root: /dbfs/tmp actions: /Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data: do: loadfiles diff --git a/src/cqml/root.py b/src/cqml/root.py index a54670f..e3df552 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -19,10 +19,10 @@ def keys(self): return list(self.pipes.keys()) def add_env(self, yml, key): if not kEnv in yml: return {} self.env[key] = yml[kEnv] - yml[kEnv] + return yml[kEnv] def set_env(self, yml, key): - folder = yml["source"]["folder"] + folder = yml[kEnv]["folder"] env = {} if self.root in self.env: env.update(self.env[self.root]) if folder in self.env: env.update(self.env[folder]) @@ -49,13 +49,15 @@ def parse(self, entry, folder): folder_key = folder.split("/")[-1] env = self.add_env(yml, folder_key) key = f"{folder_key}/{file_key}" - yml["source"] = { + source = { "file": name, "file_key": file_key, "folder": folder, "key": key, "path": entry.path, - } + } + env.update(source) + yml[kEnv] = env self.pipes[key] = yml elif entry.is_dir(): print(entry.name) From 5f6389ccb5bf663e8180400786b254f0b6739fdc Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:42:48 -0700 Subject: [PATCH 035/107] project/package --- archive-v45/cqml45.py | 4 ++-- src/cqml/root.py | 6 +++--- src/cqml/yml.py | 2 +- tests/db_mock.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py index 6b4536d..8348c78 100755 --- a/archive-v45/cqml45.py +++ b/archive-v45/cqml45.py @@ -29,12 +29,12 @@ def extract(root): for file in os.scandir(folder.path): if file.name.endswith(".yml"): yml = read_yaml(file.path) - node = {"file":file.name, "folder":folder.name, "yml": yml} + node = {"file":file.name,"project":folder.name, "yml": yml} tree.append(node) return tree def convert(root, node): - dir = node["folder"] + dir = node["project"] file = ''.join([c for c in node["file"] if not c.isdigit()]) prefix = file.split("_")[0] name = file.split("_")[1] diff --git a/src/cqml/root.py b/src/cqml/root.py index e3df552..fcfa5d4 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -22,7 +22,7 @@ def add_env(self, yml, key): return yml[kEnv] def set_env(self, yml, key): - folder = yml[kEnv]["folder"] + folder = yml[kEnv]["project"] env = {} if self.root in self.env: env.update(self.env[self.root]) if folder in self.env: env.update(self.env[folder]) @@ -51,8 +51,8 @@ def parse(self, entry, folder): key = f"{folder_key}/{file_key}" source = { "file": name, - "file_key": file_key, - "folder": folder, + "package": file_key, + "project": folder, "key": key, "path": entry.path, } diff --git a/src/cqml/yml.py b/src/cqml/yml.py index bfca1f4..52f4a46 100644 --- a/src/cqml/yml.py +++ b/src/cqml/yml.py @@ -26,7 +26,7 @@ def yml_folder(folder, nodes): if entry.name.endswith(".yml"): key = os.path.splitext(entry.name)[0] #yml = read_yaml(entry.path) - node = {"file":entry.name, "folder":folder.name, "path": entry.path, "key": key} + node = {"file":entry.name,"project":folder.name, "path": entry.path, "key": key} nodes.append(node) elif entry.is_dir(): print(folder.name) diff --git a/tests/db_mock.py b/tests/db_mock.py index 66db241..5ea5fe2 100644 --- a/tests/db_mock.py +++ b/tests/db_mock.py @@ -10,7 +10,7 @@ def get_items(self): return [self] class MockCol(object): def __init__(self, name): self.name = name - def alias(self, name,metadata={"meta":"data"}): return name + def alias(self, name,metadata={"env":"data"}): return name def contains(self, value): return True def desc(self): return True From b0ed477eb0ed3253c4d5c376dd39b11708d3d590 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:46:38 -0700 Subject: [PATCH 036/107] meta -> env --- src/cqml/db2quilt.py | 4 +++- tests/cqml_test.yml | 5 +++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index e1fc09e..a66e864 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -301,8 +301,10 @@ def write_summary(self): # def extract_pkg(cvm): - id, config = itemgetter('id','meta')(cvm.yaml) + print(cvm.yaml) + config = itemgetter(kEnv)(cvm.yaml) proj = Project(config) + id = config["package"] pkg_id = id + DEBUG_SUFFIX if cvm.debug == True else id print("extract_pkg: "+pkg_id) pkg = proj.package(pkg_id) diff --git a/tests/cqml_test.yml b/tests/cqml_test.yml index 6fe193e..9979dfd 100644 --- a/tests/cqml_test.yml +++ b/tests/cqml_test.yml @@ -1,8 +1,9 @@ -cqml: 0.2 +cqml: 0.5 id: cqml_test -meta: +env: org: nauto project: sangam + package: cqml bucket: biz-databricks-root-prod-us catalog: quilt root: /dbfs/tmp From 9532f2e36e1b080edaa96864e424a4e5654f2cec Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:53:30 -0700 Subject: [PATCH 037/107] from .root import Root --- src/cqml/__init__.py | 1 + src/cqml/root.py | 4 ++-- src/cqml/wrappers.py | 1 - tests/cqml_test.py | 3 ++- tests/root_test.py | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/cqml/__init__.py b/src/cqml/__init__.py index 4df6dec..9c80fe4 100644 --- a/src/cqml/__init__.py +++ b/src/cqml/__init__.py @@ -6,3 +6,4 @@ from .yml import * from .wrappers import * +from .root import Root diff --git a/src/cqml/root.py b/src/cqml/root.py index fcfa5d4..45d76ac 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -1,6 +1,6 @@ import os, yaml from .keys import * -from .cvm import CVM +from .wrappers import CQML def read_yaml(yaml_file): with open(yaml_file) as data: @@ -33,7 +33,7 @@ def set_env(self, yml, key): def new(self, spark, key, debug=False): pipe = self.pipes[key] self.set_env(pipe, key) - cvm = CVM(pipe, spark) + cvm = CQML(pipe, spark) if debug: cvm.debug = True return cvm diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py index f9b1887..5450b80 100644 --- a/src/cqml/wrappers.py +++ b/src/cqml/wrappers.py @@ -4,7 +4,6 @@ from .db2quilt import cvm2pkg, extract_pkg from .cvm import CVM from .yml import * -from .root import Root class CQML(CVM): def __init__(self, yaml_data, spark): diff --git a/tests/cqml_test.py b/tests/cqml_test.py index 18de986..2a7f781 100644 --- a/tests/cqml_test.py +++ b/tests/cqml_test.py @@ -5,7 +5,8 @@ @pytest.fixture def cvm(): - cvm = cqml.from_file(TEST_YAML, spark) + root = cqml.Root("pipes") + cvm = root.new(spark, "test/cqml") cvm.test_id(DDIR) return cvm diff --git a/tests/root_test.py b/tests/root_test.py index bc4a757..ee96f89 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -7,7 +7,7 @@ @pytest.fixture def root(): - root = cqml.root("pipes") + root = cqml.Root("pipes") return root def test_root(root): From a0f49ce6695434430f5f959f68085b7d3cbe4294 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:54:30 -0700 Subject: [PATCH 038/107] remove un-rooted yml --- src/cqml/wrappers.py | 3 - tests/cqml_test.yml | 228 ------------------------------------------- tests/demo.yml | 142 --------------------------- 3 files changed, 373 deletions(-) delete mode 100644 tests/cqml_test.yml delete mode 100644 tests/demo.yml diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py index 5450b80..8f9bf2b 100644 --- a/src/cqml/wrappers.py +++ b/src/cqml/wrappers.py @@ -25,9 +25,6 @@ def do_save(self, action): pkg = cvm2pkg(self, False) # do not re-run return pkg -def root(folder="pipes"): - return Root(folder) - def from_file(yaml_file, spark): print("Loading "+yaml_file) with open(yaml_file) as data: diff --git a/tests/cqml_test.yml b/tests/cqml_test.yml deleted file mode 100644 index 9979dfd..0000000 --- a/tests/cqml_test.yml +++ /dev/null @@ -1,228 +0,0 @@ -cqml: 0.5 -id: cqml_test -env: - org: nauto - project: sangam - package: cqml - bucket: biz-databricks-root-prod-us - catalog: quilt - root: /dbfs/tmp -actions: - /Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data: - do: loadfiles - as_number: devices - tables: - test1: test1.csv - test2: test2.csv - today: - do: call - from: test1 - sort: dat - ensure_current: today - function: current_date - args: [] - unique_num: - do: unique - from: test1 - sort: dat - save: grid - ensure_unique: num - cols: - num: tbd - count: [letter] - box_details: - do: box - from: test1 - group: num - +doc: Split by num, upload, store URLs - box: - root_folder: CQML_Test - root_id: 156760265584 - data_dir: test - file_ext: csv - expiration_date: '2200-02-02' - cols: - letter: sort - dat: sort - widget-report: - do: report - from: test1 - cols: - text: Combobox -# num: slider - grouped: - do: group - from: test1 - agg: - num: sum - dat: max - sort: max_dat - cols: - text: tbd - letter: tbd - selected: - do: select - from: test1 - where: - letter: - A: contains - B: contains - text: - null: is_not - cols: - num: tbd - dat: tbd - letter: tbd - text: tbd - merged: - do: merge - #save: grid - into: test1 - join: num - from: test2 - join_type: inner - where: - letter: - A: contains - cols: - num: tbd - letter|better: tbd - dat|dot: tbd - rename_merged: - do: merge - into: test1 - join: num - from: test2 - join_type: inner - where: - letter: - A: contains - cols: - num|next: tbd - letter|better: tbd - dat|dot: tbd - plus_operator: - do: call - from: rename_merged - operator: + - args: - - next - - num - calc_quarters: - do: calc - from: plus_operator - operator: / - round: 3 - else: 0 - args: - - $col - - 4 - cols: - num: qnum - next: qnumb - plus_operator: qdumb - flagged: - do: flag - from: test1 - where: - text: - null: is - count_days: - do: call - from: test1 - function: datediff - args: - - current_date() - - dat - check_count: - do: flag - from: $id - where: - count_days: - 1: lesser - union: - do: union - into: test1 - from: test2 - drop: - - note - cqml_test_inner_dupes: - do: merge - save: table - into: test1 - join: num - join_type: inner - from: test2 - keep_join: left - cols: - num: tbd - letter|character: tbd - macro_def: - do: macro|merge - into: "{into}" - join: "{join}" - from: test2 - join_type: inner - cols: - "{join}": tbd - num|numerate: tbd - text|texas: tbd - dat|database: tbd - macro_run: - do: macro.macro_def - into: test1 - join: letter - char: A - mult_number: - do: assign - from: macro_run - pivoted: - do: pivot - skip: true - from: $id - agg: - sum_num: count - sort: text - pivot: letter - cols: - text: tbd - cqml_test_summarize: - do: summary - save: [grid, parquet] - from: test1 - count: num - as: total - cols: - letter: tbd - text: tbd - call_coalesce: - do: call - from: test1 - function: coalesce - args: - - text - - "'Unassigned'" - concat_space: - do: call - from: $id - operator: '||' - args: - - num - - "' '" - - letter - aggregates: - do: group - from: $id - agg: - concat_space: count - call_coalesce: sum - sort: n_concat_space - cols: - num: tbd - letter: tbd - leftsemi_match: - do: select - from: test1 - matching: - num: - test2: num diff --git a/tests/demo.yml b/tests/demo.yml deleted file mode 100644 index 6d61f63..0000000 --- a/tests/demo.yml +++ /dev/null @@ -1,142 +0,0 @@ -cqml: 0.4.4 -id: cqml_demo -meta: - org: nauto - project: sangam - bucket: biz-databricks-root-prod-us - catalog: quilt - root: /dbfs/tmp -actions: - /Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data: - do: loadfiles - as_number: devices - tables: {test1: test1.csv, test2: test2.csv} - selected: - +doc: simple select query - do: select - from: test1 - where: - letter: {A: contains, B: contains} - text: {null: is_not} - cols: {num: tbd, dat: tbd} - merged: - +doc: join and save into a datagrid - do: merge - save: grid - into: $id - join: num - from: test2 - join_type: inner - cols: - num: join - letter|better: tbd - dat|dot: tbd - box_export: - +doc: Split by num, upload, store URLs - do: box - from: $id - group: num - box: - root_folder: CQML_Test - root_id: 156760265584 - data_dir: test - file_ext: csv - expiration_date: '2200-02-02' - cols: - letter: sort - dat: sort - widget-report: - do: report - from: test1 - cols: - text: Combobox - grouped: - do: group - from: test1 - agg: - num: sum - dat: max - sort: max_dat - cols: - text: tbd - letter: tbd - plus_operator: - do: call - from: grouped - operator: + - args: - - next - - num - calc_quarters: - do: calc - from: plus_operator - operator: / - round: 3 - else: 0 - args: - - $col - - 4 - cols: - num: qnum - next: qnumb - plus_operator: qdumb - flagged: - do: flag - from: test1 - where: - text: - null: is - union: - do: union - into: test1 - from: test2 - drop: - - note - macro_def: - do: macro|merge - into: "{into}" - join: "{join}" - from: test2 - join_type: inner - cols: - "{join}": tbd - num|numerate: tbd - text|texas: tbd - dat|database: tbd - macro_run: - do: macro.macro_def - into: test1 - join: letter - char: A - rename: - do: assign - from: macro_run - pivoted: - do: pivot - skip: true - from: $id - agg: - sum_num: count - sort: text - pivot: letter - cols: - text: tbd - cqml_test_summarize: - do: summary - save: [grid, parquet] - from: test1 - count: num - as: total - cols: - letter: tbd - text: tbd - aggregates: - do: group - from: $id - agg: - concat_space: count - call_coalesce: sum - sort: n_concat_space - cols: - num: tbd - letter: tbd From 5d73e56f3137f985bef559e6ef9a6ae6c763f166 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:56:30 -0700 Subject: [PATCH 039/107] TEST_KEY --- tests/context.py | 2 +- tests/cqml_test.py | 4 ++-- tests/root_test.py | 8 +++----- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/context.py b/tests/context.py index 3b1a237..440cc20 100644 --- a/tests/context.py +++ b/tests/context.py @@ -1,7 +1,7 @@ import os import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '.','./src','..','../src'))) -TEST_YAML="tests/cqml_test.yml" +TEST_KEY="test/cqml" DDIR="/Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data" import cqml diff --git a/tests/cqml_test.py b/tests/cqml_test.py index 2a7f781..4522308 100644 --- a/tests/cqml_test.py +++ b/tests/cqml_test.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 import pytest -from .context import cqml, TEST_YAML, DDIR +from .context import cqml, TEST_KEY, DDIR from .db_mock import spark @pytest.fixture def cvm(): root = cqml.Root("pipes") - cvm = root.new(spark, "test/cqml") + cvm = root.new(spark, TEST_KEY) cvm.test_id(DDIR) return cvm diff --git a/tests/root_test.py b/tests/root_test.py index ee96f89..683ab38 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -1,10 +1,8 @@ #!/usr/bin/env python3 import pytest -from .context import cqml, TEST_YAML +from .context import cqml, TEST_KEY from .db_mock import spark -T_KEY="test/cqml" - @pytest.fixture def root(): root = cqml.Root("pipes") @@ -17,9 +15,9 @@ def test_keys(root): keys = root.keys() assert keys assert keys[0] - assert T_KEY in keys + assert TEST_KEY in keys def test_new(root): - cvm = root.new(spark, T_KEY) + cvm = root.new(spark, TEST_KEY) assert cvm assert "test" == cvm.log("test") From a6c5e159dd2aece11f666497b54ac770bf46a81c Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:57:16 -0700 Subject: [PATCH 040/107] "0.4.6.dev3_"2022-05-15T03:57:16Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 8cc13bd..2e27d84 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev2 +0.4.6.dev3 diff --git a/version.txt.prev b/version.txt.prev index 60da6e1..8cc13bd 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.6.dev1 +0.4.6.dev2 From b86734ffd098acc79d27f6a13d66fdac5fc4bdeb Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 20:57:17 -0700 Subject: [PATCH 041/107] "post-regen-tests/cqml_test_db.py-0.4.6.dev3" --- tests/cqml_test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 158a811..05d672f 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,13 +9,13 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.4.6.dev2 +!pip install cqml==0.4.6.dev3 import cqml # COMMAND ---------- -CQML_ROOT="pipes" +CQML_ROOT="../pipes" root = cqml.root(CQML_ROOT) keys = root.keys() dbutils.widgets.dropdown("CONF", keys[0], keys) From f7a82f7c4fa7fbe772cfeb6e05f6e0c04728e800 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 21:12:34 -0700 Subject: [PATCH 042/107] create package correctly --- src/cqml/db2quilt.py | 5 +++-- src/cqml/root.py | 9 +++++++-- src/cqml/wrappers.py | 23 +++++++++++++---------- tests/pkg_test.py | 18 ++++++++++++++++++ tests/pkg_test_skip.py | 14 -------------- 5 files changed, 41 insertions(+), 28 deletions(-) create mode 100644 tests/pkg_test.py delete mode 100644 tests/pkg_test_skip.py diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index a66e864..491626c 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -155,7 +155,7 @@ def package(self, id): return Package(id, self) class Package: - def __init__(self, id, proj, reset=False): + def __init__(self, id, proj, reset=True): self.id = id self.name = f"{proj.name}/{id}" self.proj = proj @@ -301,8 +301,9 @@ def write_summary(self): # def extract_pkg(cvm): - print(cvm.yaml) +# print(cvm.yaml) config = itemgetter(kEnv)(cvm.yaml) + print(f"extract_pkg.config: {config}") proj = Project(config) id = config["package"] pkg_id = id + DEBUG_SUFFIX if cvm.debug == True else id diff --git a/src/cqml/root.py b/src/cqml/root.py index 45d76ac..556cd7c 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -1,6 +1,6 @@ import os, yaml from .keys import * -from .wrappers import CQML +from .wrappers import CQML, pkg_cvm def read_yaml(yaml_file): with open(yaml_file) as data: @@ -37,6 +37,11 @@ def new(self, spark, key, debug=False): if debug: cvm.debug = True return cvm + def pkg(self, spark, key, debug=False): + cvm = self.new(spark, key, debug) + cvm.run(); + return pkg_cvm(cvm) + def scan(self, root): for entry in os.scandir(root): self.parse(entry, root) @@ -52,7 +57,7 @@ def parse(self, entry, folder): source = { "file": name, "package": file_key, - "project": folder, + "project": folder_key, "key": key, "path": entry.path, } diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py index 8f9bf2b..57c3b4e 100644 --- a/src/cqml/wrappers.py +++ b/src/cqml/wrappers.py @@ -51,20 +51,23 @@ def exec_cqml(name, spark, folder="pipes"): cvm.run() return cvm -def pkg_cqml(name, spark, folder="pipes"): - print("\npkg_cqml: "+name) - cvm = exec_cqml(name, spark, folder) +def pkg_cvm(cvm): pkg = cvm2pkg(cvm, False) return { - 'pkg': pkg, - 'html': pkg.html, - 'url': pkg.url, - 'actions': cvm.actions, - 'sizes': cvm.sizes, - 'times': cvm.times, - 'frames': cvm.df, + 'pkg': pkg, + 'html': pkg.html, + 'url': pkg.url, + 'actions': cvm.actions, + 'sizes': cvm.sizes, + 'times': cvm.times, + 'frames': cvm.df, } +def pkg_cqml(name, spark, folder="pipes"): + print("\npkg_cqml: "+name) + cvm = exec_cqml(name, spark, folder) + return pkg_cvm(cvm) + def pkg_all(spark, folder="pipes"): keys = yml_keys(folder) pkgs = {key:pkg_cqml(key, spark, folder) for key in keys} diff --git a/tests/pkg_test.py b/tests/pkg_test.py new file mode 100644 index 0000000..f99e548 --- /dev/null +++ b/tests/pkg_test.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python3 +import pytest +from .context import cqml, TEST_KEY +from .db_mock import spark + +@pytest.fixture +def root(): + root = cqml.Root("pipes") + return root + +def test_pkg(root): + dict = root.pkg(spark, TEST_KEY, True) + assert 'pkg' in dict + assert 'html' in dict + assert 'actions' in dict + +#def test_all(): dict = cqml.pkg_all(spark, 'tests') + #assert 'cqml_test' in dict diff --git a/tests/pkg_test_skip.py b/tests/pkg_test_skip.py deleted file mode 100644 index 3e74018..0000000 --- a/tests/pkg_test_skip.py +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env python3 -import pytest -from .context import cqml, TEST_YAML -from .db_mock import spark - -def test_pkg(): - dict = cqml.pkg_cqml('cqml_test', spark, 'tests') - assert 'pkg' in dict - assert 'html' in dict - assert 'actions' in dict - -def test_all(): - dict = cqml.pkg_all(spark, 'tests') - assert 'cqml_test' in dict From e3a9bbeae1924fee23b3009144b8e03b19908e5a Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 21:21:12 -0700 Subject: [PATCH 043/107] "0.4.6.dev4_"2022-05-15T04:21:12Z"" --- src/cqml/db2quilt.py | 4 ++-- src/cqml/root.py | 4 ++++ tests/cqml_test_db.py.tmp | 2 +- tests/pkg_test.py | 12 +++++++++--- version.txt | 2 +- version.txt.prev | 2 +- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index 491626c..bd93abe 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -303,11 +303,11 @@ def write_summary(self): def extract_pkg(cvm): # print(cvm.yaml) config = itemgetter(kEnv)(cvm.yaml) - print(f"extract_pkg.config: {config}") + #print(f"extract_pkg.config: {config}") proj = Project(config) id = config["package"] pkg_id = id + DEBUG_SUFFIX if cvm.debug == True else id - print("extract_pkg: "+pkg_id) + #print("extract_pkg: "+pkg_id) pkg = proj.package(pkg_id) #pkg.setup() return pkg diff --git a/src/cqml/root.py b/src/cqml/root.py index 556cd7c..a9815fd 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -42,6 +42,10 @@ def pkg(self, spark, key, debug=False): cvm.run(); return pkg_cvm(cvm) + def pkg_all(self, spark, debug=False): + pkgs = {key:self.pkg(spark, key, debug) for key in self.keys()} + return pkgs + def scan(self, root): for entry in os.scandir(root): self.parse(entry, root) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index 6e4d76f..fb6d133 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -16,7 +16,7 @@ import cqml # COMMAND ---------- CQML_ROOT="../pipes" -root = cqml.root(CQML_ROOT) +root = cqml.Root(CQML_ROOT) keys = root.keys() dbutils.widgets.dropdown("CONF", keys[0], keys) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) diff --git a/tests/pkg_test.py b/tests/pkg_test.py index f99e548..1b6e0fd 100644 --- a/tests/pkg_test.py +++ b/tests/pkg_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import pytest -from .context import cqml, TEST_KEY +from .context import cqml, TEST_KEY, DDIR from .db_mock import spark @pytest.fixture @@ -14,5 +14,11 @@ def test_pkg(root): assert 'html' in dict assert 'actions' in dict -#def test_all(): dict = cqml.pkg_all(spark, 'tests') - #assert 'cqml_test' in dict +def test_box(root): + cvm = root.new(spark, TEST_KEY) + cvm.test_id(DDIR) + it = cvm.test_id("box_details") + assert it + +#def test_all(root): dict = root.pkg_all(spark, True) + #assert 'cqml' in dict diff --git a/version.txt b/version.txt index 2e27d84..069416b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev3 +0.4.6.dev4 diff --git a/version.txt.prev b/version.txt.prev index 8cc13bd..2e27d84 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.6.dev2 +0.4.6.dev3 From f43f9b7fdf3bb822be82fbf30f02fe446cd5621c Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 21:21:13 -0700 Subject: [PATCH 044/107] "post-regen-tests/cqml_test_db.py-0.4.6.dev4" --- tests/cqml_test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 05d672f..2f701a9 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,14 +9,14 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.4.6.dev3 +!pip install cqml==0.4.6.dev4 import cqml # COMMAND ---------- CQML_ROOT="../pipes" -root = cqml.root(CQML_ROOT) +root = cqml.Root(CQML_ROOT) keys = root.keys() dbutils.widgets.dropdown("CONF", keys[0], keys) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) From 26dea05135e2feed12a84913e4c82efb1c2c4877 Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 21:23:05 -0700 Subject: [PATCH 045/107] "0.4.6.dev5_"2022-05-15T04:23:05Z"" --- tests/cqml_test_db.py.tmp | 2 +- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index fb6d133..32b714e 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -25,7 +25,7 @@ dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False -print(f'Parameters[{CONF}]DEBUG={DEBUG} {CF}') +print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) print(cvm.status()) diff --git a/version.txt b/version.txt index 069416b..6d94c1a 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev4 +0.4.6.dev5 diff --git a/version.txt.prev b/version.txt.prev index 2e27d84..069416b 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.6.dev3 +0.4.6.dev4 From 201d2f1a38e5ea2b7b1b129139908ccfb8bc101d Mon Sep 17 00:00:00 2001 From: drernie Date: Sat, 14 May 2022 21:23:06 -0700 Subject: [PATCH 046/107] "post-regen-tests/cqml_test_db.py-0.4.6.dev5" --- tests/cqml_test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 2f701a9..d3bb809 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.4.6.dev4 +!pip install cqml==0.4.6.dev5 import cqml @@ -25,7 +25,7 @@ CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False -print(f'Parameters[{CONF}]DEBUG={DEBUG} {CF}') +print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) print(cvm.status()) From a564a6cbe381f4177089bf2d17c127dba38e1887 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:37:28 -0700 Subject: [PATCH 047/107] run cvm directly --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index d3bb809..7cc8416 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -31,7 +31,7 @@ # COMMAND ---------- -if not DEBUG: +#if not DEBUG: cvm.run() print(cvm.sizes) dbutils.notebook.exit(0) From a04e8b6fefdccfcd81d697413439e3644b6ba97c Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:37:32 -0700 Subject: [PATCH 048/107] "0.4.6.dev6_"2022-05-15T23:37:32Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 6d94c1a..8e2b858 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev5 +0.4.6.dev6 diff --git a/version.txt.prev b/version.txt.prev index 069416b..6d94c1a 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.6.dev4 +0.4.6.dev5 From 985caf4c9e5f7deade0467457f75898bde1c4205 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:37:33 -0700 Subject: [PATCH 049/107] "post-regen-tests/cqml_test_db.py-0.4.6.dev6" --- tests/cqml_test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 7cc8416..47f2fb8 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.4.6.dev5 +!pip install cqml==0.4.6.dev6 import cqml @@ -31,7 +31,7 @@ # COMMAND ---------- -#if not DEBUG: +if not DEBUG: cvm.run() print(cvm.sizes) dbutils.notebook.exit(0) From 132933e6cac320df693d46bb42668fae8786e897 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:38:54 -0700 Subject: [PATCH 050/107] v0.5 --- version.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.txt b/version.txt index 8e2b858..8adb4af 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.4.6.dev6 +0.5.0.dev1 From b665922befe8442837dbf0f83811e25ec11b589c Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:40:31 -0700 Subject: [PATCH 051/107] "0.5.0.dev2_"2022-05-15T23:40:31Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 8adb4af..4557e6f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev1 +0.5.0.dev2 diff --git a/version.txt.prev b/version.txt.prev index 6d94c1a..8adb4af 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.4.6.dev5 +0.5.0.dev1 From 2cc6d99461f2bbe108b0ac731efaf7dd1870fd15 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:40:32 -0700 Subject: [PATCH 052/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev2" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 47f2fb8..2e43504 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.4.6.dev6 +!pip install cqml==0.5.0.dev2 import cqml From 587de7e3a072fed881004f5151f6e64dae0ed269 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:49:33 -0700 Subject: [PATCH 053/107] run directly --- tests/cqml_test_db.py.tmp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index 32b714e..5b2d08e 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -27,11 +27,11 @@ CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) -print(cvm.status()) +#print(cvm.status()) # COMMAND ---------- -if not DEBUG: +#if not DEBUG: cvm.run() print(cvm.sizes) dbutils.notebook.exit(0) From 420540e3c131637f9d6004127a1c7abd8e393106 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:49:36 -0700 Subject: [PATCH 054/107] "0.5.0.dev3_"2022-05-15T23:49:36Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 4557e6f..ba92135 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev2 +0.5.0.dev3 diff --git a/version.txt.prev b/version.txt.prev index 8adb4af..4557e6f 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev1 +0.5.0.dev2 From 72704fd8ff6353fed1080528858ea7d20a342649 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 16:49:37 -0700 Subject: [PATCH 055/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev3" --- tests/cqml_test_db.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 2e43504..7e5507d 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev2 +!pip install cqml==0.5.0.dev3 import cqml @@ -27,11 +27,11 @@ DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) -print(cvm.status()) +#print(cvm.status()) # COMMAND ---------- -if not DEBUG: +#if not DEBUG: cvm.run() print(cvm.sizes) dbutils.notebook.exit(0) From fe55a83ef973daaee6ac1933cd88bbc121328acf Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 20:47:55 -0700 Subject: [PATCH 056/107] print(cvm.yaml) --- tests/cqml_test_db.py.tmp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index 5b2d08e..cd6fc5f 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -27,14 +27,14 @@ CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) -#print(cvm.status()) +print(cvm.yaml) # COMMAND ---------- #if not DEBUG: - cvm.run() - print(cvm.sizes) - dbutils.notebook.exit(0) +cvm.run() +print(cvm.sizes) +dbutils.notebook.exit(0) # COMMAND ---------- From 80522bcd9a806463f04448060af3b7f79cf1a99f Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 20:48:05 -0700 Subject: [PATCH 057/107] "0.5.0.dev4_"2022-05-16T03:48:05Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index ba92135..306323f 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev3 +0.5.0.dev4 diff --git a/version.txt.prev b/version.txt.prev index 4557e6f..ba92135 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev2 +0.5.0.dev3 From 237bc8cede54edb72ac9b9335aa4319f6d166571 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 20:48:06 -0700 Subject: [PATCH 058/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev4" --- tests/cqml_test_db.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 7e5507d..d397de4 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev3 +!pip install cqml==0.5.0.dev4 import cqml @@ -27,14 +27,14 @@ DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) -#print(cvm.status()) +print(cvm.yaml) # COMMAND ---------- #if not DEBUG: - cvm.run() - print(cvm.sizes) - dbutils.notebook.exit(0) +cvm.run() +print(cvm.sizes) +dbutils.notebook.exit(0) # COMMAND ---------- From cf0e0e1410fd6d37e3eac64f6bd7716179f5d0db Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 20:54:48 -0700 Subject: [PATCH 059/107] assert 'org' in yml['env'] --- tests/cqml_test_db.py.tmp | 2 +- tests/root_test.py | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index cd6fc5f..2aedbac 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -27,7 +27,7 @@ CONF=getArgument("CONF") DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) -print(cvm.yaml) +print(cvm.yaml['env']) # COMMAND ---------- diff --git a/tests/root_test.py b/tests/root_test.py index 683ab38..9ff57a9 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -21,3 +21,6 @@ def test_new(root): cvm = root.new(spark, TEST_KEY) assert cvm assert "test" == cvm.log("test") + yml = cvm.yaml + assert 'env' in yml + assert 'org' in yml['env'] From fa1f9adbf1a719cf26b232362e6acdb7fc5afa57 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 20:54:53 -0700 Subject: [PATCH 060/107] "0.5.0.dev5_"2022-05-16T03:54:53Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 306323f..806177c 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev4 +0.5.0.dev5 diff --git a/version.txt.prev b/version.txt.prev index ba92135..306323f 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev3 +0.5.0.dev4 From dd93e0bd8a3893f9ee72eca89692456cade0e4d3 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 20:54:54 -0700 Subject: [PATCH 061/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev5" --- tests/cqml_test_db.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index d397de4..75cea8c 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev4 +!pip install cqml==0.5.0.dev5 import cqml @@ -27,7 +27,7 @@ DEBUG=True if getArgument("DEBUG") == "DEBUG" else False print(f'Parameters[{CONF}]DEBUG={DEBUG}') cvm = root.new(spark, CONF, DEBUG) -print(cvm.yaml) +print(cvm.yaml['env']) # COMMAND ---------- From d0f2817b3e9efc570f3759c5a026db0ef2c7de7d Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:02:13 -0700 Subject: [PATCH 062/107] set_env debug --- src/cqml/root.py | 4 ++++ tests/root_test.py | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/src/cqml/root.py b/src/cqml/root.py index a9815fd..947de01 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -22,11 +22,15 @@ def add_env(self, yml, key): return yml[kEnv] def set_env(self, yml, key): + print(f'set_env:{key}') folder = yml[kEnv]["project"] env = {} if self.root in self.env: env.update(self.env[self.root]) + print(f'set_env.root:{env}') if folder in self.env: env.update(self.env[folder]) + print(f'set_env.folder:{env}') if kEnv in yml: env.update(yml[kEnv]) + print(f'set_env.kEnv:{env}') yml[kEnv] = env return env diff --git a/tests/root_test.py b/tests/root_test.py index 9ff57a9..96992b5 100644 --- a/tests/root_test.py +++ b/tests/root_test.py @@ -24,3 +24,10 @@ def test_new(root): yml = cvm.yaml assert 'env' in yml assert 'org' in yml['env'] + +def test_demo(root): + cvm = root.new(spark, 'demo/demo') + yml = cvm.yaml + assert 'env' in yml + assert 'org' in yml['env'] + assert 'nauto' in yml['env']['org'] From 2d09c9fd09326f90fd945bf8ad74228f5bc453a9 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:02:17 -0700 Subject: [PATCH 063/107] "0.5.0.dev6_"2022-05-16T04:02:17Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 806177c..c19dfe8 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev5 +0.5.0.dev6 diff --git a/version.txt.prev b/version.txt.prev index 306323f..806177c 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev4 +0.5.0.dev5 From 5fc503a7442625e4d646fab01d2c9d6c9c74aa44 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:02:18 -0700 Subject: [PATCH 064/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev6" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 75cea8c..3a8e5d5 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev5 +!pip install cqml==0.5.0.dev6 import cqml From e51e7f3fe2bc593f48b25f9459fc4cee7bf62f35 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:07:57 -0700 Subject: [PATCH 065/107] root_key --- src/cqml/root.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cqml/root.py b/src/cqml/root.py index 947de01..8b864cf 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -9,7 +9,7 @@ def read_yaml(yaml_file): class Root: def __init__(self, root): - self.root = root + self.root = root.split("/")[-1] self.pipes = {} self.env = {} self.scan(root) From a141a46d32f0d448faf474c3f42ab5f213bd3843 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:08:02 -0700 Subject: [PATCH 066/107] "0.5.0.dev7_"2022-05-16T04:08:02Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index c19dfe8..313bd94 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev6 +0.5.0.dev7 diff --git a/version.txt.prev b/version.txt.prev index 806177c..c19dfe8 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev5 +0.5.0.dev6 From 3ffa1d92bb82f141f645e9887b2230678f542f45 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:08:03 -0700 Subject: [PATCH 067/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev7" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 3a8e5d5..ce9e689 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev6 +!pip install cqml==0.5.0.dev7 import cqml From a2b2c19265f8eef01908f0cd21e3811f6414f7e0 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:25:51 -0700 Subject: [PATCH 068/107] pipes/all --- pipes/all.yml | 16 ++++++++++++++++ pipes/cqml.yml | 22 ---------------------- src/cqml/root.py | 7 +------ tests/context.py | 1 + tests/cqml_test_db.py.tmp | 1 + tests/pkg_test.py | 13 ++++++++++--- 6 files changed, 29 insertions(+), 31 deletions(-) create mode 100644 pipes/all.yml delete mode 100644 pipes/cqml.yml diff --git a/pipes/all.yml b/pipes/all.yml new file mode 100644 index 0000000..dd6bce8 --- /dev/null +++ b/pipes/all.yml @@ -0,0 +1,16 @@ +cqml: 0.6 +order: 0.0 +id: main +env: + org: nauto + bucket: biz-databricks-root-prod-us + catalog: quilt + root: /dbfs/tmp + path: . +actions: + all: + do: run + start: 0500 + pipes: + - demo + - test diff --git a/pipes/cqml.yml b/pipes/cqml.yml deleted file mode 100644 index 574f909..0000000 --- a/pipes/cqml.yml +++ /dev/null @@ -1,22 +0,0 @@ -cqml: 0.6 -order: 0.0 -id: main -env: - org: nauto - bucket: biz-databricks-root-prod-us - catalog: quilt - root: /dbfs/tmp - path: . -refresh_trips: - do: schedule - start: 0100 - folders: - - refresh - - fleet -core: - do: schedule - start: 0500 - folders: - - nauto - - user - - rnr diff --git a/src/cqml/root.py b/src/cqml/root.py index 8b864cf..d4e30d5 100644 --- a/src/cqml/root.py +++ b/src/cqml/root.py @@ -26,11 +26,8 @@ def set_env(self, yml, key): folder = yml[kEnv]["project"] env = {} if self.root in self.env: env.update(self.env[self.root]) - print(f'set_env.root:{env}') if folder in self.env: env.update(self.env[folder]) - print(f'set_env.folder:{env}') if kEnv in yml: env.update(yml[kEnv]) - print(f'set_env.kEnv:{env}') yml[kEnv] = env return env @@ -46,9 +43,7 @@ def pkg(self, spark, key, debug=False): cvm.run(); return pkg_cvm(cvm) - def pkg_all(self, spark, debug=False): - pkgs = {key:self.pkg(spark, key, debug) for key in self.keys()} - return pkgs + #def pkg_all(self, spark, debug=False): return {key:self.pkg(spark, key, debug) for key in self.keys()} def scan(self, root): for entry in os.scandir(root): diff --git a/tests/context.py b/tests/context.py index 440cc20..9bfd5e3 100644 --- a/tests/context.py +++ b/tests/context.py @@ -2,6 +2,7 @@ import sys sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '.','./src','..','../src'))) TEST_KEY="test/cqml" +TEST_DEMO="demo/demo" DDIR="/Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data" import cqml diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index 2aedbac..e28f534 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -18,6 +18,7 @@ import cqml CQML_ROOT="../pipes" root = cqml.Root(CQML_ROOT) keys = root.keys() +print(f'{root.root}: {root.env}') dbutils.widgets.dropdown("CONF", keys[0], keys) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) diff --git a/tests/pkg_test.py b/tests/pkg_test.py index 1b6e0fd..1a190ca 100644 --- a/tests/pkg_test.py +++ b/tests/pkg_test.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 import pytest -from .context import cqml, TEST_KEY, DDIR +from .context import * from .db_mock import spark @pytest.fixture @@ -14,11 +14,18 @@ def test_pkg(root): assert 'html' in dict assert 'actions' in dict +def test_pkg_demo(root): + dict = root.pkg(spark, TEST_DEMO, True) + assert 'pkg' in dict + assert 'html' in dict + assert 'actions' in dict + def test_box(root): cvm = root.new(spark, TEST_KEY) cvm.test_id(DDIR) it = cvm.test_id("box_details") assert it -#def test_all(root): dict = root.pkg_all(spark, True) - #assert 'cqml' in dict +def test_all(root): + dict = root.pkg(spark, 'pipes/all', True) + assert 'all' in dict From 68a81f76f05b70c7cf4c0a9597f75ff212339b6c Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:25:54 -0700 Subject: [PATCH 069/107] "0.5.0.dev8_"2022-05-16T04:25:54Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 313bd94..8992c89 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev7 +0.5.0.dev8 diff --git a/version.txt.prev b/version.txt.prev index c19dfe8..313bd94 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev6 +0.5.0.dev7 From 893295119f0c506e46627cfb7600716176b11640 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:25:56 -0700 Subject: [PATCH 070/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev8" --- tests/cqml_test_db.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index ce9e689..d41c409 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev7 +!pip install cqml==0.5.0.dev8 import cqml @@ -18,6 +18,7 @@ CQML_ROOT="../pipes" root = cqml.Root(CQML_ROOT) keys = root.keys() +print(f'{root.root}: {root.env}') dbutils.widgets.dropdown("CONF", keys[0], keys) dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"]) From 694a0dce713224f0e407f0b84d197563d63c1eb0 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:28:13 -0700 Subject: [PATCH 071/107] box better --- pipes/all.yml | 4 ++-- pipes/demo/demo.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipes/all.yml b/pipes/all.yml index dd6bce8..9cd75cd 100644 --- a/pipes/all.yml +++ b/pipes/all.yml @@ -12,5 +12,5 @@ actions: do: run start: 0500 pipes: - - demo - - test + - demo/demo + - test/cqml diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml index 4fce33a..4f02fb8 100644 --- a/pipes/demo/demo.yml +++ b/pipes/demo/demo.yml @@ -37,7 +37,7 @@ actions: file_ext: csv expiration_date: '2200-02-02' cols: - letter: sort + better: sort dat: sort widget-report: do: report From de7b4ae3076df04573c072c07a7055a327d2e5ee Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:28:57 -0700 Subject: [PATCH 072/107] "0.5.0.dev9_"2022-05-16T04:28:57Z"" --- tests/pkg_test.py | 4 +--- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/pkg_test.py b/tests/pkg_test.py index 1a190ca..6130436 100644 --- a/tests/pkg_test.py +++ b/tests/pkg_test.py @@ -26,6 +26,4 @@ def test_box(root): it = cvm.test_id("box_details") assert it -def test_all(root): - dict = root.pkg(spark, 'pipes/all', True) - assert 'all' in dict +#def test_all(root): dict = root.pkg(spark, 'pipes/all', True) diff --git a/version.txt b/version.txt index 8992c89..f25c9bc 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev8 +0.5.0.dev9 diff --git a/version.txt.prev b/version.txt.prev index 313bd94..8992c89 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev7 +0.5.0.dev8 From 5db561b410354dee24dc4470b4d17acbeb2d2354 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:28:58 -0700 Subject: [PATCH 073/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev9" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index d41c409..075942a 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev8 +!pip install cqml==0.5.0.dev9 import cqml From e45bdf08137abe9b71904490c3681a885b8627f1 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:39:02 -0700 Subject: [PATCH 074/107] "0.5.0.dev10_"2022-05-16T04:39:02Z"" --- src/cqml/boxquilt.py | 6 ++++-- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/cqml/boxquilt.py b/src/cqml/boxquilt.py index 8b62124..7190e56 100644 --- a/src/cqml/boxquilt.py +++ b/src/cqml/boxquilt.py @@ -147,5 +147,7 @@ def create_or_update_box(self, skipUpdate=False): def box_table(self): array = list(self.rows.values()) - print(f'box_table: {len(array)}') - return self.spark.createDataFrame([Row(**i) for i in array]) + if len(array) > 0: + print(f'box_table: {len(array)}') + return self.spark.createDataFrame([Row(**i) for i in array]) + return self.spark.createDataFrame(data = [], schema = StructType([])) diff --git a/version.txt b/version.txt index f25c9bc..2866430 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev9 +0.5.0.dev10 diff --git a/version.txt.prev b/version.txt.prev index 8992c89..f25c9bc 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev8 +0.5.0.dev9 From 9d1fd82324fe493f834ae77d2611001ee5c0277d Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:39:04 -0700 Subject: [PATCH 075/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev10" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 075942a..e6f5a3b 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev9 +!pip install cqml==0.5.0.dev10 import cqml From 5053b232e0dfb344d3d3aa668d080ac8a3522fd5 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:44:35 -0700 Subject: [PATCH 076/107] "0.5.0.dev11_"2022-05-16T04:44:35Z"" --- src/cqml/boxquilt.py | 4 ++-- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/cqml/boxquilt.py b/src/cqml/boxquilt.py index 7190e56..a285984 100644 --- a/src/cqml/boxquilt.py +++ b/src/cqml/boxquilt.py @@ -12,7 +12,7 @@ from boxsdk import Client, OAuth2, JWTAuth from pyspark.sql import Row from pyspark.sql.functions import udf,lit -from pyspark.sql.types import StringType +from pyspark.sql.types import StringType, StructType import os def dir_row(folder): @@ -150,4 +150,4 @@ def box_table(self): if len(array) > 0: print(f'box_table: {len(array)}') return self.spark.createDataFrame([Row(**i) for i in array]) - return self.spark.createDataFrame(data = [], schema = StructType([])) + return self.spark.createDataFrame([], StructType([])) diff --git a/version.txt b/version.txt index 2866430..1690eb2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev10 +0.5.0.dev11 diff --git a/version.txt.prev b/version.txt.prev index f25c9bc..2866430 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev9 +0.5.0.dev10 From 031b99d13a85cf6a5140aeee5ba7ed79977709a3 Mon Sep 17 00:00:00 2001 From: drernie Date: Sun, 15 May 2022 21:44:37 -0700 Subject: [PATCH 077/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev11" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index e6f5a3b..11866eb 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev10 +!pip install cqml==0.5.0.dev11 import cqml From 4fb72632070b8329a3b368012c89051cde01dcaa Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 10:43:45 -0700 Subject: [PATCH 078/107] skip empty box table --- src/cqml/boxquilt.py | 2 +- tests/db_mock.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cqml/boxquilt.py b/src/cqml/boxquilt.py index a285984..163054f 100644 --- a/src/cqml/boxquilt.py +++ b/src/cqml/boxquilt.py @@ -150,4 +150,4 @@ def box_table(self): if len(array) > 0: print(f'box_table: {len(array)}') return self.spark.createDataFrame([Row(**i) for i in array]) - return self.spark.createDataFrame([], StructType([])) + return self.spark.createDataFrame([], StructType([])) diff --git a/tests/db_mock.py b/tests/db_mock.py index 5ea5fe2..6e33195 100644 --- a/tests/db_mock.py +++ b/tests/db_mock.py @@ -70,7 +70,7 @@ def withColumn(self, *arg): return self class MockSpark(object): def __init__(self): self.columns = [] - def createDataFrame(self, list): return self.table('list') + def createDataFrame(self, list, schema=[]): return self.table('list') def count(self): return self def distinct(self,arg=None): return self def get(self, arg): return f'mock.get:{arg}' From 88ccab23e17aa350651e6aeee83cafb0410befab Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 10:43:52 -0700 Subject: [PATCH 079/107] "0.5.0.dev12_"2022-05-16T17:43:52Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 1690eb2..bbef4a1 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev11 +0.5.0.dev12 diff --git a/version.txt.prev b/version.txt.prev index 2866430..1690eb2 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev10 +0.5.0.dev11 From 029e65f455b27d4354312510989325d69501dd60 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 10:43:53 -0700 Subject: [PATCH 080/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev12" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 11866eb..b2f88ac 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev11 +!pip install cqml==0.5.0.dev12 import cqml From 8c480a429650f153206108b57b173aca7191858f Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 11:56:14 -0700 Subject: [PATCH 081/107] "0.5.0.dev13_"2022-05-16T18:56:14Z"" --- pipes/demo/demo.yml | 4 ++-- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml index 4f02fb8..e460960 100644 --- a/pipes/demo/demo.yml +++ b/pipes/demo/demo.yml @@ -59,8 +59,8 @@ actions: from: grouped operator: + args: - - next - - num + - text + - sum_num calc_quarters: do: calc from: plus_operator diff --git a/version.txt b/version.txt index bbef4a1..eb22cfe 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev12 +0.5.0.dev13 diff --git a/version.txt.prev b/version.txt.prev index 1690eb2..bbef4a1 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev11 +0.5.0.dev12 From 31dd2c112cdfcd8a8c2119511485eb6ef6a7d1c2 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 11:56:15 -0700 Subject: [PATCH 082/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev13" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index b2f88ac..5101076 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev12 +!pip install cqml==0.5.0.dev13 import cqml From a5238c00f5be6bf6f2bb0894d9fcfecc97d170cd Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:02:52 -0700 Subject: [PATCH 083/107] "0.5.0.dev14_"2022-05-16T19:02:52Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index eb22cfe..58a40df 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev13 +0.5.0.dev14 diff --git a/version.txt.prev b/version.txt.prev index bbef4a1..eb22cfe 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev12 +0.5.0.dev13 From b56fdaa1ce86448d6bf79316d4b34bf6716f6483 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:02:53 -0700 Subject: [PATCH 084/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev14" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 5101076..dbf6b76 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev13 +!pip install cqml==0.5.0.dev14 import cqml From f6a315de3c92b08972c946ac3cde374902be9d0c Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:13:57 -0700 Subject: [PATCH 085/107] "0.5.0.dev15_"2022-05-16T19:13:57Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 58a40df..9472129 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev14 +0.5.0.dev15 diff --git a/version.txt.prev b/version.txt.prev index eb22cfe..58a40df 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev13 +0.5.0.dev14 From 064a231ddf203c05efa669e6a6bad8977b437262 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:13:59 -0700 Subject: [PATCH 086/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev15" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index dbf6b76..7f2ae7a 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev14 +!pip install cqml==0.5.0.dev15 import cqml From 55ece6d8d962632fe4ece762bb60124414fb390e Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:18:06 -0700 Subject: [PATCH 087/107] calc_quarters --- pipes/demo/demo.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml index e460960..8f866f4 100644 --- a/pipes/demo/demo.yml +++ b/pipes/demo/demo.yml @@ -71,8 +71,8 @@ actions: - $col - 4 cols: - num: qnum - next: qnumb + sum_num: qnum + text: qnumb plus_operator: qdumb flagged: do: flag From 623b7b0384abf972ddc5d6e994d95b49693b0c7a Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:18:10 -0700 Subject: [PATCH 088/107] "0.5.0.dev16_"2022-05-16T19:18:10Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 9472129..77495d2 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev15 +0.5.0.dev16 diff --git a/version.txt.prev b/version.txt.prev index 58a40df..9472129 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev14 +0.5.0.dev15 From 4ba01bfe212b6fdf1dd10e509a6df85e2bf4b033 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:18:11 -0700 Subject: [PATCH 089/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev16" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 7f2ae7a..f9a3d97 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev15 +!pip install cqml==0.5.0.dev16 import cqml From 3c426a4eb4bddf682569d0c24d23c30c5d5fe56f Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:20:02 -0700 Subject: [PATCH 090/107] drop aggregates --- pipes/demo/demo.yml | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/pipes/demo/demo.yml b/pipes/demo/demo.yml index 8f866f4..1ef71ce 100644 --- a/pipes/demo/demo.yml +++ b/pipes/demo/demo.yml @@ -124,13 +124,3 @@ actions: cols: letter: tbd text: tbd - aggregates: - do: group - from: $id - agg: - concat_space: count - call_coalesce: sum - sort: n_concat_space - cols: - num: tbd - letter: tbd From d260100c04770bd3195db2d38f937044e3765ded Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:20:09 -0700 Subject: [PATCH 091/107] "0.5.0.dev17_"2022-05-16T19:20:08Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 77495d2..8b15f08 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev16 +0.5.0.dev17 diff --git a/version.txt.prev b/version.txt.prev index 9472129..77495d2 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev15 +0.5.0.dev16 From 962882845d4140c8ea5e47b21b646aa1163b2d73 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:20:10 -0700 Subject: [PATCH 092/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev17" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index f9a3d97..79f3e22 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev16 +!pip install cqml==0.5.0.dev17 import cqml From bc54d1e260ead111a03b9572351acaae6b920298 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 12:27:18 -0700 Subject: [PATCH 093/107] cvm.do_save --- src/cqml/wrappers.py | 2 +- tests/cqml_test_db.py.tmp | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py index 57c3b4e..5968800 100644 --- a/src/cqml/wrappers.py +++ b/src/cqml/wrappers.py @@ -21,7 +21,7 @@ def do_run(self, action): pkgs = {cqml:pkg_cqml(cqml, self.spark) for cqml in runs} return pkgs - def do_save(self, action): + def do_save(self, action={}): pkg = cvm2pkg(self, False) # do not re-run return pkg diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp index e28f534..a59ebf2 100644 --- a/tests/cqml_test_db.py.tmp +++ b/tests/cqml_test_db.py.tmp @@ -35,6 +35,8 @@ print(cvm.yaml['env']) #if not DEBUG: cvm.run() print(cvm.sizes) +cvm.do_save() + dbutils.notebook.exit(0) # COMMAND ---------- From 820bb8c2e3f3167d41d9adceb4829408369102a2 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 20:30:52 -0700 Subject: [PATCH 094/107] use reports s3 bucket --- pipes/all.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipes/all.yml b/pipes/all.yml index 9cd75cd..35c9b88 100644 --- a/pipes/all.yml +++ b/pipes/all.yml @@ -3,7 +3,7 @@ order: 0.0 id: main env: org: nauto - bucket: biz-databricks-root-prod-us + bucket: biz-databricks-prod-reports catalog: quilt root: /dbfs/tmp path: . From 6b4441c9f6db2893327d93a90a4a8a1cbb12b5c6 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 20:32:39 -0700 Subject: [PATCH 095/107] "0.5.0.dev18_"2022-05-17T03:32:39Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 8b15f08..3d5efb4 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev17 +0.5.0.dev18 diff --git a/version.txt.prev b/version.txt.prev index 77495d2..8b15f08 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev16 +0.5.0.dev17 From a3c3fc71a09c601994e2c6a00cf68b1d642b93a0 Mon Sep 17 00:00:00 2001 From: drernie Date: Mon, 16 May 2022 20:32:41 -0700 Subject: [PATCH 096/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev18" --- tests/cqml_test_db.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 79f3e22..24a7a42 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev17 +!pip install cqml==0.5.0.dev18 import cqml @@ -35,6 +35,8 @@ #if not DEBUG: cvm.run() print(cvm.sizes) +cvm.do_save() + dbutils.notebook.exit(0) # COMMAND ---------- From 4d3de630c48fd76fba8840c3997d6016c8f4982c Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 20 May 2022 15:49:36 -0700 Subject: [PATCH 097/107] "0.5.0.dev19_"2022-05-20T22:49:36Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 3d5efb4..1cbdb29 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev18 +0.5.0.dev19 diff --git a/version.txt.prev b/version.txt.prev index 8b15f08..3d5efb4 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev17 +0.5.0.dev18 From 6d6a0cc83b5e8a2ca80a1293bcf85091b1507166 Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 20 May 2022 15:49:38 -0700 Subject: [PATCH 098/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev19" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 24a7a42..34ea6a7 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev18 +!pip install cqml==0.5.0.dev19 import cqml From f24522fd863ac92f420eae1b900be4611ad1589a Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 20 May 2022 16:55:35 -0700 Subject: [PATCH 099/107] "0.5.0.dev20_"2022-05-20T23:55:35Z"" --- src/cqml/db2quilt.py | 10 ++++++---- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index bd93abe..6e68cde 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -17,7 +17,6 @@ from pathlib import Path import pprint pp = pprint.PrettyPrinter(indent=4) -QPKG = q3.Package() def cleanup_names(df): for c in df.columns: @@ -162,13 +161,14 @@ def __init__(self, id, proj, reset=True): self.url = f"{proj.url}/{self.name}/" self.path = f"{proj.path}/{self.name}/" self.dir = to_dir(self.path) + self.pkg = q3.Package.browse(self.name, registry=self.proj.repo, dest=self.path) if reset: shutil.rmtree(self.path,ignore_errors=True) make_dir(self.path) self.summaries={} def setup(self): - QPKG.install(self.name, registry=self.proj.repo, dest=self.path) + self.pkg.install(self.name, registry=self.proj.repo, dest=self.path) def read_csv(self, filename): path = self.path+filename @@ -178,8 +178,10 @@ def read_csv(self, filename): def cleanup(self, msg, meta = {"db2quilt":"v0.1"}): self.write_summary() - QPKG.set_dir('/',path=self.path, meta=meta) - QPKG.push(self.name, self.proj.repo, message=msg,force=True) #, + self.pkg.set_dir('/',path=self.path, meta=meta) + print(self.pkg) + + self.pkg.push(self.name, self.proj.repo, message=msg,force=True) #, #shutil.rmtree(self.path) self.html = f'Published {self.name} for {msg}' return self diff --git a/version.txt b/version.txt index 1cbdb29..90f9822 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev19 +0.5.0.dev20 diff --git a/version.txt.prev b/version.txt.prev index 3d5efb4..1cbdb29 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev18 +0.5.0.dev19 From 1894a533c6c4819bbb784bed5a58d1d6492f84e3 Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 20 May 2022 16:55:36 -0700 Subject: [PATCH 100/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev20" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 34ea6a7..70ac794 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev19 +!pip install cqml==0.5.0.dev20 import cqml From 3556d961021ac5e787313d491398f1103b74c395 Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 20 May 2022 21:38:33 -0700 Subject: [PATCH 101/107] "0.5.0.dev21_"2022-05-21T04:38:33Z"" --- src/cqml/db2quilt.py | 2 +- version.txt | 2 +- version.txt.prev | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py index 6e68cde..aea6181 100644 --- a/src/cqml/db2quilt.py +++ b/src/cqml/db2quilt.py @@ -161,7 +161,7 @@ def __init__(self, id, proj, reset=True): self.url = f"{proj.url}/{self.name}/" self.path = f"{proj.path}/{self.name}/" self.dir = to_dir(self.path) - self.pkg = q3.Package.browse(self.name, registry=self.proj.repo, dest=self.path) + self.pkg = q3.Package.browse(self.name, registry=self.proj.repo) if reset: shutil.rmtree(self.path,ignore_errors=True) make_dir(self.path) diff --git a/version.txt b/version.txt index 90f9822..6cb2d5b 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev20 +0.5.0.dev21 diff --git a/version.txt.prev b/version.txt.prev index 1cbdb29..90f9822 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev19 +0.5.0.dev20 From bad9958ca545703113853d72302877943c7ab8b2 Mon Sep 17 00:00:00 2001 From: drernie Date: Fri, 20 May 2022 21:38:34 -0700 Subject: [PATCH 102/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev21" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 70ac794..57c719b 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev20 +!pip install cqml==0.5.0.dev21 import cqml From a4afd85760cd28083e6b44ee6c729c1a19452a5d Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 24 May 2022 17:07:47 -0700 Subject: [PATCH 103/107] "0.5.0.dev22_"2022-05-25T00:07:47Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 6cb2d5b..f2b1dad 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev21 +0.5.0.dev22 diff --git a/version.txt.prev b/version.txt.prev index 90f9822..6cb2d5b 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev20 +0.5.0.dev21 From 2fc52ef0e5c25224f0a92f3198759a5fa81f109d Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 24 May 2022 17:07:48 -0700 Subject: [PATCH 104/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev22" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 57c719b..7fef9d9 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev21 +!pip install cqml==0.5.0.dev22 import cqml From db29ec3f8e43c9567083180918bd3b3e89725b8c Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 24 May 2022 17:09:29 -0700 Subject: [PATCH 105/107] "0.5.0.dev23_"2022-05-25T00:09:29Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index f2b1dad..11c0430 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev22 +0.5.0.dev23 diff --git a/version.txt.prev b/version.txt.prev index 6cb2d5b..f2b1dad 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev21 +0.5.0.dev22 From 36ba831f4951d472188670e2a7ef55bec3924767 Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 24 May 2022 17:12:49 -0700 Subject: [PATCH 106/107] "0.5.0.dev24_"2022-05-25T00:12:49Z"" --- version.txt | 2 +- version.txt.prev | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/version.txt b/version.txt index 11c0430..da07663 100644 --- a/version.txt +++ b/version.txt @@ -1 +1 @@ -0.5.0.dev23 +0.5.0.dev24 diff --git a/version.txt.prev b/version.txt.prev index f2b1dad..11c0430 100644 --- a/version.txt.prev +++ b/version.txt.prev @@ -1 +1 @@ -0.5.0.dev22 +0.5.0.dev23 From 5698bd90df0e4d569610f629459cc68c8e8468c1 Mon Sep 17 00:00:00 2001 From: drernie Date: Tue, 24 May 2022 17:12:50 -0700 Subject: [PATCH 107/107] "post-regen-tests/cqml_test_db.py-0.5.0.dev24" --- tests/cqml_test_db.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py index 7fef9d9..11fe3d7 100644 --- a/tests/cqml_test_db.py +++ b/tests/cqml_test_db.py @@ -9,7 +9,7 @@ !pip install --upgrade pip #!pip install cqml !pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45 -!pip install cqml==0.5.0.dev22 +!pip install cqml==0.5.0.dev24 import cqml