diff --git a/CHANGELOG.md b/CHANGELOG.md
index b0f8ef1..56a5fa0 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,21 +1,16 @@
-0.4.6 (unreleased)
+0.5.0 (unreleased)
------------------
-- Nothing changed yet.
+- Use extrinsic folder names for project/package
+- do: schedule for top-level ordering and scheduling
0.4.5 (2022-05-10)
------------------
- Fix Manifest
-
-
-0.4.4 (unreleased)
-------------------
-
- YMM 0.6.1 support: open direct URL for test runs
-
0.4.3 (2022-04-18)
------------------
diff --git a/archive-v45/cqml45.py b/archive-v45/cqml45.py
new file mode 100755
index 0000000..8348c78
--- /dev/null
+++ b/archive-v45/cqml45.py
@@ -0,0 +1,58 @@
+#!/usr/bin/env python3
+import os, yaml
+
+#
+# Take two paths: root4 root5
+# Copy files in each subfolder
+# Strip out meta
+# Strip leading numbers
+#
+
+ROOT="/Users/nauto/Developer"
+R4=f"{ROOT}/it/databricks"
+R5=f"{ROOT}/dbt/cqml"
+
+def read_yaml(yaml_file):
+ with open(yaml_file) as data:
+ raw_yaml = yaml.full_load(data)
+ return raw_yaml
+
+def write_yaml(yaml_file, raw_yaml):
+ with open(yaml_file, 'w') as file:
+ yaml.dump(raw_yaml, file, sort_keys=False)
+
+def extract(root):
+ tree = []
+ for folder in os.scandir(root):
+ if folder.is_dir():
+ print(folder.name)
+ for file in os.scandir(folder.path):
+ if file.name.endswith(".yml"):
+ yml = read_yaml(file.path)
+ node = {"file":file.name,"project":folder.name, "yml": yml}
+ tree.append(node)
+ return tree
+
+def convert(root, node):
+ dir = node["project"]
+ file = ''.join([c for c in node["file"] if not c.isdigit()])
+ prefix = file.split("_")[0]
+ name = file.split("_")[1]
+ if prefix == "rnr": dir = prefix
+ if prefix == "sierra": name = f"{prefix}.yml"
+ path = os.path.join(root, dir, name)
+ yml = node["yml"]
+ del yml["meta"]
+ yml["cqml"] = 0.5
+ yml["project"] = dir
+ yml["package"] = name
+ write_yaml(path, yml)
+ return path
+
+t = extract(R4)
+print(f"\nExtracted: {len(t)} files\n")
+#print(t[0]["yml"])
+for n in t:
+ print(n["file"])
+ p = convert(R5, n)
+ print("\t",p)
diff --git a/pipes/all.yml b/pipes/all.yml
new file mode 100644
index 0000000..35c9b88
--- /dev/null
+++ b/pipes/all.yml
@@ -0,0 +1,16 @@
+cqml: 0.6
+order: 0.0
+id: main
+env:
+ org: nauto
+ bucket: biz-databricks-prod-reports
+ catalog: quilt
+ root: /dbfs/tmp
+ path: .
+actions:
+ all:
+ do: run
+ start: 0500
+ pipes:
+ - demo/demo
+ - test/cqml
diff --git a/tests/demo.yml b/pipes/demo/demo.yml
similarity index 83%
rename from tests/demo.yml
rename to pipes/demo/demo.yml
index ff7bfdc..1ef71ce 100644
--- a/tests/demo.yml
+++ b/pipes/demo/demo.yml
@@ -1,11 +1,5 @@
-cqml: 0.4.4
+cqml: 0.5.0
id: cqml_demo
-meta:
- org: nauto
- project: sangam
- s3.bucket: biz-databricks-root-prod-us
- catalog: quilt
- root: /dbfs/tmp
actions:
/Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data:
do: loadfiles
@@ -43,7 +37,7 @@ actions:
file_ext: csv
expiration_date: '2200-02-02'
cols:
- letter: sort
+ better: sort
dat: sort
widget-report:
do: report
@@ -62,11 +56,11 @@ actions:
letter: tbd
plus_operator:
do: call
- from: rename_merged
+ from: grouped
operator: +
args:
- - next
- - num
+ - text
+ - sum_num
calc_quarters:
do: calc
from: plus_operator
@@ -77,8 +71,8 @@ actions:
- $col
- 4
cols:
- num: qnum
- next: qnumb
+ sum_num: qnum
+ text: qnumb
plus_operator: qdumb
flagged:
do: flag
@@ -130,13 +124,3 @@ actions:
cols:
letter: tbd
text: tbd
- aggregates:
- do: group
- from: $id
- agg:
- concat_space: count
- call_coalesce: sum
- sort: n_concat_space
- cols:
- num: tbd
- letter: tbd
diff --git a/tests/cqml_test.yml b/pipes/test/cqml.yml
similarity index 96%
rename from tests/cqml_test.yml
rename to pipes/test/cqml.yml
index e91dbca..fb41d9b 100644
--- a/tests/cqml_test.yml
+++ b/pipes/test/cqml.yml
@@ -1,11 +1,5 @@
cqml: 0.2
id: cqml_test
-meta:
- org: nauto
- project: sangam
- s3.bucket: biz-databricks-root-prod-us
- catalog: quilt
- root: /dbfs/tmp
actions:
/Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data:
do: loadfiles
diff --git a/src/cqml/__init__.py b/src/cqml/__init__.py
index dc7855f..9c80fe4 100644
--- a/src/cqml/__init__.py
+++ b/src/cqml/__init__.py
@@ -4,4 +4,6 @@
# TODO: https://github.com/LucaCanali/sparkMeasure
+from .yml import *
from .wrappers import *
+from .root import Root
diff --git a/src/cqml/boxquilt.py b/src/cqml/boxquilt.py
index cb64a17..163054f 100644
--- a/src/cqml/boxquilt.py
+++ b/src/cqml/boxquilt.py
@@ -12,7 +12,7 @@
from boxsdk import Client, OAuth2, JWTAuth
from pyspark.sql import Row
from pyspark.sql.functions import udf,lit
-from pyspark.sql.types import StringType
+from pyspark.sql.types import StringType, StructType
import os
def dir_row(folder):
@@ -119,7 +119,7 @@ def create_or_update_box(self, skipUpdate=False):
dbfs = list(self.rows.keys())
to_create = list(set(dbfs) - set(box))
to_update = list(set(dbfs).intersection(box))
- print(f"create:{len(to_create)} update:{len(to_update)}")
+ print(f"box_create:{len(to_create)} update:{len(to_update)}")
n = 0
for name in to_create:
@@ -147,5 +147,7 @@ def create_or_update_box(self, skipUpdate=False):
def box_table(self):
array = list(self.rows.values())
- print(f'box_table: {len(array)}')
- return self.spark.createDataFrame([Row(**i) for i in array])
+ if len(array) > 0:
+ print(f'box_table: {len(array)}')
+ return self.spark.createDataFrame([Row(**i) for i in array])
+ return self.spark.createDataFrame([], StructType([]))
diff --git a/src/cqml/db2quilt.py b/src/cqml/db2quilt.py
index a54233d..aea6181 100644
--- a/src/cqml/db2quilt.py
+++ b/src/cqml/db2quilt.py
@@ -17,7 +17,6 @@
from pathlib import Path
import pprint
pp = pprint.PrettyPrinter(indent=4)
-QPKG = q3.Package()
def cleanup_names(df):
for c in df.columns:
@@ -112,8 +111,8 @@ def callback(x): grid.restore(filter=[['{KEY}', '==', x]])
grid
"""
def make_widget(opts):
- print('make_widget')
- print(opts)
+ #print('make_widget')
+ #print(opts)
code = [NB_WIDGET.format(KEY=col,WIDGET=w) for col, w in opts.items()]
cells = [[True, c] for c in code]
return cells
@@ -133,9 +132,17 @@ def make_slug(name): return re.sub(r'[^\w-]', '_', name.lower())
Quilt Wrappers
"""
+DEFAULT_ENV={
+ 'catalog': PKG_DIR,
+ 'root': PYROOT,
+}
+
+def get_env(env, key, default):
+ return env[key] if key in env else default
+
class Project:
def __init__(self, config):
- org, bucket, project = itemgetter('org','s3.bucket','project')(config)
+ org, bucket, project = itemgetter('org','bucket','project')(config)
pkg_dir = config['catalog'] if 'catalog' in config else PKG_DIR
root = config['root'] if 'root' in config else PYROOT
self.repo = "s3://"+bucket
@@ -147,20 +154,21 @@ def package(self, id):
return Package(id, self)
class Package:
- def __init__(self, id, proj, reset=False):
+ def __init__(self, id, proj, reset=True):
self.id = id
self.name = f"{proj.name}/{id}"
self.proj = proj
self.url = f"{proj.url}/{self.name}/"
self.path = f"{proj.path}/{self.name}/"
self.dir = to_dir(self.path)
+ self.pkg = q3.Package.browse(self.name, registry=self.proj.repo)
if reset:
shutil.rmtree(self.path,ignore_errors=True)
make_dir(self.path)
self.summaries={}
def setup(self):
- QPKG.install(self.name, registry=self.proj.repo, dest=self.path)
+ self.pkg.install(self.name, registry=self.proj.repo, dest=self.path)
def read_csv(self, filename):
path = self.path+filename
@@ -170,8 +178,10 @@ def read_csv(self, filename):
def cleanup(self, msg, meta = {"db2quilt":"v0.1"}):
self.write_summary()
- QPKG.set_dir('/',path=self.path, meta=meta)
- QPKG.push(self.name, self.proj.repo, message=msg,force=True) #,
+ self.pkg.set_dir('/',path=self.path, meta=meta)
+ print(self.pkg)
+
+ self.pkg.push(self.name, self.proj.repo, message=msg,force=True) #,
#shutil.rmtree(self.path)
self.html = f'Published {self.name} for {msg}'
return self
@@ -293,10 +303,13 @@ def write_summary(self):
#
def extract_pkg(cvm):
- id, config = itemgetter('id','meta')(cvm.yaml)
+# print(cvm.yaml)
+ config = itemgetter(kEnv)(cvm.yaml)
+ #print(f"extract_pkg.config: {config}")
proj = Project(config)
+ id = config["package"]
pkg_id = id + DEBUG_SUFFIX if cvm.debug == True else id
- print("extract_pkg: "+pkg_id)
+ #print("extract_pkg: "+pkg_id)
pkg = proj.package(pkg_id)
#pkg.setup()
return pkg
diff --git a/src/cqml/keys.py b/src/cqml/keys.py
index c689436..962868b 100644
--- a/src/cqml/keys.py
+++ b/src/cqml/keys.py
@@ -9,6 +9,7 @@
TRACE=True
cAlias='|'
+kALL='_all_'
kAbove='above'
kAny='any'
kArgs='args'
@@ -17,6 +18,7 @@
kCount='count'
kDoc='+doc'
kDrop='drop'
+kEnv='env'
kExt='file_ext'
kFunc='function'
kGroup='group'
diff --git a/src/cqml/root.py b/src/cqml/root.py
new file mode 100644
index 0000000..d4e30d5
--- /dev/null
+++ b/src/cqml/root.py
@@ -0,0 +1,72 @@
+import os, yaml
+from .keys import *
+from .wrappers import CQML, pkg_cvm
+
+def read_yaml(yaml_file):
+ with open(yaml_file) as data:
+ raw_yaml = yaml.full_load(data)
+ return raw_yaml
+
+class Root:
+ def __init__(self, root):
+ self.root = root.split("/")[-1]
+ self.pipes = {}
+ self.env = {}
+ self.scan(root)
+
+ def keys(self): return list(self.pipes.keys())
+
+ def add_env(self, yml, key):
+ if not kEnv in yml: return {}
+ self.env[key] = yml[kEnv]
+ return yml[kEnv]
+
+ def set_env(self, yml, key):
+ print(f'set_env:{key}')
+ folder = yml[kEnv]["project"]
+ env = {}
+ if self.root in self.env: env.update(self.env[self.root])
+ if folder in self.env: env.update(self.env[folder])
+ if kEnv in yml: env.update(yml[kEnv])
+ yml[kEnv] = env
+ return env
+
+ def new(self, spark, key, debug=False):
+ pipe = self.pipes[key]
+ self.set_env(pipe, key)
+ cvm = CQML(pipe, spark)
+ if debug: cvm.debug = True
+ return cvm
+
+ def pkg(self, spark, key, debug=False):
+ cvm = self.new(spark, key, debug)
+ cvm.run();
+ return pkg_cvm(cvm)
+
+ #def pkg_all(self, spark, debug=False): return {key:self.pkg(spark, key, debug) for key in self.keys()}
+
+ def scan(self, root):
+ for entry in os.scandir(root):
+ self.parse(entry, root)
+
+ def parse(self, entry, folder):
+ name = entry.name
+ if name.endswith(".yml"):
+ yml = read_yaml(entry.path)
+ file_key = os.path.splitext(name)[0]
+ folder_key = folder.split("/")[-1]
+ env = self.add_env(yml, folder_key)
+ key = f"{folder_key}/{file_key}"
+ source = {
+ "file": name,
+ "package": file_key,
+ "project": folder_key,
+ "key": key,
+ "path": entry.path,
+ }
+ env.update(source)
+ yml[kEnv] = env
+ self.pipes[key] = yml
+ elif entry.is_dir():
+ print(entry.name)
+ self.scan(entry.path)
diff --git a/src/cqml/vm.py b/src/cqml/vm.py
index f3496fb..77b930f 100644
--- a/src/cqml/vm.py
+++ b/src/cqml/vm.py
@@ -73,6 +73,7 @@ def log(self, str, name=False):
if self.debug:
if name: print(name)
print(str)
+ return str
def macro(self, todo, action):
mdef = todo.split("|")
diff --git a/src/cqml/wrappers.py b/src/cqml/wrappers.py
index 71c45cb..5968800 100644
--- a/src/cqml/wrappers.py
+++ b/src/cqml/wrappers.py
@@ -3,6 +3,7 @@
from operator import itemgetter
from .db2quilt import cvm2pkg, extract_pkg
from .cvm import CVM
+from .yml import *
class CQML(CVM):
def __init__(self, yaml_data, spark):
@@ -20,18 +21,10 @@ def do_run(self, action):
pkgs = {cqml:pkg_cqml(cqml, self.spark) for cqml in runs}
return pkgs
- def do_save(self, action):
+ def do_save(self, action={}):
pkg = cvm2pkg(self, False) # do not re-run
return pkg
-def upgrade_file(yaml_file):
- print("Upgrading "+yaml_file)
- with open(yaml_file) as data:
- raw_yaml = yaml.full_load(data)
- # insert converter here
- with open(yaml_file, 'w') as file:
- yaml.dump(raw_yaml, file, sort_keys=False)
-
def from_file(yaml_file, spark):
print("Loading "+yaml_file)
with open(yaml_file) as data:
@@ -58,26 +51,22 @@ def exec_cqml(name, spark, folder="pipes"):
cvm.run()
return cvm
-def pkg_cqml(name, spark, folder="pipes"):
- print("\npkg_cqml: "+name)
- cvm = exec_cqml(name, spark, folder)
+def pkg_cvm(cvm):
pkg = cvm2pkg(cvm, False)
return {
- 'pkg': pkg,
- 'html': pkg.html,
- 'url': pkg.url,
- 'actions': cvm.actions,
- 'sizes': cvm.sizes,
- 'times': cvm.times,
- 'frames': cvm.df,
+ 'pkg': pkg,
+ 'html': pkg.html,
+ 'url': pkg.url,
+ 'actions': cvm.actions,
+ 'sizes': cvm.sizes,
+ 'times': cvm.times,
+ 'frames': cvm.df,
}
-def yml_keys(folder="pipes"):
- files = os.listdir(folder)
- keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")]
- keys.sort()
- print(keys)
- return keys
+def pkg_cqml(name, spark, folder="pipes"):
+ print("\npkg_cqml: "+name)
+ cvm = exec_cqml(name, spark, folder)
+ return pkg_cvm(cvm)
def pkg_all(spark, folder="pipes"):
keys = yml_keys(folder)
diff --git a/src/cqml/yml.py b/src/cqml/yml.py
new file mode 100644
index 0000000..52f4a46
--- /dev/null
+++ b/src/cqml/yml.py
@@ -0,0 +1,42 @@
+import os, yaml
+
+def upgrade_file(yaml_file):
+ print("Upgrading "+yaml_file)
+ with open(yaml_file) as data:
+ raw_yaml = yaml.full_load(data)
+ # insert converter here
+ with open(yaml_file, 'w') as file:
+ yaml.dump(raw_yaml, file, sort_keys=False)
+
+def yml_keys(folder="pipes"):
+ files = os.listdir(folder)
+ print(files)
+ keys = [os.path.splitext(file)[0] for file in files if file.endswith("ml")]
+ keys.sort()
+ print(keys)
+ return keys
+
+def yml_tree(folder="pipes"):
+ keys = extract(folder)
+ return keys
+
+def yml_folder(folder, nodes):
+ print(folder.name)
+ for entry in os.scandir(folder.path):
+ if entry.name.endswith(".yml"):
+ key = os.path.splitext(entry.name)[0]
+ #yml = read_yaml(entry.path)
+ node = {"file":entry.name,"project":folder.name, "path": entry.path, "key": key}
+ nodes.append(node)
+ elif entry.is_dir():
+ print(folder.name)
+ yml_folder(folder, tree)
+ return nodes
+
+def extract(root):
+ tree = []
+ for folder in os.scandir(root):
+ if folder.is_dir():
+ print(folder.name)
+ yml_folder(folder, tree)
+ return tree
diff --git a/tests/context.py b/tests/context.py
index 3b1a237..9bfd5e3 100644
--- a/tests/context.py
+++ b/tests/context.py
@@ -1,7 +1,8 @@
import os
import sys
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), '.','./src','..','../src')))
-TEST_YAML="tests/cqml_test.yml"
+TEST_KEY="test/cqml"
+TEST_DEMO="demo/demo"
DDIR="/Workspace/Repos/ernest.prabhakar@nauto.com/cqml/data"
import cqml
diff --git a/tests/cqml_test.py b/tests/cqml_test.py
index 18de986..4522308 100644
--- a/tests/cqml_test.py
+++ b/tests/cqml_test.py
@@ -1,11 +1,12 @@
#!/usr/bin/env python3
import pytest
-from .context import cqml, TEST_YAML, DDIR
+from .context import cqml, TEST_KEY, DDIR
from .db_mock import spark
@pytest.fixture
def cvm():
- cvm = cqml.from_file(TEST_YAML, spark)
+ root = cqml.Root("pipes")
+ cvm = root.new(spark, TEST_KEY)
cvm.test_id(DDIR)
return cvm
diff --git a/tests/cqml_test_db.py b/tests/cqml_test_db.py
index 48e1dd5..11fe3d7 100644
--- a/tests/cqml_test_db.py
+++ b/tests/cqml_test_db.py
@@ -8,36 +8,62 @@
!pip install --upgrade pip
#!pip install cqml
-!pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@ymm-6
-!pip install cqml==0.4.4.dev2
+!pip --no-cache-dir install git+https://github.com/TheSwanFactory/cqml.git@convert45
+!pip install cqml==0.5.0.dev24
import cqml
# COMMAND ----------
-KEY="cqml_test"
-cvm = cqml.load_cqml(KEY,spark, '.')
-cvm.debug = True
+CQML_ROOT="../pipes"
+root = cqml.Root(CQML_ROOT)
+keys = root.keys()
+print(f'{root.root}: {root.env}')
+dbutils.widgets.dropdown("CONF", keys[0], keys)
+dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"])
+
+# COMMAND ----------
+
+CONF=getArgument("CONF")
+DEBUG=True if getArgument("DEBUG") == "DEBUG" else False
+print(f'Parameters[{CONF}]DEBUG={DEBUG}')
+cvm = root.new(spark, CONF, DEBUG)
+print(cvm.yaml['env'])
+
+# COMMAND ----------
+
+#if not DEBUG:
cvm.run()
print(cvm.sizes)
+cvm.do_save()
+
+dbutils.notebook.exit(0)
# COMMAND ----------
-K = list(cvm.df.keys())
-print(K)
-def d(i): return cvm.df[K[i]]
-def view(i):
- print(K[i])
- d(i).show()
-def values(i, col): return d(i).select(col).distinct().collect()
-view(-1)
+cvm.init()
+steps = cvm.steps()
+print(steps)
+#dbutils.notebook.exit(0)
+
# COMMAND ----------
-cvm.do_save({})
-displayHTML(cvm.pkg.html)
+df = {}
+def values(s, col): return df[s].select(col).distinct().collect()
+
+for step in steps:
+ print(step)
+ df[step] = cvm.test(step)
+ df[step].show()
# COMMAND ----------
+
+cvm.save()
+displayHTML(cvm.result())
+
+# COMMAND ----------
+
dbutils.notebook.exit(0)
#spark.sql('create database nauto')
diff --git a/tests/cqml_test_db.py.tmp b/tests/cqml_test_db.py.tmp
index a78da20..a59ebf2 100644
--- a/tests/cqml_test_db.py.tmp
+++ b/tests/cqml_test_db.py.tmp
@@ -15,29 +15,55 @@ import cqml
# COMMAND ----------
-KEY="cqml_test"
-cvm = cqml.load_cqml(KEY,spark, '.')
-cvm.debug = True
+CQML_ROOT="../pipes"
+root = cqml.Root(CQML_ROOT)
+keys = root.keys()
+print(f'{root.root}: {root.env}')
+dbutils.widgets.dropdown("CONF", keys[0], keys)
+dbutils.widgets.dropdown("DEBUG", "DEBUG", ["DEBUG", "PROD"])
+
+# COMMAND ----------
+
+CONF=getArgument("CONF")
+DEBUG=True if getArgument("DEBUG") == "DEBUG" else False
+print(f'Parameters[{CONF}]DEBUG={DEBUG}')
+cvm = root.new(spark, CONF, DEBUG)
+print(cvm.yaml['env'])
+
+# COMMAND ----------
+
+#if not DEBUG:
cvm.run()
print(cvm.sizes)
+cvm.do_save()
+
+dbutils.notebook.exit(0)
# COMMAND ----------
-K = list(cvm.df.keys())
-print(K)
-def d(i): return cvm.df[K[i]]
-def view(i):
- print(K[i])
- d(i).show()
-def values(i, col): return d(i).select(col).distinct().collect()
-view(-1)
+cvm.init()
+steps = cvm.steps()
+print(steps)
+#dbutils.notebook.exit(0)
+
# COMMAND ----------
-cvm.do_save({})
-displayHTML(cvm.pkg.html)
+df = {}
+def values(s, col): return df[s].select(col).distinct().collect()
+
+for step in steps:
+ print(step)
+ df[step] = cvm.test(step)
+ df[step].show()
# COMMAND ----------
+
+cvm.save()
+displayHTML(cvm.result())
+
+# COMMAND ----------
+
dbutils.notebook.exit(0)
#spark.sql('create database nauto')
diff --git a/tests/db_mock.py b/tests/db_mock.py
index 66db241..6e33195 100644
--- a/tests/db_mock.py
+++ b/tests/db_mock.py
@@ -10,7 +10,7 @@ def get_items(self): return [self]
class MockCol(object):
def __init__(self, name): self.name = name
- def alias(self, name,metadata={"meta":"data"}): return name
+ def alias(self, name,metadata={"env":"data"}): return name
def contains(self, value): return True
def desc(self): return True
@@ -70,7 +70,7 @@ def withColumn(self, *arg): return self
class MockSpark(object):
def __init__(self): self.columns = []
- def createDataFrame(self, list): return self.table('list')
+ def createDataFrame(self, list, schema=[]): return self.table('list')
def count(self): return self
def distinct(self,arg=None): return self
def get(self, arg): return f'mock.get:{arg}'
diff --git a/tests/pkg_test.py b/tests/pkg_test.py
index eed5093..6130436 100644
--- a/tests/pkg_test.py
+++ b/tests/pkg_test.py
@@ -1,14 +1,29 @@
#!/usr/bin/env python3
import pytest
-from .context import cqml, TEST_YAML
+from .context import *
from .db_mock import spark
-def test_pkg():
- dict = cqml.pkg_cqml('cqml_test', spark, 'tests')
+@pytest.fixture
+def root():
+ root = cqml.Root("pipes")
+ return root
+
+def test_pkg(root):
+ dict = root.pkg(spark, TEST_KEY, True)
+ assert 'pkg' in dict
+ assert 'html' in dict
+ assert 'actions' in dict
+
+def test_pkg_demo(root):
+ dict = root.pkg(spark, TEST_DEMO, True)
assert 'pkg' in dict
assert 'html' in dict
assert 'actions' in dict
-def skip_test_all():
- dict = cqml.pkg_all(spark, 'tests')
- assert 'cqml_test' in dict
+def test_box(root):
+ cvm = root.new(spark, TEST_KEY)
+ cvm.test_id(DDIR)
+ it = cvm.test_id("box_details")
+ assert it
+
+#def test_all(root): dict = root.pkg(spark, 'pipes/all', True)
diff --git a/tests/root_test.py b/tests/root_test.py
new file mode 100644
index 0000000..96992b5
--- /dev/null
+++ b/tests/root_test.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+import pytest
+from .context import cqml, TEST_KEY
+from .db_mock import spark
+
+@pytest.fixture
+def root():
+ root = cqml.Root("pipes")
+ return root
+
+def test_root(root):
+ assert root
+
+def test_keys(root):
+ keys = root.keys()
+ assert keys
+ assert keys[0]
+ assert TEST_KEY in keys
+
+def test_new(root):
+ cvm = root.new(spark, TEST_KEY)
+ assert cvm
+ assert "test" == cvm.log("test")
+ yml = cvm.yaml
+ assert 'env' in yml
+ assert 'org' in yml['env']
+
+def test_demo(root):
+ cvm = root.new(spark, 'demo/demo')
+ yml = cvm.yaml
+ assert 'env' in yml
+ assert 'org' in yml['env']
+ assert 'nauto' in yml['env']['org']
diff --git a/version.txt b/version.txt
index 93ee737..da07663 100644
--- a/version.txt
+++ b/version.txt
@@ -1 +1 @@
-0.4.6.dev0
+0.5.0.dev24
diff --git a/version.txt.prev b/version.txt.prev
index 8ebd7c0..11c0430 100644
--- a/version.txt.prev
+++ b/version.txt.prev
@@ -1 +1 @@
-0.4.4.dev1
+0.5.0.dev23