From db501cd5da7ef3bf2fbf762a8b04f7930ed21a6d Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Sat, 30 Oct 2021 01:23:34 +0200 Subject: [PATCH 1/4] Updated README indicating Python 3.5 compatibility Signed-off-by: Ole Herman Schumacher Elgesem --- README.md | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0a5ebac1..07a00bfd 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The modules you use can be written by the CFEngine team, others in the community ## Installation -Requires Python 3.6 or newer and `pip`. +Requires Python 3.5 or newer and `pip`. ``` pip install cfbs @@ -30,12 +30,10 @@ pip install cf-remote ### Dependencies -`cfbs` is implemented in Python and has some dependencies on python version and libraries: - -* Python 3.6 or newer - -Additionally, some command line tools are required (not installed by pip): +`cfbs` is implemented in Python and has a few dependencies: +* Python 3.5 or newer +* `requests` python library * `git` CLI installed and in PATH ## Usage From 610a760b659edae715d17d3394463c1daef4ad7c Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Sat, 30 Oct 2021 02:18:14 +0200 Subject: [PATCH 2/4] Fixed dependencies for modules added by URL Signed-off-by: Ole Herman Schumacher Elgesem --- cfbs/commands.py | 2 +- cfbs/index.py | 57 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 16 deletions(-) diff --git a/cfbs/commands.py b/cfbs/commands.py index 3802dc1d..afd749db 100644 --- a/cfbs/commands.py +++ b/cfbs/commands.py @@ -529,7 +529,7 @@ def _add_using_url( modules = [provides[k] for k in to_add] for module in modules: - config.add(module) + config.add(module, remote_config) return 0 diff --git a/cfbs/index.py b/cfbs/index.py index e6894493..4cfeec3f 100644 --- a/cfbs/index.py +++ b/cfbs/index.py @@ -118,12 +118,12 @@ def get_modules(self) -> dict: def exists(self, module): return os.path.exists(module) or (module in self) - def get_build_step(self, module): - return ( - self.get_modules()[module] - if not module.startswith("./") - else generate_index_for_local_module(module) - ) + def get_build_step(self, name): + if name.startswith("./"): + return generate_index_for_local_module(name) + + module = OrderedDict({"name": name}) + module.update(self.get_modules()[name]) def _expand_index(thing): @@ -132,6 +132,16 @@ def _expand_index(thing): return get_or_read_json(thing)["index"] return thing +def _construct_provided_module(name, data, url): + module = OrderedDict() + module["name"] = name + module["description"] = data["description"] + module["provided_by"] = url + dependencies = data.get("dependencies") + if dependencies: + module["dependencies"] = dependencies + module["steps"] = data["steps"] + return module class CFBSConfig: def __init__(self, index_argument=None, path="./cfbs.json", data=None, url=None): @@ -164,22 +174,28 @@ def using_default_index(self): return self._unexpanded_index == self._default_index def __getitem__(self, key): - if key == "index": - return self.index + assert key != "index" return self._data[key] + def __contains__(self, key): + return key in self._data + def get_provides(self): print(pretty(self._data)) modules = OrderedDict() for k, v in self._data["provides"].items(): - module = OrderedDict() - module["name"] = k - module["description"] = v["description"] - module["provided_by"] = self.url - module["steps"] = v["steps"] + module = _construct_provided_module(k, v, self.url) modules[k] = module return modules + def get_module_for_build(self, name, dependent): + if "provides" in self._data and name in self._data["provides"]: + module = self._data["provides"][name] + return _construct_provided_module(name, module, self.url) + if name in self.index: + return self.index.get_build_step(name) + return None + def save(self, data=None): if data: if type(data) is CFBSConfig: @@ -191,12 +207,23 @@ def save(self, data=None): def _module_is_in_build(self, module): return module["name"] in (m["name"] for m in self["build"]) - def add(self, module): + def add(self, module, remote_config=None, dependent=None): + if type(module) is str: + module_str = module + module = (remote_config or self).get_module_for_build(module, dependent) + if not module: + user_error("Module '%s' not found" % module_str) assert "name" in module assert "steps" in module if self._module_is_in_build(module): print("Skipping already added module '%s'" % module["name"]) return + if "dependencies" in module: + for dep in module["dependencies"]: + self.add(dep, remote_config, dependent) self._data["build"].append(module) self.save() - print("Added module: %s" % module["name"]) + if dependent: + print("Added module: %s (Dependency of %s)" % (module["name"], dependent)) + else: + print("Added module: %s" % module["name"]) From 8891f3ce7725305d4a72d47eb583753b58d64040 Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Sat, 30 Oct 2021 02:32:02 +0200 Subject: [PATCH 3/4] Fixed small mistakes introduced in recent commits Signed-off-by: Ole Herman Schumacher Elgesem --- cfbs/index.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cfbs/index.py b/cfbs/index.py index 4cfeec3f..fc94a5a3 100644 --- a/cfbs/index.py +++ b/cfbs/index.py @@ -124,7 +124,7 @@ def get_build_step(self, name): module = OrderedDict({"name": name}) module.update(self.get_modules()[name]) - + return module def _expand_index(thing): assert type(thing) in (dict, list, str) @@ -137,6 +137,9 @@ def _construct_provided_module(name, data, url): module["name"] = name module["description"] = data["description"] module["provided_by"] = url + subdirectory = data.get("subdirectory") + if subdirectory: + module["subdirectory"] = subdirectory dependencies = data.get("dependencies") if dependencies: module["dependencies"] = dependencies @@ -181,7 +184,6 @@ def __contains__(self, key): return key in self._data def get_provides(self): - print(pretty(self._data)) modules = OrderedDict() for k, v in self._data["provides"].items(): module = _construct_provided_module(k, v, self.url) @@ -220,7 +222,7 @@ def add(self, module, remote_config=None, dependent=None): return if "dependencies" in module: for dep in module["dependencies"]: - self.add(dep, remote_config, dependent) + self.add(dep, remote_config, module["name"]) self._data["build"].append(module) self.save() if dependent: From 512aa7c478ce1d5a74e1b99c70847409e3ad09f2 Mon Sep 17 00:00:00 2001 From: Ole Herman Schumacher Elgesem Date: Sat, 30 Oct 2021 02:57:57 +0200 Subject: [PATCH 4/4] Fixed confusion around url / repo Signed-off-by: Ole Herman Schumacher Elgesem --- JSON.md | 5 +---- cfbs/commands.py | 13 +++++++------ cfbs/index.py | 22 +++++++++++++++++----- 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/JSON.md b/JSON.md index b87d3d4b..e5f1adc4 100644 --- a/JSON.md +++ b/JSON.md @@ -138,10 +138,8 @@ If you put your modules in a repo and don't have them in the index (yet), you ca "type": "module", "provides": { "example-module": { - "name": "example-module", "description": "Just an example", "tags": ["local"], - "version": "1.0.0", "dependencies": ["autorun"], "steps": ["copy ./test.cf services/autorun/test.cf"] } @@ -165,8 +163,7 @@ cfbs init && cfbs add https://github.com/cfengine/some-repo "name": "example-module", "description": "Just an example", "tags": ["local"], - "repo": "https://github.com/cfengine/some-repo", - "version": "1.0.0", + "url": "https://github.com/cfengine/some-repo", "commit": "be3bc015f6a19e945bb7a9fa0ed78c97e2cecf61", "dependencies": ["autorun"], "steps": ["copy ./test.cf services/autorun/test.cf"], diff --git a/cfbs/commands.py b/cfbs/commands.py index afd749db..e94ae068 100644 --- a/cfbs/commands.py +++ b/cfbs/commands.py @@ -501,14 +501,14 @@ def _add_using_url( checksum=None, non_interactive=False, ): - url_repo_commit = None + url_commit = None if url.endswith(_SUPPORTED_ARCHIVES): - config_path, url_repo_commit = _fetch_archive(url, checksum) + config_path, url_commit = _fetch_archive(url, checksum) else: assert url.startswith(("https://", "git://", "ssh://")) - config_path, url_repo_commit = _clone_url_repo(url) + config_path, url_commit = _clone_url_repo(url) - remote_config = CFBSConfig(path=config_path, url=url) + remote_config = CFBSConfig(path=config_path, url=url, url_commit=url_commit) config = CFBSConfig(index_argument=index_path) provides = remote_config.get_provides() @@ -726,7 +726,7 @@ def get_download_path(module) -> str: if not is_a_commit_hash(commit): user_error("'%s' is not a commit reference" % commit) - url = module["repo"] + url = module.get("url") or module["repo"] if url.endswith(_SUPPORTED_ARCHIVES): url = os.path.dirname(url) else: @@ -753,7 +753,8 @@ def download_dependencies(prefer_offline=False, redownload=False): if not is_a_commit_hash(commit): user_error("'%s' is not a commit reference" % commit) - url = strip_right(module["repo"], ".git") + url = module.get("url") or module["repo"] + url = strip_right(url, ".git") commit_dir = get_download_path(module) if redownload: rm(commit_dir, missing_ok=True) diff --git a/cfbs/index.py b/cfbs/index.py index fc94a5a3..509e7d03 100644 --- a/cfbs/index.py +++ b/cfbs/index.py @@ -126,17 +126,20 @@ def get_build_step(self, name): module.update(self.get_modules()[name]) return module + def _expand_index(thing): assert type(thing) in (dict, list, str) if type(thing) is str: return get_or_read_json(thing)["index"] return thing -def _construct_provided_module(name, data, url): + +def _construct_provided_module(name, data, url, url_commit): module = OrderedDict() module["name"] = name module["description"] = data["description"] - module["provided_by"] = url + module["url"] = url + module["commit"] = url_commit subdirectory = data.get("subdirectory") if subdirectory: module["subdirectory"] = subdirectory @@ -146,10 +149,19 @@ def _construct_provided_module(name, data, url): module["steps"] = data["steps"] return module + class CFBSConfig: - def __init__(self, index_argument=None, path="./cfbs.json", data=None, url=None): + def __init__( + self, + index_argument=None, + path="./cfbs.json", + data=None, + url=None, + url_commit=None, + ): self.path = path self.url = url + self.url_commit = url_commit if data: self._data = data else: @@ -186,14 +198,14 @@ def __contains__(self, key): def get_provides(self): modules = OrderedDict() for k, v in self._data["provides"].items(): - module = _construct_provided_module(k, v, self.url) + module = _construct_provided_module(k, v, self.url, self.url_commit) modules[k] = module return modules def get_module_for_build(self, name, dependent): if "provides" in self._data and name in self._data["provides"]: module = self._data["provides"][name] - return _construct_provided_module(name, module, self.url) + return _construct_provided_module(name, module, self.url, self.url_commit) if name in self.index: return self.index.get_build_step(name) return None