Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Update from upstream: #54

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Update from upstream: #54

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
2ec4fd2
Update from upstream:
romain-intel Dec 6, 2024
6c9b173
Add import of parsers at top level
romain-intel Dec 6, 2024
aadce54
Fixes and more compatibility with Mamba 2.0
romain-intel Dec 10, 2024
19096c7
Typos
romain-intel Dec 10, 2024
654f10a
More fixes from upstream
romain-intel Dec 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion metaflow_extensions/netflix_ext/cmd/debug/debug_stub_generator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@ def __init__(self, task_pathspec):
self.run_id = self.task.parent.parent.id
self.flow_name = self.task.parent.parent.parent.id
self.is_new_conda_step = self.is_new_conda_step()
self.workflow_dag = self.task["_graph_info"].data

parameters_task = Step(
"%s/_parameters" % self.task.parent.parent.pathspec, _namespace_check=False
).task

self.workflow_dag = parameters_task["_graph_info"].data
self.file_name = self.workflow_dag["file"]
self._dag_structure = self.get_dag_structure(self.workflow_dag["steps"])
self.step_type = self.get_step_type(self.step_name)
Expand Down
190 changes: 7 additions & 183 deletions metaflow_extensions/netflix_ext/cmd/environment/environment_cmd.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
import json
import os
import platform
import re
import shutil
import subprocess
import tempfile
Expand All @@ -23,7 +22,6 @@
CONDA_ALL_ARCHS,
CONDA_DEPENDENCY_RESOLVER,
CONDA_TEST,
CONDA_SYS_DEPENDENCIES,
DEFAULT_DATASTORE,
DEFAULT_METADATA,
get_pinned_conda_libs,
Expand All @@ -40,6 +38,11 @@
env_type_for_deps,
)
from metaflow_extensions.netflix_ext.plugins.conda.envsresolver import EnvsResolver

from metaflow_extensions.netflix_ext.plugins.conda.parsers import (
parse_req_value,
parse_yml_value,
)
from metaflow_extensions.netflix_ext.plugins.conda.utils import (
AliasType,
arch_id,
Expand All @@ -52,24 +55,12 @@
tstr_to_dict,
)

from metaflow._vendor.packaging.requirements import InvalidRequirement, Requirement
from metaflow._vendor.packaging.specifiers import SpecifierSet
from metaflow._vendor.packaging.utils import canonicalize_version

from .utils import download_mf_version


REQ_SPLIT_LINE = re.compile(r"([^~<=>]*)([~<=>]+.*)?")

# Allows things like:
# pkg = <= version
# pkg <= version
# pkg = version
# pkg = ==version or pkg = =version
# In other words, the = is optional but possible
YML_SPLIT_LINE = re.compile(r"(?:=\s)?(<=|>=|~=|==|<|>|=)")


class CommandObj:
def __init__(self):
pass
Expand Down Expand Up @@ -1110,92 +1101,8 @@ def _parse_req_file(
np_deps: Dict[str, str],
sys_deps: Dict[str, str],
) -> Optional[str]:
python_version = None
with open(file_name, mode="r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
splits = line.split(maxsplit=1)
first_word = splits[0]
if len(splits) > 1:
rem = splits[1]
else:
rem = None
if first_word in ("-i", "--index-url"):
raise InvalidEnvironmentException(
"To specify a base PYPI index, set `METAFLOW_CONDA_DEFAULT_PYPI_SOURCE; "
"you can specify additional indices using --extra-index-url"
)
elif first_word == "--extra-index-url" and rem:
sources.setdefault("pypi", []).append(rem)
elif first_word in ("-f", "--find-links", "--trusted-host") and rem:
extra_args.setdefault("pypi", []).append(" ".join([first_word, rem]))
elif first_word in ("--pre", "--no-index"):
extra_args.setdefault("pypi", []).append(first_word)
elif first_word == "--conda-channel" and rem:
sources.setdefault("conda", []).append(rem)
elif first_word == "--conda-pkg":
# Special extension to allow non-python conda package specification
split_res = REQ_SPLIT_LINE.match(splits[1])
if split_res is None:
raise InvalidEnvironmentException(
"Could not parse conda package '%s'" % splits[1]
)
s = split_res.groups()
if s[1] is None:
np_deps[s[0].replace(" ", "")] = ""
else:
np_deps[s[0].replace(" ", "")] = s[1].replace(" ", "").lstrip("=")
elif first_word == "--sys-pkg":
# Special extension to allow the specification of system dependencies
# (currently __cuda and __glibc)
split_res = REQ_SPLIT_LINE.match(splits[1])
if split_res is None:
raise InvalidEnvironmentException(
"Could not parse system package '%s'" % splits[1]
)
s = split_res.groups()
pkg_name = s[0].replace(" ", "")
if pkg_name not in CONDA_SYS_DEPENDENCIES:
raise InvalidEnvironmentException(
"System package '%s' not allowed. Values allowed are: %s"
% (pkg_name, str(CONDA_SYS_DEPENDENCIES))
)
if s[1] is None:
raise InvalidEnvironmentException(
"System package '%s' requires a version" % pkg_name
)
sys_deps[pkg_name] = s[1].replace(" ", "").lstrip("=")
elif first_word.startswith("#"):
continue
elif first_word.startswith("-"):
raise InvalidEnvironmentException(
"'%s' is not a supported line in a requirements.txt" % line
)
else:
try:
parsed_req = Requirement(line)
except InvalidRequirement as ex:
raise InvalidEnvironmentException(
"Could not parse '%s'" % line
) from ex
if parsed_req.marker is not None:
raise InvalidEnvironmentException(
"Environment markers are not supported for '%s'" % line
)
dep_name = parsed_req.name
if parsed_req.extras:
dep_name += "[%s]" % ",".join(parsed_req.extras)
if parsed_req.url:
dep_name += "@%s" % parsed_req.url
specifier = str(parsed_req.specifier).lstrip(" =")
if dep_name == "python":
if specifier:
python_version = specifier
else:
deps[dep_name] = specifier
return python_version
return parse_req_value(f.read(), extra_args, sources, deps, np_deps, sys_deps)


def _parse_yml_file(
Expand All @@ -1206,91 +1113,8 @@ def _parse_yml_file(
pypi_deps: Dict[str, str],
sys_deps: Dict[str, str],
) -> Optional[str]:
python_version = None # type: Optional[str]
with open(file_name, mode="r", encoding="utf-8") as f:
# Very poor man's yaml parsing
mode = None
for line in f:
if not line:
continue
elif line[0] not in (" ", "-"):
line = line.strip()
if line == "channels:":
mode = "sources"
elif line == "dependencies:":
mode = "deps"
elif line == "pypi-indices:":
mode = "pypi_sources"
else:
mode = "ignore"
elif mode and mode.endswith("sources"):
line = line.lstrip(" -").rstrip()
sources.setdefault("conda" if mode == "sources" else "pypi", []).append(
line
)
elif mode and mode.endswith("deps"):
line = line.lstrip(" -").rstrip()
if line == "pip:":
mode = "pypi_deps"
elif line == "sys:":
mode = "sys_deps"
else:
to_update = (
conda_deps
if mode == "deps"
else pypi_deps if mode == "pypi_deps" else sys_deps
)
splits = YML_SPLIT_LINE.split(line.replace(" ", ""), maxsplit=1)
if len(splits) == 1:
if splits[0] != "python":
if mode == "sys_deps":
raise InvalidEnvironmentException(
"System package '%s' requires a version" % splits[0]
)
to_update[splits[0]] = ""
else:
dep_name, dep_operator, dep_version = splits
if dep_operator not in ("=", "=="):
if mode == "sys_deps":
raise InvalidEnvironmentException(
"System package '%s' requires a specific version not '%s'"
% (splits[0], dep_operator + dep_version)
)
dep_version = dep_operator + dep_version
if dep_name == "python":
if dep_version:
if python_version:
raise InvalidEnvironmentException(
"Python versions specified multiple times in "
"the YAML file."
)
python_version = dep_version
else:
if (
dep_name.startswith("/")
or dep_name.startswith("git+")
or dep_name.startswith("https://")
or dep_name.startswith("ssh://")
):
# Handle the case where only the URL is specified
# without a package name
depname_and_maybe_tag = dep_name.split("/")[-1]
depname = depname_and_maybe_tag.split("@")[0]
if depname.endswith(".git"):
depname = depname[:-4]
dep_name = "%s@%s" % (depname, dep_name)

if (
mode == "sys_deps"
and dep_name not in CONDA_SYS_DEPENDENCIES
):
raise InvalidEnvironmentException(
"System package '%s' not allowed. Values allowed are: %s"
% (dep_name, str(CONDA_SYS_DEPENDENCIES))
)
to_update[dep_name] = dep_version

return python_version
return parse_yml_value(f.read(), {}, sources, conda_deps, pypi_deps, sys_deps)


# @environment.command(help="List resolved environments for a set of dependencies")
Expand Down
Loading
Loading