From 67b53bc1f0071ad926048bc537aae4a707424d44 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 09:41:16 -0300 Subject: [PATCH 1/9] add a install+activate jl env function --- install_julia_packages.jl | 5 -- setup.py | 26 -------- xbitinfo/julia_helpers.py | 127 ++++++++++++++++++++++++++++++++++++++ xbitinfo/xbitinfo.py | 6 ++ 4 files changed, 133 insertions(+), 31 deletions(-) delete mode 100644 install_julia_packages.jl create mode 100644 xbitinfo/julia_helpers.py diff --git a/install_julia_packages.jl b/install_julia_packages.jl deleted file mode 100644 index da55c94e..00000000 --- a/install_julia_packages.jl +++ /dev/null @@ -1,5 +0,0 @@ -import Pkg -Pkg.add("PyCall") -Pkg.add("BitInformation") -Pkg.add("StatsBase") -Pkg.status() diff --git a/setup.py b/setup.py index 5325fcd9..46246172 100644 --- a/setup.py +++ b/setup.py @@ -2,30 +2,8 @@ """The setup script.""" -import os from setuptools import find_packages, setup -from setuptools.command.develop import develop -from setuptools.command.install import install - -julia_install_command = "julia install_julia_packages.jl" - - -class PostDevelopCommand(develop): - """Post-installation for development mode.""" - - def run(self): - develop.run(self) - os.system(julia_install_command) - - -class PostInstallCommand(install): - """Post-installation for installation mode.""" - - def run(self): - install.run(self) - os.system(julia_install_command) - with open("README.md") as readme_file: readme = readme_file.read() @@ -66,10 +44,6 @@ def run(self): "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", ], - cmdclass={ - "develop": PostDevelopCommand, - "install": PostInstallCommand, - }, description="Retrieve information content and compress accordingly.", install_requires=requirements, license="MIT license", diff --git a/xbitinfo/julia_helpers.py b/xbitinfo/julia_helpers.py new file mode 100644 index 00000000..f444d4c5 --- /dev/null +++ b/xbitinfo/julia_helpers.py @@ -0,0 +1,127 @@ +"""Functions for initializing the Julia environment and installing deps.""" +import warnings +from pathlib import Path +import os + +from ._version import __version__ + + +def install(julia_project=None, quiet=False): # pragma: no cover + """ + Install PyCall.jl and all required dependencies for xbitinfo. + + Also updates the local Julia registry. + """ + import julia + + julia.install(quiet=quiet) + + julia_project, is_shared = _get_julia_project(julia_project) + + Main = init_julia() + Main.eval("using Pkg") + + io = "devnull" if quiet else "stderr" + io_arg = f"io={io}" if is_julia_version_greater_eq(Main, "1.6") else "" + + # Can't pass IO to Julia call as it evaluates to PyObject, so just directly + # use Main.eval: + Main.eval( + f'Pkg.activate("{_escape_filename(julia_project)}", shared = Bool({int(is_shared)}), {io_arg})' + ) + if is_shared: + _add_to_julia_project(Main, io_arg) + + Main.eval(f"Pkg.instantiate({io_arg})") + Main.eval(f"Pkg.precompile({io_arg})") + if not quiet: + warnings.warn( + "It is recommended to restart Python so that the Julia environment is properly initialized." + ) + already_ran = True + return already_ran + + +def import_error_string(julia_project=None): + s = """ + Required dependencies are not installed or built. Run the following code in the Python REPL: + + >>> import xbitinfo + >>> xbitinfo.install() + """ + + if julia_project is not None: + s += f""" + Tried to activate project {julia_project} but failed.""" + + return s + + +def _get_julia_project(julia_project): + if julia_project is None: + is_shared = True + julia_project = f"xbitinfo-{__version__}" + else: + is_shared = False + julia_project = Path(julia_project) + return julia_project, is_shared + + +def is_julia_version_greater_eq(Main, version="1.6"): + """Check if Julia version is greater than specified version.""" + return Main.eval(f'VERSION >= v"{version}"') + + +def init_julia(): + """Initialize julia binary, turning off compiled modules if needed.""" + from julia.core import JuliaInfo, UnsupportedPythonError + + try: + info = JuliaInfo.load(julia="julia") + except FileNotFoundError: + env_path = os.environ["PATH"] + raise FileNotFoundError( + f"Julia is not installed in your PATH. Please install Julia and add it to your PATH.\n\nCurrent PATH: {env_path}", + ) + + if not info.is_pycall_built(): + raise ImportError(import_error_string()) + + Main = None + try: + from julia import Main as _Main + + Main = _Main + except UnsupportedPythonError: + # Static python binary, so we turn off pre-compiled modules. + from julia.core import Julia + + jl = Julia(compiled_modules=False) + from julia import Main as _Main + + Main = _Main + + return Main + + +def _add_to_julia_project(Main, io_arg): + Main.bitinformation_spec = Main.PackageSpec( + name="BitInformation", + url="https://github.com/milankl/BitInformation.jl", + rev="v0.6.0", + ) + Main.eval(f"Pkg.add(bitinformation_spec, {io_arg})") + + Main.statsbase_spec = Main.PackageSpec( + name="StatsBase", + url="https://github.com/JuliaStats/StatsBase.jl", + rev="v0.33.21", + ) + Main.eval(f"Pkg.add(statsbase_spec, {io_arg})") + + +def _escape_filename(filename): + """Turns a file into a string representation with correctly escaped backslashes""" + str_repr = str(filename) + str_repr = str_repr.replace("\\", "\\\\") + return str_repr \ No newline at end of file diff --git a/xbitinfo/xbitinfo.py b/xbitinfo/xbitinfo.py index 46660f63..b201d0fc 100644 --- a/xbitinfo/xbitinfo.py +++ b/xbitinfo/xbitinfo.py @@ -8,6 +8,12 @@ from tqdm.auto import tqdm from . import __version__ +from .julia_helpers import install + +already_ran = False +if not already_ran: + already_ran = install(quiet=True) + jl = Julia(compiled_modules=False, debug=False) from julia import Main # noqa: E402 From eae15b2abd3dbd6aa672f93aa6b4d1b45150f767 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 10:00:43 -0300 Subject: [PATCH 2/9] add upstream refs --- xbitinfo/julia_helpers.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/xbitinfo/julia_helpers.py b/xbitinfo/julia_helpers.py index f444d4c5..0a211e9b 100644 --- a/xbitinfo/julia_helpers.py +++ b/xbitinfo/julia_helpers.py @@ -1,4 +1,9 @@ """Functions for initializing the Julia environment and installing deps.""" + +# Lifted from: +# https://github.com/MilesCranmer/PySR/blob/master/pysr/julia_helpers.py +# https://github.com/MilesCranmer/PySR/blob/master/LICENSE + import warnings from pathlib import Path import os From 6c4b334f50003d70d82cc978e3ee5a7e210c93fb Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 10:05:47 -0300 Subject: [PATCH 3/9] pre-commit fixes --- xbitinfo/julia_helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xbitinfo/julia_helpers.py b/xbitinfo/julia_helpers.py index 0a211e9b..07fb50e3 100644 --- a/xbitinfo/julia_helpers.py +++ b/xbitinfo/julia_helpers.py @@ -4,9 +4,9 @@ # https://github.com/MilesCranmer/PySR/blob/master/pysr/julia_helpers.py # https://github.com/MilesCranmer/PySR/blob/master/LICENSE +import os import warnings from pathlib import Path -import os from ._version import __version__ @@ -101,7 +101,7 @@ def init_julia(): # Static python binary, so we turn off pre-compiled modules. from julia.core import Julia - jl = Julia(compiled_modules=False) + jl = Julia(compiled_modules=False) # noqa from julia import Main as _Main Main = _Main @@ -129,4 +129,4 @@ def _escape_filename(filename): """Turns a file into a string representation with correctly escaped backslashes""" str_repr = str(filename) str_repr = str_repr.replace("\\", "\\\\") - return str_repr \ No newline at end of file + return str_repr From 3fb2c7286f79a02371f9582956253e6e36895f9c Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 13:58:19 -0300 Subject: [PATCH 4/9] debug windows --- xbitinfo/xbitinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xbitinfo/xbitinfo.py b/xbitinfo/xbitinfo.py index b201d0fc..d20bd01d 100644 --- a/xbitinfo/xbitinfo.py +++ b/xbitinfo/xbitinfo.py @@ -12,7 +12,7 @@ already_ran = False if not already_ran: - already_ran = install(quiet=True) + already_ran = install(quiet=False) jl = Julia(compiled_modules=False, debug=False) From dc2f0b12de96391be9ad9bb2ac79e9a1aef074d7 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 14:05:02 -0300 Subject: [PATCH 5/9] sync the multiple dependency lists --- .github/workflows/benchmarks.yml | 2 -- .readthedocs.yaml | 2 -- docs/environment.yml | 2 +- environment.yml | 4 ++-- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml index 2bdcdffa..47ee16cb 100644 --- a/.github/workflows/benchmarks.yml +++ b/.github/workflows/benchmarks.yml @@ -37,8 +37,6 @@ jobs: which conda pip install asv pip install . - julia install_julia_packages.jl - julia -e "using Pkg; Pkg.status()" sudo apt-get update -y - name: Run benchmarks shell: bash -l {0} diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 8c18d70c..5861c5b1 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -10,8 +10,6 @@ build: - conda list - which python - which conda - - julia install_julia_packages.jl # https://docs.readthedocs.io/en/stable/config-file/v2.html # https://github.com/readthedocs/readthedocs.org/issues/1740 - - julia -e "import Pkg; Pkg.status()" conda: environment: docs/environment.yml diff --git a/docs/environment.yml b/docs/environment.yml index 21e65490..06a23210 100644 --- a/docs/environment.yml +++ b/docs/environment.yml @@ -22,7 +22,7 @@ dependencies: - sphinx-copybutton - sphinx-book-theme>=0.1.7 - myst-nb + - numcodecs>=0.10.0 - pip - pip: - - git+https://github.com/zarr-developers/numcodecs - -e ../. diff --git a/environment.yml b/environment.yml index 9dbe29a7..792bd8fc 100644 --- a/environment.yml +++ b/environment.yml @@ -27,8 +27,8 @@ dependencies: - sphinx-copybutton - sphinx-book-theme - myst-nb + - numcodecs>=0.10.0 + - pytest-lazy-fixture - pip - pip: - - numcodecs - - pytest-lazy-fixture - -e . From b59a3f50b9388616077f6c9c767a9a68509f7144 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 14:49:12 -0300 Subject: [PATCH 6/9] let's try this GHA for julia --- .github/workflows/ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e256830a..b0c9409d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -136,6 +136,10 @@ jobs: uses: actions/setup-python@v3 with: python-version: 3.8 + - name: "Set up Julia" + uses: julia-actions/setup-julia@v1.6.0 + with: + version: "1.7.1" - name: Install dependencies run: | pip install . From 7229a2c3e432ad99342bb98cb2505fe23fcd8c78 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Thu, 1 Sep 2022 16:07:08 -0300 Subject: [PATCH 7/9] make it quiet in the import --- xbitinfo/xbitinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xbitinfo/xbitinfo.py b/xbitinfo/xbitinfo.py index d20bd01d..790ab339 100644 --- a/xbitinfo/xbitinfo.py +++ b/xbitinfo/xbitinfo.py @@ -15,7 +15,7 @@ already_ran = install(quiet=False) -jl = Julia(compiled_modules=False, debug=False) +jl = Julia(compiled_modules=False, debug=True) from julia import Main # noqa: E402 path_to_julia_functions = os.path.join( From e19df5b8d137d25e3280a719d2691411d024bf35 Mon Sep 17 00:00:00 2001 From: Filipe Date: Tue, 6 Sep 2022 13:28:17 -0300 Subject: [PATCH 8/9] consolidate install calls Co-authored-by: Mark Kittisopikul --- xbitinfo/julia_helpers.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/xbitinfo/julia_helpers.py b/xbitinfo/julia_helpers.py index 07fb50e3..c3a66896 100644 --- a/xbitinfo/julia_helpers.py +++ b/xbitinfo/julia_helpers.py @@ -115,14 +115,12 @@ def _add_to_julia_project(Main, io_arg): url="https://github.com/milankl/BitInformation.jl", rev="v0.6.0", ) - Main.eval(f"Pkg.add(bitinformation_spec, {io_arg})") - Main.statsbase_spec = Main.PackageSpec( name="StatsBase", url="https://github.com/JuliaStats/StatsBase.jl", rev="v0.33.21", ) - Main.eval(f"Pkg.add(statsbase_spec, {io_arg})") + Main.eval(f"Pkg.add([bitinformation_spec, statsbase_spec], {io_arg})") def _escape_filename(filename): From 82fa5e370ecc77abe9eb24a5f7320f927588ed42 Mon Sep 17 00:00:00 2001 From: Filipe Fernandes Date: Tue, 6 Sep 2022 13:30:50 -0300 Subject: [PATCH 9/9] make installation silent again --- xbitinfo/xbitinfo.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xbitinfo/xbitinfo.py b/xbitinfo/xbitinfo.py index 790ab339..ee8739eb 100644 --- a/xbitinfo/xbitinfo.py +++ b/xbitinfo/xbitinfo.py @@ -12,7 +12,7 @@ already_ran = False if not already_ran: - already_ran = install(quiet=False) + already_ran = install(quiet=True) jl = Julia(compiled_modules=False, debug=True)