Skip to content

Commit

Permalink
simplify shell for dbr104 setup
Browse files Browse the repository at this point in the history
  • Loading branch information
R7L208 committed Apr 28, 2024
1 parent 4f18dc0 commit e4e8f6a
Show file tree
Hide file tree
Showing 6 changed files with 20 additions and 24 deletions.
1 change: 1 addition & 0 deletions python/install_cmd.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pip install --no-binary pyarrow,pandas,scipy {opts} {packages}
1 change: 1 addition & 0 deletions python/requirements/dbr104/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
install_cmd.txt
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
numpy~=1.20.1
delta-spark~=1.1.0
ipython~=7.22.0
numpy~=1.20.1
pandas~=1.2.4
pyarrow~=4.0.0
pyspark~=3.2.1
Expand Down
10 changes: 10 additions & 0 deletions python/requirements/dbr104/set_install_command.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
#!/bin/bash
# Based on architecture, set an appropriate install command
SYS_ARCH=$(uname -m)
export SYS_ARCH
if [ "$SYS_ARCH" = "arm64" ]; then
NO_BINARY_PACKAGES="pyarrow,pandas,scipy"
echo "pip install --no-binary $NO_BINARY_PACKAGES {opts} {packages}" > install_cmd.txt
else
echo "pip install {opts} {packages}" > install_cmd.txt
fi
7 changes: 0 additions & 7 deletions python/requirements/dbr104_arm64.txt

This file was deleted.

23 changes: 7 additions & 16 deletions python/tox.ini
Original file line number Diff line number Diff line change
Expand Up @@ -30,25 +30,16 @@ commands =
coverage run -m unittest discover -s tests -p '*_tests.py'

[testenv:dbr104]
allowlist_externals = bash
commands_pre =
bash -c "./requirements/{envname}/set_install_command.sh"
install_command =
bash -c "cat requirements/{envname}/install_cmd.txt"
deps =
-rrequirements/dev.txt
;NB: dependency order matters for this env
-rrequirements/{envname}/{envname}.txt
commands =
# Get the architecture of the system
sys_arch=$(uname -m)
echo "System Architecture: $sys_arch"
# Check if the architecture is arm64
if [ "sys_arch" = "arm64" ]; then
;Building with no dependencies because NumPy<=1.20 is not supported on
;MacOS with arm64 arch. The Numpy version is pinned in the requirements.txt to
;1.20 to avoid the issue, and is compatible with the other dependencies.
;https://github.com/apache/arrow/blob/release-4.0.0/python/requirements-wheel-build.txt
;https://pandas.pydata.org/pandas-docs/version/1.2/getting_started/install.html#dependencies
;https://docs.scipy.org/doc/scipy/dev/toolchain.html#numpy
;NB: Installation order matters in the requirements file
pip install --no-deps -r requirements/{envname}_${sys_arch}.txt
else
pip install -r requirements/{envname}.txt
fi
coverage erase
coverage run -m unittest discover -s tests -p '*_tests.py'

Expand Down

0 comments on commit e4e8f6a

Please sign in to comment.