From 4c0043da56bfc75c0fc07b1648ec1ca539e244ac Mon Sep 17 00:00:00 2001 From: Theophile du Laz Date: Thu, 14 Sep 2023 16:19:04 -0700 Subject: [PATCH] No macos specific python dependencies (#241) TensorFlow itself is now available for Mac arm64 chips. No need to have separate dependencies and commands for Mac. --- Makefile | 16 --------- README.md | 31 +--------------- requirements/requirements_ingester.txt | 2 +- requirements/requirements_ingester_macos.txt | 37 -------------------- 4 files changed, 2 insertions(+), 84 deletions(-) delete mode 100644 requirements/requirements_ingester_macos.txt diff --git a/Makefile b/Makefile index 2442913b..feb3da56 100644 --- a/Makefile +++ b/Makefile @@ -35,9 +35,6 @@ python_dependencies_api: python_dependencies_ingester: $(PYTHON) kowalski/tools/pip_install_requirements.py requirements/requirements_ingester.txt -python_dependencies_ingester_macos: - $(PYTHON) kowalski/tools/pip_install_requirements.py requirements/requirements_ingester_macos.txt - # SUPERVISORD generate_supervisord_conf: $(PYTHON) kowalski/tools/generate_supervisord_conf.py all @@ -56,12 +53,8 @@ setup_api: system_dependencies python_dependencies_api generate_supervisord_api_ setup_ingester: system_dependencies python_dependencies_ingester generate_supervisord_ingester_conf -setup_ingester_macos: system_dependencies python_dependencies_ingester_macos generate_supervisord_ingester_conf - setup_all: setup python_dependencies_api python_dependencies_ingester generate_supervisord_conf -setup_all_macos: setup python_dependencies_api python_dependencies_ingester_macos generate_supervisord_conf - init_models: $(PYTHON) kowalski/tools/init_models.py @@ -71,18 +64,12 @@ init_kafka: run: setup_all init_models init_kafka $(SUPERVISORD) -run_macos : setup_all_macos init_models init_kafka - $(SUPERVISORD) - run_api: setup_api $(SUPERVISORD_API) run_ingester: setup_ingester init_models init_kafka $(SUPERVISORD_INGESTER) -run_ingester_macos: setup_ingester_macos init_models init_kafka - $(SUPERVISORD_INGESTER) - monitor: $(SUPERVISORCTL) -i @@ -95,9 +82,6 @@ monitor_ingester: test: setup_all init_models init_kafka $(PYTHON) kowalski/tools/tests.py -test_macos: setup_all_macos init_models init_kafka - $(PYTHON) kowalski/tools/tests.py - docker_setup: setup $(PYTHON) kowalski/tools/docker.py setup $(FLAGS) diff --git a/README.md b/README.md index 0f680342..1ff5b6a2 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,6 @@ source env/bin/activate to create your virtual environment. If you are told that pip is not found, try using pip3 instead. For the following steps however (in the virtualenv), pip should work. -*We do not suggest using conda, as we experienced trouble installing arm64 specific packages like tensorflow-macos and tensor flow-metal.* The python dependencies will be install automatically when you start the app. The same will happen for Kafka and the ML models. @@ -127,8 +126,6 @@ source env/bin/activate to create your virtual environment. If you are told that pip is not found, try using pip3 instead. For the following steps however (in the virtualenv), pip should work. -*We do not suggest using conda, as we experienced trouble installing arm64 specific packages like tensorflow-macos and tensor flow-metal.* - The python dependencies will be install automatically when you start the app. The same will happen for Kafka and the ML models. ## Spin up your own `kowalski` **without Docker** @@ -169,7 +166,7 @@ exit If you are using a mongodb atlas cluster, kowalski won't be able to create admin users, so you will need to do so manually on the cluster's web interface. You will need to create 2 users: admin user and user, based on what usernames and passwords you've set in the config file in the `database` section. *Don't forget to also allow access from your IP address, or simply allow access from anywhere.* -### Start and test the app **on Linux amd64** +### Start and test the app #### Starting the app @@ -195,32 +192,6 @@ If you want to run a specific test, you can do so by running: PYTHONPATH=. pytest -s kowalski/tests/ ``` -### Start and test the app **on MacOS arm64 (M1 or M2)** - -#### Starting the app - -To start the app, run: - -```bash -make run_macos -``` - -This will start the API, the dask clusters and alert brokers. - -#### Run the tests - -To run the tests, run: - -```bash -make test_macos -``` - -If you want to run a specific test, you can do so by running: - -```bash -PYTHONPATH=. pytest -s kowalski/tests/ -``` - ### Ingester (Pushing alerts to a local kafka topic) Once the broker is running, you might want to create a local kafka stream of alerts to test it. To do so, you can run the ingester with diff --git a/requirements/requirements_ingester.txt b/requirements/requirements_ingester.txt index 7cc86142..c008b69b 100644 --- a/requirements/requirements_ingester.txt +++ b/requirements/requirements_ingester.txt @@ -31,6 +31,6 @@ requests==2.31.0 slack_sdk==3.20.0 supervisor==4.2.5 tables==3.8.0 -tensorflow==2.12.0 +tensorflow==2.13.0 tqdm==4.64.1 uvloop==0.17.0 diff --git a/requirements/requirements_ingester_macos.txt b/requirements/requirements_ingester_macos.txt deleted file mode 100644 index 191701fa..00000000 --- a/requirements/requirements_ingester_macos.txt +++ /dev/null @@ -1,37 +0,0 @@ -aiohttp==3.8.4 -astropy==5.2.1 -bcrypt==4.0.1 -beautifulsoup4==4.11.2 -bokeh==2.4.3 -confluent-kafka==2.1.1 -cryptography==41.0.1 -dask==2023.7.0 -deepdiff==6.2.3 -distributed==2023.7.0 -fastavro==1.6.0 #breaking changes in newer versions -fire==0.5.0 -gsutil==5.20 -gunicorn==20.1.0 -h5py==3.8.0 -jinja2>=2.11.2 -matplotlib==3.7.0 -motor==3.1.1 -numba==0.56.4 -numpy==1.23.5 -pandas==1.5.3 -protobuf==4.22.1 -pyarrow==11.0.0 -PyJWT==2.6.0 -pymongo==4.4.0 -pyopenssl==23.2.0 -pytest==7.2.1 -pytz==2022.7.1 -PyYAML==6.0 -requests==2.31.0 -slack_sdk==3.20.0 -supervisor==4.2.5 -tables==3.8.0 -tensorflow-macos==2.12.0 -tensorflow-metal==0.8.0 -tqdm==4.64.1 -uvloop==0.17.0