diff --git a/.dockerignore b/.dockerignore index c292b73..fef7f62 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,2 +1,3 @@ **/.DS_Store -.git \ No newline at end of file +.git +.venv \ No newline at end of file diff --git a/.gitignore b/.gitignore index e6aff51..7134b9e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,162 @@ .DS_Store +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ diff --git a/.vscode/settings.json b/.vscode/settings.json index 85ff09c..ed274e5 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,30 +5,38 @@ "iris.script": "objectscript" }, "objectscript.conn" :{ - "ns": "USER", - "username": "_SYSTEM", + "ns": "IRISAPP", + "username": "SUPERUSER", "password": "SYS", "docker-compose": { "service": "iris", "internalPort": 52773 }, - "active": true + "active": true, + "links": { + "FlaskTest": "http://localhost:55030/" + } }, "sqltools.connections": [ { - "namespace": "USER", + "namespace": "IRISAPP", "connectionMethod": "Server and Port", "showSystem": false, "previewLimit": 50, "server": "localhost", - "port": 32770, + "port": 55038, "askForPassword": false, "driver": "InterSystems IRIS", "name": "objectscript-docker", - "username": "_SYSTEM", + "username": "SuperUser", "password": "SYS" } ], - "python.pythonPath": "/usr/local/bin/python3" + "python.pythonPath": "/usr/local/bin/python3", + "python.testing.pytestArgs": [ + "src/python" + ], + "python.testing.unittestEnabled": false, + "python.testing.pytestEnabled": true } \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index e745ccc..bf6ba1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ ARG IMAGE=intersystemsdc/irishealth-community -ARG IMAGE=intersystemsdc/iris-community ARG IMAGE=intersystemsdc/iris-community:preview +ARG IMAGE=intersystemsdc/iris-community FROM $IMAGE WORKDIR /home/irisowner/dev @@ -9,19 +9,18 @@ ARG TESTS=0 ARG MODULE="iris-python-template" ARG NAMESPACE="USER" + # create Python env ## Embedded Python environment -ENV IRISUSERNAME "SuperUser" -ENV IRISPASSWORD "SYS" -ENV IRISNAMESPACE "USER" +ENV IRISNAMESPACE "IRISAPP" ENV PYTHON_PATH=/usr/irissys/bin/ -ENV PATH "/usr/irissys/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/irisowner/bin" - +ENV PATH "/usr/irissys/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/irisowner/bin:/home/irisowner/.local/bin" +# ENV LIBRARY_PATH=${ISC_PACKAGE_INSTALLDIR}/bin:${LIBRARY_PATH} ## Start IRIS RUN --mount=type=bind,src=.,dst=. \ pip3 install -r requirements.txt && \ iris start IRIS && \ - iris session IRIS < iris.script && \ - ([ $TESTS -eq 0 ] || iris session iris -U $NAMESPACE "##class(%ZPM.PackageManager).Shell(\"test $MODULE -v -only\",1,1)") && \ + iris merge IRIS merge.cpf && \ + irispython iris-script.py && \ iris stop IRIS quietly diff --git a/README.md b/README.md index 072258d..306ad44 100644 --- a/README.md +++ b/README.md @@ -3,13 +3,32 @@ This is a template to work with Embedded Python in InterSystems IRIS It demonstrates how to call python libs from ObjectScript in dc.python.test class. And it demonstrates how to deal with IRIS from python scripts - python/irisapp.py +## What is Embedded Python ? + +Embedded Python is a feature of InterSystems IRIS that allows you to **run python code in the same process** as the IRIS database engine. + +The benefits of Embedded Python are: + +- **Performance** + - no need to serialize data between IRIS and Python + - speed of data processing in Python is comparable to ObjectScript +- **Simplicity** + - no need to install and configure Python separately + - easy to deploy IRIS and Python together + - easy access to ObjectScript code and functionality from Python +- **Security** + - no need to open any additional ports for communication between IRIS and Python + +And the main benefit is that you can use all the power of Python libraries and frameworks in your InterSystems IRIS solutions. + ## Prerequisites Make sure you have [git](https://git-scm.com/book/en/v2/Getting-Started-Installing-Git) and [Docker desktop](https://www.docker.com/products/docker-desktop) installed. ## Installation ZPM +Open IRIS terminal in your IRIS with installed IPM. Run the command: -``` +```objectscript USER>zpm "install iris-python-template" ``` @@ -17,61 +36,251 @@ USER>zpm "install iris-python-template" Clone/git pull the repo into any local directory -``` +```bash $ git clone https://github.com/intersystems-community/iris-embedded-python-template.git ``` Open the terminal in this directory and run: -``` +```bash $ docker-compose build ``` 3. Run the IRIS container with your project: -``` +```bash $ docker-compose up -d ``` -## How to work with it +### IRIS Initialization +In this template two approaches are provided to initialize iris: merge and python. + +1. Using merge to initialize IRIS and create IRIS Database and Namespace +Notice merge.cpf file that is being implemented during docker image build in Dockerfile +``` +iris merge IRIS merge.cpf && \ +``` + that contains: +``` +[Actions] +CreateResource:Name=%DB_IRISAPP_DATA,Description="IRISAPP_DATA database" +CreateDatabase:Name=IRISAPP_DATA,Directory=/usr/irissys/mgr/IRISAPP_DATA +CreateResource:Name=%DB_IRISAPP_CODE,Description="IRISAPP_CODE database" +CreateDatabase:Name=IRISAPP_CODE,Directory=/usr/irissys/mgr/IRISAPP_CODE +CreateNamespace:Name=IRISAPP,Globals=IRISAPP_DATA,Routines=IRISAPP_CODE,Interop=1 +ModifyService:Name=%Service_CallIn,Enabled=1,AutheEnabled=48 +ModifyUser:Name=SuperUser,PasswordHash=a31d24aecc0bfe560a7e45bd913ad27c667dc25a75cbfd358c451bb595b6bd52bd25c82cafaa23ca1dd30b3b4947d12d3bb0ffb2a717df29912b743a281f97c1,0a4c463a2fa1e7542b61aa48800091ab688eb0a14bebf536638f411f5454c9343b9aa6402b4694f0a89b624407a5f43f0a38fc35216bb18aab7dc41ef9f056b1,10000,SHA512 +``` +As you can see it creates dabasases IRISAPP_DATA and IRISAPP_CODE for data and code, the related IRISAPP namespace to access it and the related resources %IRISAPP_DATA and %IRISAPP_CODE" to manage the access. + +Also it enables Callin service to make Embedded python work via ModifyService clause. +and it updates the password for the built-in user SuperUser to "SYS". The hash for this password is obtained via the following command: +```bash +docker run --rm -it containers.intersystems.com/intersystems/passwordhash:1.1 -algorithm SHA512 -workfactor 10000 +``` + +2. Using python to initialize IRIS. +Often we used a special iris.script file to run ObjectScript commands during the initialization. +This template shows you how to use python for the same purpose. +It is being executed via the line in Dockerfile: +``` +irispython iris-script.py && \ +``` +the iris-script.py file contains examples how developer can initialize different services of iris via Python code. + + +## How to test it ### Working with Python libs from ObjectScript Open IRIS terminal: -``` +```objectscript $ docker-compose exec iris iris session iris -USER> +USER>zn "IRISAPP" ``` The first test demonstrates the call to a standard python library working with dates datetime -``` -USER>d ##class(dc.python.test).Today() +```objectscript +IRISAPP>d ##class(dc.python.test).Today() 2021-02-09 ``` Another example shows the work of a custom lib sample.py which is installed with repo or ZPM. It has function hello which returns string "world": -``` -USER>d ##class(dc.python.test).Hello() +```objectscript +IRISAPP>d ##class(dc.python.test).Hello() World ``` -Another example shows how to work with files and use pandas and numpy libs. +Another example shows how to work with files and use pandas and numpy libs. It calculates the mean age of Titanic passengers: -``` -USER>d ##class(dc.python.test).TitanicMeanAge() +```objectscript +IRISAPP>d ##class(dc.python.test).TitanicMeanAge() mean age=29.69911764705882 ``` ### Working with IRIS from Embedded Python -Open VSCode in Devcontainer - this is the bell(notifications) button in the left bottom corner, where you will see the suggestion to open VSCOde in DevContainer mode. + +As mentioned Embedded Python works in the **same process as IRIS**. + +So you have 2 options to work with Embedded Python in IRIS: + +1. Develop in VSCode locally and then run the code in IRIS container with a shared folder. +2. Bind VsCode to the running IRIS container. + +### Develop python scripts locally and run the code in IRIS container + +By default, the template is configured to use the shared folder `./src` for python scripts to `/home/irisowner/dev/src` in IRIS container. + +You can change the folder according to your preferences. + +It's recommended to work with a virtual environment. + +Create a virtual environment in the project directory. Click New Terminal in the VS Code menu: +```bash +$ python3 -m venv .venv +``` +Activate the virtual environment: +``` +$ source .venv/bin/activate +``` + +Install the requirements: +``` +$ pip install -r requirements.txt +``` + +#### Run the python script in iris container: + +```bash +# attach to the running IRIS container +docker-compose exec iris bash +# run the script +$ irispython ./python/irisapp.py +``` +The script contains different samples of working with IRIS from python and goes through it. +it should return something like this: +``` +Hello World +Method call: +It works! +42 +Iris Version: +IRIS for UNIX (Ubuntu Server LTS for ARM64 Containers) 2023.2 (Build 227U) Mon Jul 31 2023 17:43:25 EDT +Creating new record in dc.python.PersistentClass +1 +Printing one IRIS Object Dump: ++----------------- general information --------------- +| oref value: 1 +| class name: dc.python.PersistentClass +| %%OID: $lb("1","dc.python.PersistentClass") +| reference count: 1 ++----------------- attribute values ------------------ +| %Concurrency = 1 +| Test = "2023-09-03 10:56:45.227577" ++----------------------------------------------------- +1 +Running SQL query Select * from dc_python.PersistentClass +[0]: ['1', '2023-09-03 10:56:45.227577'] +Printing the whole global of the persistence storage for the class dc.python.PersistentClass:^dc.Package4C8F.PersistentC1A93D +key=['1']: 2023-09-03 10:56:45.227577 +James +Jim +John +``` + +#### Bind VSCode to the running IRIS container + +Open VSCode in the project directory. + +Go to the `docker-compose.yml` file, right-click on it and select `Compose Up`. + +Once the container is up and running you can open the docker extension and right-click on the container name and select `Attach Visual Studio Code`. + +#### Working with IRIS from Embedded Python +Open VSCode in Devcontainer - this is the bell(notifications) button in the left bottom corner, where you will see the suggestion to open VSCOde in DevContainer mode. Follow it - it will let to execute Embedded Python scripts vs IRIS and develop it at the same time. Once devcontainer is opened go to /python/irisapp.py and run it, either with Run button in the top right corner, or in terminal via: +```bash +$ irispython /python/irisapp.py ``` -irispython /python/irisapp.py + + +### Working with flask + +The template also contains samples of working with flask. + +Connect to the running with a bash terminal: +```bash +$ docker-compose exec iris bash ``` -The script contains different samples of working with IRIS from python and goes through it. + +Run the following commands to start the flask server: +``` +irispython /python/flask/app.py +``` + +That will start the flask server and you will see the following output: +``` + * Serving Flask app 'app' + * Debug mode: off +WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. + * Running on http://127.0.0.1:5000 +``` + +`5000` is mapped to `55030` in docker-compose.yml + + +#### Test it + +Hello world : +```http +GET http://localhost:55030/ +Accept: application/json +``` + +Result: +```json +{ + "message": "Hello world" +} +``` + +Post a new persistent class + +```http +POST http://localhost:55030/persistentclass +Content-Type: application/json +Accept: application/json +{ + "test": "toto" +} +``` + +Result: +```json +{ + "id": 1, + "test": "toto" +} +``` + +Get the persistent class + +```http +GET http://localhost:55030/persistentclass/1 +Accept: application/json +``` + +Result: +```json +{ + "id": 1, + "test": "toto" +} +``` + Feel free to use the template for your own development just by adding new py files. diff --git a/dev.md b/dev.md index 898d7d4..2e5dd69 100644 --- a/dev.md +++ b/dev.md @@ -91,3 +91,7 @@ IRISAPP:zpm>test package-name do ##class(%SYS.Python).Shell() + + + iris session IRIS < iris.script && \ + ([ $TESTS -eq 0 ] || iris session iris -U $NAMESPACE "##class(%ZPM.PackageManager).Shell(\"test $MODULE -v -only\",1,1)") && \ diff --git a/docker-compose.yml b/docker-compose.yml index 52028e0..efa283e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,5 +9,6 @@ services: - 1972 - 55038:52773 - 53773 + - 55030:5000 volumes: - ./:/home/irisowner/dev diff --git a/iris-script.py b/iris-script.py new file mode 100644 index 0000000..3cf3f1c --- /dev/null +++ b/iris-script.py @@ -0,0 +1,31 @@ +import glob +import os +import iris +import pandas as pd +from sqlalchemy import create_engine +from iris import ipm + +# switch namespace to the %SYS namespace +iris.system.Process.SetNamespace("%SYS") + +# set credentials to not expire +iris.cls('Security.Users').UnExpireUserPasswords("*") + +# switch namespace to IRISAPP built by merge.cpf +iris.system.Process.SetNamespace("IRISAPP") + +# load ipm package listed in module.xml +#iris.cls('%ZPM.PackageManager').Shell("load /home/irisowner/dev -v") +assert ipm('load /home/irisowner/dev -v') + +# load demo data +engine = create_engine('iris+emb:///') +# list all csv files in the demo data folder +for files in glob.glob('/home/irisowner/dev/data/*.csv'): + # get the file name without the extension + table_name = os.path.splitext(os.path.basename(files))[0] + # load the csv file into a pandas dataframe + df = pd.read_csv(files) + # write the dataframe to IRIS + df.to_sql(table_name, engine, if_exists='replace', index=False, schema='dc_demo') + diff --git a/merge.cpf b/merge.cpf new file mode 100644 index 0000000..36d7cec --- /dev/null +++ b/merge.cpf @@ -0,0 +1,8 @@ +[Actions] +CreateResource:Name=%DB_IRISAPP_DATA,Description="IRISAPP_DATA database" +CreateDatabase:Name=IRISAPP_DATA,Directory=/usr/irissys/mgr/IRISAPP_DATA +CreateResource:Name=%DB_IRISAPP_CODE,Description="IRISAPP_CODE database" +CreateDatabase:Name=IRISAPP_CODE,Directory=/usr/irissys/mgr/IRISAPP_CODE +CreateNamespace:Name=IRISAPP,Globals=IRISAPP_DATA,Routines=IRISAPP_CODE,Interop=1 +ModifyService:Name=%Service_CallIn,Enabled=1,AutheEnabled=48 +ModifyUser:Name=SuperUser,PasswordHash=a31d24aecc0bfe560a7e45bd913ad27c667dc25a75cbfd358c451bb595b6bd52bd25c82cafaa23ca1dd30b3b4947d12d3bb0ffb2a717df29912b743a281f97c1,0a4c463a2fa1e7542b61aa48800091ab688eb0a14bebf536638f411f5454c9343b9aa6402b4694f0a89b624407a5f43f0a38fc35216bb18aab7dc41ef9f056b1,10000,SHA512 \ No newline at end of file diff --git a/module.xml b/module.xml index d335c08..12dd0c8 100644 --- a/module.xml +++ b/module.xml @@ -3,7 +3,7 @@ iris-python-template - 2.0.6 + 3.0.0 The simplest template to run embedded python module src diff --git a/python/flask/app.py b/python/flask/app.py new file mode 100644 index 0000000..c12b018 --- /dev/null +++ b/python/flask/app.py @@ -0,0 +1,30 @@ +from flask import Flask, jsonify, request, abort + +import iris + +app = Flask(__name__) + +@app.route('/') +def hello_world(): + #return an json object + return jsonify({'message':'Hello World'}) + +@app.route('/persistentclass', methods=['POST']) +def create_persistentclass(): + if not request.json or not 'test' in request.json: + abort(400) + obj=iris.cls('dc.python.PersistentClass')._New() + obj.Test=request.json['test'] + obj._Save() + return jsonify({'id':obj._Id(),'test':obj.Test}), 201 + +@app.route('/persistentclass/', methods=['GET']) +def get_one_persistentclass(id): + if iris.cls('dc.python.PersistentClass')._ExistsId(id): + obj=iris.cls('dc.python.PersistentClass')._OpenId(id) + return jsonify({'id':id,'test':obj.Test}) + else: + return jsonify({'error':'not found'}), 404 + +if __name__ == '__main__': + app.run(host='0.0.0.0',port=5000) diff --git a/requirements.txt b/requirements.txt index a31d49e..c903d5c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,6 @@ numpy==1.23.4 -pandas==1.5.0 \ No newline at end of file +pandas==1.5.0 +dataclasses-json==0.5.7 +sqlalchemy-iris==0.10.5 +flask==2.3.3 +requests==2.31.0