[TH2-5212] added endpoint to request file by path (#5)

* added endpoint to request file by path

* update description of request functions

* update replacePathServerToLocal to raise Exception and remove unused endpoint

* changed use data access from `ro` to `rw`

* Corrected log configuration

* added /files/all request and parsing for file path parameter

* added `umask 0007` to `~/.bashrc` file

---------

Co-authored-by: nikita.smirnov <[email protected]>

Dockerfile

@@ -9,9 +9,7 @@ COPY . /app
 # groupadd --system - create a system account
 # useradd --system - create a system account
 # useradd --gid - name or ID of the primary group of the new account
-# usermod --append - append the user to the supplemental GROUPS mentioned by the -G/--groups option without removing the user from other groups
-# usermod --groups - new list of supplementary GROUPS
-RUN groupadd --system json-stream && useradd --system --gid json-stream --uid 1000 json-stream && usermod --append --groups users json-stream
+RUN useradd --system --gid users --uid 1000 json-stream
 
 ENV TH2_CFG_DIR="/app/var/th2/config/"
 ENV HOME="/home/json-stream"
@@ -41,6 +39,7 @@ ENV PYTHONPATH="${PYTHONPATH}:${PYTHON_LOCAL_LIB_PATH}:${PYTHON_SHARED_LIB_PATH}
 ENV PIP_CONFIG_FILE="${HOME}/.pip/pip.conf"
 
 RUN mkdir -p "${PYTHON_SHARED_LIB_PATH}"
+RUN echo 'umask 0007' >> "${HOME}/.bashrc"
 
 ENTRYPOINT ["python", "/app/server.py"]
 CMD ["/var/th2/config/custom.json"]

README.md

@@ -1,4 +1,4 @@
-# th2-json-stream-provider (j-sp) (0.0.3)
+# th2-json-stream-provider (j-sp) (0.0.5)
 
 This python server is made to launch Jupyter notebooks (*.ipynb) and get results from them.
 
@@ -83,6 +83,20 @@ You can put required files for you jupyter notebooks into `local-run/with-jupyte
 Or you can mount own folder by changing value of `USER_DATA_DIR` environment variable in the `local-run/with-jupyter-notebook/.evn` file.<br>
 Or change the `local-run/with-jupyter-notebook/compose.yml` file. Please note you should mount the same dictionary by the same path to `jupyter_notebook` and `json_stream_provider` services.
 
+#### provide permission for `local-run/with-jupyter-notebook/user_data` folder
+`jupyter-notebook` and `json-stream-provider` use user from default linux `users` group. 
+It means that:
+* `user_data` folder internal folder should have `rwx` permission for `users` group.
+* files in `user_data` folder should have `rw` permission for `users` group.
+
+Perhaps you will need sudo permission for the next commands
+
+```shell
+cd local-run/with-jupyter-notebook
+chgrp -R users user_data/
+chmod -R g=u user_data/
+```
+
 #### start command
 ```shell
 cd local-run/with-jupyter-notebook
@@ -105,6 +119,26 @@ docker compose build
 
 ## Release notes:
 
+### 0.0.5
+
+* added `umask 0007` to `~/.bashrc` file to provide rw file access for `users` group 
+* added `/file` request for loading content of single jsonl file
+* removed ability to get any file from machine via `/file` REST APIs
+* added sorting on requests `/files/notebooks` and `/files/results`
+* added `/files/all` request to list all files in `/notebooks` and `/results/` directories
+* added `convert_parameter` function for parsing parameter depending on it's type
+* update local run with jupyter-notebook:
+  * updated th2-rpt-viewer:
+    * added option to change default view type of result group
+    * added display of #display-table field in Table view type
+    * added option to view last N results of Notebook
+    * added validation of Notebook's parameters
+    * added timestamp and file path parameter types
+    * fixed clearing of Notebook's parameters on run
+    * increased width of parameters' inputs
+  * updated compose:
+    * changed use data access from `ro` to `rw`
+
 ### 0.0.4
 
 * added `${HOME}/python/lib` into `PYTHONPATH` environment variable

local-run/with-jupyter-notebook/compose.yml

@@ -1,14 +1,14 @@
 services:
   json_stream_provider:
-    image: ghcr.io/th2-net/th2-json-stream-provider-py:0.0.4-dev
+    image: ghcr.io/th2-net/th2-json-stream-provider-py:0.0.5-th2-5212-9837013984-c5168cb
     ports:
       - "8081:8080"
     volumes:
       - j_sp_volume:/home/json-stream/:rw
       - jupyter_volume:/home/jovyan/:rw
       - python_lib:/home/json-stream/python/lib:ro
-      - ${USER_DATA_DIR}:/home/jovyan/user-data/:ro
-      - ./json-stream-provider/custom.json:/var/th2/config/custom.json:ro
+      - ${USER_DATA_DIR}:/home/jovyan/user-data/:rw
+      - ./json-stream-provider:/var/th2/config:ro
     networks:
       - th2_network
 
@@ -21,12 +21,12 @@ services:
     volumes:
       - jupyter_volume:/home/jovyan/:rw
       - python_lib:/home/jovyan/python/lib:rw
-      - ${USER_DATA_DIR}:/home/jovyan/user-data/:ro
+      - ${USER_DATA_DIR}:/home/jovyan/user-data/:rw
     networks:
       - th2_network
 
   th2_rpt_viewer:
-    image: ghcr.io/th2-net/th2-rpt-viewer:5.2.8-th2-5208-9697982356
+    image: ghcr.io/th2-net/th2-rpt-viewer:5.2.8-th2-5212-9836473151
     ports:
       - "8083:8080"
     networks:

local-run/with-jupyter-notebook/json-stream-provider/log4py.conf

@@ -0,0 +1,22 @@
+[loggers]
+keys=root
+
+[handlers]
+keys=consoleHandler
+
+[formatters]
+keys=formatter
+
+[logger_root]
+level=INFO
+handlers=consoleHandler
+
+[handler_consoleHandler]
+class=StreamHandler
+level=INFO
+formatter=formatter
+args=(sys.stdout,)
+
+[formatter_formatter]
+format=%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s
+datefmt=%Y-%m-%d %H:%M:%S

local-run/with-jupyter-notebook/jupyter-notebook/Dockerfile

@@ -3,7 +3,8 @@ FROM jupyter/datascience-notebook:python-3.9
 RUN echo "c.ServerApp.base_url = '/jupyter/'" >> /etc/jupyter/jupyter_server_config.py
 
 ENV PYTHON_SHARED_LIB_PATH="${HOME}/python/lib"
+ENV PYTHONPATH="${PYTHONPATH}:${PYTHON_SHARED_LIB_PATH}"
+ENV PIP_TARGET="${PYTHON_SHARED_LIB_PATH}"
 
 RUN mkdir -p "${PYTHON_SHARED_LIB_PATH}"
-ENV PYTHONPATH="${PYTHONPATH}:${PYTHON_SHARED_LIB_PATH}"
-ENV PIP_TARGET="${PYTHON_SHARED_LIB_PATH}"
+RUN echo 'umask 0007' >> "${HOME}/.bashrc"

log_configuratior.py

@@ -0,0 +1,34 @@
+import logging.config
+import os
+
+log4py_file = '/var/th2/config/log4py.conf'
+def configureLogging():
+    if os.path.exists(log4py_file):
+        logging.config.fileConfig(log4py_file, disable_existing_loggers=False)
+        logging.getLogger(__name__).info(f'Logger is configured by {log4py_file} file')
+    else:
+        default_logging_config = {
+            'version': 1,
+            'formatters': {
+                'default': {
+                    'format': '%(asctime)s.%(msecs)03d - %(name)s - %(levelname)s - %(message)s',
+                    'datefmt': '%Y-%m-%d %H:%M:%S'
+                },
+            },
+            'handlers': {
+                'console': {
+                    'class': 'logging.StreamHandler',
+                    'formatter': 'default',
+                    'level': 'DEBUG',
+                    'stream': 'ext://sys.stdout'
+                },
+            },
+            'root': {
+                'handlers': ['console'],
+                'level': 'DEBUG',
+            },
+        }
+        logging.config.dictConfig(default_logging_config)
+        logging.getLogger(__name__).info('Logger is configured by default')
+
+

package_info.json

@@ -1,4 +1,4 @@
 {
   "package_name": "th2-json-stream-provider",
-  "package_version": "0.0.4"
+  "package_version": "0.0.5"
 }