[Snyk] Security upgrade axios from 0.21.1 to 0.21.3

.gitignore

@@ -1,10 +1,14 @@
+migration/
+cache.*.json
+user-quotas.json
 build/
 build_qadocs/
 log.lsf.txt
 log.txt
 ssl/
 commits/
 user/
+drafts/
 
 image.batches.yaml
 iter.batches.yaml

.gitlab-ci.yml

@@ -56,7 +56,7 @@ variables:
 .deploy: &deploy
   stage: deploy
   script:
-    - ssh qa "bash -c \"cd $(pwd) && source .envrc && docker-compose -f docker-compose.yml -f production.yml  -f sirc.yml up -d --build\""
+    - ssh $DOCKER_HOST "bash -c \"cd $(pwd) && source .envrc && ./at-sirc-before-up.py && docker-compose -f docker-compose.yml -f production.yml  -f sirc.yml up -d --build\""
 
 deploy:production:
   <<: *deploy

backend/Dockerfile

@@ -26,9 +26,14 @@ RUN apt-get update -qq \
 # https://www.psycopg.org/docs/install.html#install-from-source
 # https://www.psycopg.org/docs/faq.html#faq-compile
 RUN apt-get update \
+    # postgresql and toolchain
     && apt-get install -y libpq-dev gcc \
+    # login packages
+    && apt-get install -y libsasl2-dev libldap2-dev libssl-dev \
     && apt-get clean
 
+# At SIRC we need to be able to turn into any user to delete their output files
+RUN apt-get update && apt-get install -y sudo
 
 # If we want uwsgi's routing support
 # https://uwsgi-docs.readthedocs.io/en/latest/InternalRouting.html
@@ -83,7 +88,7 @@ CMD ["/qaboard/backend/init.sh"]
 
 # Where we keep a cache of application data (e.g. git clones)
 VOLUME /var/qaboard
-
+RUN mkdir -p /var/qaboard && chmod -R 777 /var/qaboard
 
 
 # TODO:

backend/README.md

@@ -2,10 +2,23 @@
 QA-Board's backend built as a [flask](https://flask.pocoo.org) application. It exposes an HTTP API used to read/write all the metadata on QA-Board's runs.
 
 ## How to start a development backend
+1. First get the code:
 ```bash
 git clone git@gitlab-srv:common-infrastructure/qaboard.git
 cd qaboard
+```
+
+2. If you want to run a frontend, [go to the README](../webapp/README.md) and without docker run `npm install`.
+
+3. Edit at the top-level of the repository _development.yml_, and replace `arthurf` with your user. Edit _services/backend/passwd_ and add a line with  your user, looking like `arthurf:*:11611:10:Arthur Flam:/home/arthurf:/bin/tcsh`. You can get  it  with `getent passwd | grep arthurf`.
+
+4. Start the server:
+
+```bash
 docker-compose -f docker-compose.yml -f development.yml up  -d 
+
+# for more build logs
+export BUILDKIT_PROGRESS=plain
 ```
 
 > **Tip:** If you called `npm install` in the *webapp/*, (see the [README](../webapp)), a frontend connected to the dev backend will also be up on port 3000.
@@ -23,6 +36,10 @@ Consult also:
 - [Troubleshooting Guide](https://samsung.github.io/qaboard/docs/backend-admin/troubleshooting).
 - To learn how to restore from a backup, read the [upgrade guide](https://samsung.github.io/qaboard/docs/backend-admin/host-upgrades).
 
+At SIRC:
+```bash
+sudo sysctl -w net.core.somaxconn=65536
+```
 
 ## Overview
 [sqlalchemy](http://docs.sqlalchemy.org/en/latest/orm/tutorial.html) maps our classes (defined in [/models](models/)) to database tables:

backend/backend/__init__.py

@@ -6,9 +6,9 @@
 from flask_cors import CORS
 app = Flask(__name__)
 
-# This would be needed to use flask's sessions (e.g. display flash messages after redirects...)
-# However we never used this feature as (1) the backend is stateless and (2) the client uses an API, now generated views. 
-# app.secret_key = os.environ.get('QABOARD_FLASK_SECRET_KEY', 'xxxxxxxxxxx')
+# This key will be used to sign session cookies
+# To generate a key: python -c 'import os; print(os.urandom(16))'
+app.secret_key = os.environ.get('SECRET_KEY', 'please-generate-your-own-secret-key')
 
 # Provide easy access to our git repositories
 from .git_utils import Repos
@@ -24,15 +24,27 @@ def shutdown_session(exception=None):
     db_session.remove()
 
 import backend.api.api
+import backend.api.commit
+import backend.api.batch
+import backend.api.outputs
 import backend.api.webhooks
 import backend.api.integrations
 import backend.api.tuning
 import backend.api.export_to_folder
 import backend.api.auto_rois
 import backend.api.milestones
+import backend.api.auth
 
 # Enable cross-origin requests to avoid development headcaches  
 # cors = CORS(app, resources={r"/api/*": {"origins": "*"}})
 CORS(app)
 
-Base.metadata.create_all(engine)
+Base.metadata.create_all(engine)
+
+# Avoids errors
+#   > sqlalchemy.exc.OperationalError: (psycopg2.OperationalError) lost synchronization with server: got message type " "
+# https://docs.sqlalchemy.org/en/13/core/pooling.html#pooling-multiprocessing
+# https://stackoverflow.com/questions/43648075/uwsgi-flask-sqlalchemy-intermittent-postgresql-errors-with-warning-there-is-al
+# https://uwsgi-docs.readthedocs.io/en/latest/articles/TheArtOfGracefulReloading.html#preforking-vs-lazy-apps-vs-lazy
+# https://stackoverflow.com/questions/41279157/connection-problems-with-sqlalchemy-and-multiple-processes
+engine.dispose()

backend/backend/api/api.py

@@ -1,27 +1,22 @@
 """
-Simple REST API to list the objects in our database.
+Access data related to Projects.
 """
-import sys
-import datetime
 import pytz
-import subprocess
 import json
-from pathlib import Path
+import datetime
 
 import ujson
-from gitdb.exc import BadName
-from flask import request, jsonify, make_response, redirect
+from flask import request, jsonify, make_response
 
 from sqlalchemy import func, and_, asc, or_
-from sqlalchemy.orm import joinedload, selectinload
-from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
+from sqlalchemy.orm import selectinload
+
 from sqlalchemy.orm.attributes import flag_modified
 from sqlalchemy.sql import label
 
 from backend import app, db_session
 from ..models import Project, CiCommit, Batch, Output
-from ..models import latest_successful_commit
-
+from ..utils import profiled
 
 
 to_datetime = lambda s: timezone.localize(datetime.datetime.strptime(s, '%Y-%m-%dT%H:%M:%S.%fZ'))
@@ -61,17 +56,20 @@ def get_commits(branch=None):
   from_date = min(latest_authored_datetime - (to_date - from_date), from_date)
   from_date = from_date - datetime.timedelta(hours=3) # timezones as above
 
-
-  ci_commits = (db_session
-                .query(CiCommit)
-                .options(selectinload(CiCommit.batches).selectinload(Batch.outputs))
-                .filter(
-                  CiCommit.authored_datetime >= from_date,
-                  CiCommit.authored_datetime <= to_date,
-                  CiCommit.project_id == project_id,
-                )
-                .order_by(CiCommit.authored_datetime.desc())
-               )
+  with_outputs = False if request.args.get('with_outputs', 'false')=='false' else True
+  ci_commits = db_session.query(CiCommit)
+  if True: # with_outputs: do this when we pre-aggregated counts of Outputs per status...
+    ci_commits = ci_commits.options(selectinload(CiCommit.batches).selectinload(Batch.outputs))
+  else:
+    ci_commits = ci_commits.options(selectinload(CiCommit.batches))
+  ci_commits = (ci_commits
+    .filter(
+      CiCommit.authored_datetime >= from_date,
+      CiCommit.authored_datetime <= to_date,
+      CiCommit.project_id == project_id,
+    )
+    .order_by(CiCommit.authored_datetime.desc())
+  )
 
   if committer_name:
     ci_commits = ci_commits.filter_by(committer_name=committer_name)
@@ -80,6 +78,9 @@ def get_commits(branch=None):
 
 
   metrics_to_aggregate = json.loads(request.args.get('metrics', '{}'))
+  if project_id.startswith("CDE-Users/HW_ALG"): # too many results to be fast...
+    with_aggregation = {}
+
   with_batches = None
   batch = request.args.get('batch', None)
   if batch:
@@ -88,11 +89,16 @@ def get_commits(branch=None):
     only_ci_batches = False if request.args.get('only_ci_batches', 'false')=='false' else True
     if only_ci_batches:
       with_batches = ['default']
-  with_outputs = False if request.args.get('with_outputs', 'false')=='false' else True
-  # from ..utils import profiled
+  serializable_commits = []
   # with profiled():
-  serializable_commits = [c.to_dict(with_aggregation=metrics_to_aggregate, with_batches=with_batches, with_outputs=with_outputs)
-                          for c in ci_commits]
+  for c in ci_commits.limit(1000):
+    if not c.batches:
+      continue
+    serializable_commits.append(c.to_dict(
+      with_aggregation=metrics_to_aggregate,
+      with_batches=with_batches,
+      with_outputs=with_outputs
+    ))
   response = make_response(ujson.dumps(serializable_commits))
   response.headers['Content-Type'] = 'application/json'
   return response
@@ -110,7 +116,6 @@ def get_branches():
   return jsonify([b[0] for b in branches])
 
 
-
 @app.route("/api/v1/projects")
 def get_projects():
   projects = (db_session
@@ -151,147 +156,4 @@ def get_project():
   return jsonify(project.data)
 
 
-@app.route("/api/v1/output/<output_id>", methods=['GET', 'DELETE'])
-@app.route("/api/v1/output/<output_id>/", methods=['GET', 'DELETE'])
-def crud_output(output_id):
-  output = Output.query.filter(Output.id==output_id).one()
-  if request.method == 'GET':
-    return jsonify(output.to_dict())
-  if request.method == 'DELETE':
-    if output.is_pending:
-      return {"error": "Please wait for the Output to finish running before deleting it"}, 500
-    output.delete(soft=False)
-    db_session.delete(output)
-    db_session.commit()
-    return {"status": "OK"}
 
-
-@app.route("/api/v1/output/<output_id>/manifest", methods=['GET'])
-@app.route("/api/v1/output/<output_id>/manifest/", methods=['GET'])
-def get_output_manifest(output_id):
-  output = Output.query.filter(Output.id==output_id).one()
-  if output.is_running or request.args.get('refresh'):
-    manifest = output.update_manifest(compute_hashes=False)
-    return jsonify(manifest)
-  else:
-    return redirect(f"{output.output_dir_url}/manifest.outputs.json", code=302)
-
-
-
-
-
-@app.route("/api/v1/commit", methods=['GET', 'POST'])
-@app.route("/api/v1/commit/", methods=['GET', 'POST'])
-@app.route("/api/v1/commit/<path:commit_id>", methods=['GET', 'POST'])
-def api_ci_commit(commit_id=None):
-  if request.method == 'POST':
-    hexsha = request.json.get('commit_sha', request.json['git_commit_sha']) if not commit_id else commit_id
-    try:
-      commit = CiCommit.get_or_create(
-        session=db_session,
-        hexsha=hexsha,
-        project_id=request.json['project'],
-        data=request.json,
-      )
-    except:
-      return f"404 ERROR:\n ({request.json['project']}): There is an issue with your commit id ({request.json['git_commit_sha']})", 404
-    if not commit.data:
-      commit.data = {}
-    # Clients can store any metadata with each commit.
-    # We've been using it to store code quality metrics per subproject in our monorepo,
-    # Then we use other tools (e.g. metabase) to create dashboards.
-    commit_data = request.json.get('data', {})
-    commit.data = {**commit.data, **commit_data}
-    flag_modified(commit, "data")
-    if commit.deleted:
-      commit.deleted = False
-    db_session.add(commit)
-    db_session.commit()
-    return jsonify({"status": "OK"})
-
-
-  project_id = request.args['project']
-  if not commit_id:
-    commit_id = request.args.get('commit', None)
-    try:
-      project = Project.query.filter(Project.id==project_id).one()
-      default_branch = project.data['qatools_config']['project']['reference_branch']
-    except:
-      default_branch = 'master'
-    branch = request.args.get('branch', default_branch)
-    ci_commit = latest_successful_commit(db_session, project_id=project_id, branch=branch, batch_label=request.args.get('batch'))
-    if not ci_commit:
-      return jsonify({'error': f'Sorry, we cant find any commit with results for the {project_id} on {branch}.'}), 404
-  else:
-    try:
-      ci_commit = (db_session
-                   .query(CiCommit)
-                   .options(
-                     joinedload(CiCommit.batches).
-                     joinedload(Batch.outputs)
-                    )
-                   .filter(
-                     CiCommit.project_id==project_id,
-                     CiCommit.hexsha.startswith(commit_id),
-                   )
-                   .one()
-                  )
-    except NoResultFound:
-      try:
-        # TODO: This is a valid use case for having read-rights to the repo,
-        #       we can identify a commit by the tag/branch
-        #       To replace this without read rights, we should listen for push events and build a database
-        project = Project.query.filter(Project.id==project_id).one()
-        commit = project.repo.tags[commit_id].commit
-        try:
-          commit = project.repo.commit(commit_id)
-        except:
-          try:
-            commit = project.repo.refs[commit_id].commit
-          except:
-            commit = project.repo.tags[commit_id].commit
-        ci_commit = CiCommit(commit, project=project)
-        db_session.add(ci_commit)
-        db_session.commit()
-      except:
-        return jsonify({'error': f'Sorry, we could not find any data on commit {commit_id} in project {project_id}.'}), 404
-    except BadName:
-      return jsonify({f'error': f'Sorry, we could not understand the commid ID {commit_id} for project {project_id}.'}), 404
-    except Exception as e:
-      raise(e)
-      return jsonify({'error': 'Sorry, the request failed.'}), 500
-
-  batch = request.args.get('batch', None)
-  with_batches = [batch] if batch else None # by default we show all batches
-  with_aggregation = json.loads(request.args.get('metrics', '{}'))
-  response = make_response(ujson.dumps(ci_commit.to_dict(with_aggregation, with_batches=with_batches, with_outputs=True)))
-  response.headers['Content-Type'] = 'application/json'
-  return response
-
-
-@app.route("/api/v1/commit/save-artifacts/", methods=['POST'])
-@app.route("/api/v1/commit/save-artifacts", methods=['POST'])
-def commit_save_artifacts():
-  hexsha = request.json.get('hexsha')
-  try:
-      ci_commits = (db_session
-                   .query(CiCommit)
-                   .filter(
-                     CiCommit.hexsha == hexsha,
-                   )
-                  )
-  except:
-    return f"404 ERROR:\n ({request.json['project']}): There is an issue with your commit id ({hexsha})", 404
-  for ci_commit in ci_commits.all():
-    if not request.json['project'].startswith(ci_commit.project_id):
-      print(f'skip {ci_commit.project_id}')
-      continue
-    print(f"[save-artifacts] {ci_commit}")
-    # FIXME: in the clean crontab we remove commits without runs
-    # if we rely on artifacts from a subproject without runs, it will cause issues... 
-    # we should use the git info to find the qatools.yaml
-    ci_commit.save_artifacts()
-    ci_commit.deleted = False
-    db_session.add(ci_commit)
-    db_session.commit()
-  return 'OK'