diff --git a/.github/workflows/requirements-update.yml b/.github/workflows/requirements-update.yml new file mode 100644 index 00000000..e692c187 --- /dev/null +++ b/.github/workflows/requirements-update.yml @@ -0,0 +1,60 @@ +name: Requirements Update + +on: + schedule: + - cron: '0 12 * * 1' # runs at 12:00 UTC on Mondays + workflow_dispatch: + +jobs: + + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v4 + with: + only-labels: dependencies,automated pr + stale-pr-message: 'This PR is stale because it has been open 7 days with no activity. Remove stale label or comment or this will be closed in 7 days.' + close-pr-message: 'This PR was closed because it has been stalled for 7 days with no activity.' + days-before-pr-stale: 7 + days-before-pr-close: 7 + delete-branch: true + + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + with: + ref: development + + - name: Setup python + uses: actions/setup-python@v2 + with: + python-version: '3.10' + + - name: Install dev Python packages + run: | + python -m pip install --upgrade pip + pip install -r dev-requirements.txt + + - name: Check for pip-tools upgrades + run: | + pip-compile --generate-hashes \ + --allow-unsafe \ + --upgrade \ + --output-file requirements.txt requirements.in + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + base: development + branch: requirements-updates + branch-suffix: timestamp + delete-branch: true + commit-message: "fix(requirements): Updated Python requirements" + title: 'Python Requirements Updates' + body: > + This PR is auto-generated by Github Actions job [requirements-update]. + labels: dependencies, automated pr diff --git a/.github/workflows/scan.yml b/.github/workflows/scan.yml new file mode 100644 index 00000000..821b6efb --- /dev/null +++ b/.github/workflows/scan.yml @@ -0,0 +1,67 @@ +name: Scan + +on: + push: + branches: [ master, development ] + pull_request: + branches: [ master, development ] + schedule: + - cron: '0 12 * * 1' # runs at 12:00 UTC on Mondays + workflow_dispatch: + +jobs: + + scan: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Set image name + id: setimagename + run: | + echo "Image name: $GITHUB_REPOSITORY:$GITHUB_SHA" + echo "::set-output name=imagename::$GITHUB_REPOSITORY:$GITHUB_SHA" + + - name: Build the image + id: buildimage + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile + push: false + tags: ${{ steps.setimagename.outputs.imagename }} + + - name: Check whether container scanning should be enabled + id: checkcontainerscanning + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + run: | + echo "Enable container scanning: ${{ env.SNYK_TOKEN != '' }}" + echo "::set-output name=enabled::${{ env.SNYK_TOKEN != '' }}" + + - name: Run Snyk to check Docker image for vulnerabilities + uses: snyk/actions/docker@master + if: steps.checkcontainerscanning.outputs.enabled == 'true' + continue-on-error: true + env: + SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }} + with: + image: ${{ steps.setimagename.outputs.imagename }} + args: --file=Dockerfile + + - name: Upload result to GitHub Code Scanning + uses: github/codeql-action/upload-sarif@v1 + if: steps.checkcontainerscanning.outputs.enabled == 'true' + with: + sarif_file: snyk.sarif diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..027a3c1f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,40 @@ +name: Test + +on: + push: + branches: [ master, development ] + pull_request: + branches: [ master, development ] + +jobs: + + build: + runs-on: ubuntu-latest + + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_HUB_USERNAME }} + password: ${{ secrets.DOCKER_HUB_PASSWORD }} + + - name: Set image name + id: setimagename + run: | + echo "Image name: $GITHUB_REPOSITORY:$GITHUB_SHA" + echo "::set-output name=imagename::$GITHUB_REPOSITORY:$GITHUB_SHA" + + - name: Build the image + id: buildimage + uses: docker/build-push-action@v2 + with: + context: ./ + file: ./Dockerfile + push: false + tags: ${{ steps.setimagename.outputs.imagename }} diff --git a/.gitignore b/.gitignore index 5d89396e..719aca1e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,9 @@ -app/assets/* -*.pyc -*/.DS_Store -.DS_Store -*.log -app/db.sqlite3 -app/hypatio/local_settings.py +app/assets/* +*.pyc +*/.DS_Store +.DS_Store +*.log +app/db.sqlite3 +app/hypatio/local_settings.py .vscode/settings.json +backup diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..8795174b --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,20 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks.git + rev: v4.2.0 + hooks: + - id: trailing-whitespace + - id: mixed-line-ending + - id: check-byte-order-marker + - id: check-merge-conflict + - id: detect-aws-credentials + - repo: https://github.com/jazzband/pip-tools + rev: 6.8.0 + hooks: + - id: pip-compile + name: pip-compile dev-requirements.in + args: [dev-requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, dev-requirements.txt] + files: ^dev-requirements\.(in|txt)$ + - id: pip-compile + name: pip-compile requirements.in + args: [requirements.in, --upgrade, --generate-hashes, --allow-unsafe, --output-file, requirements.txt] + files: ^requirements\.(in|txt)$ diff --git a/Dockerfile b/Dockerfile index 047ca177..ec9dfa65 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,41 +1,50 @@ -FROM python:3.6-alpine3.8 AS builder - -# Install dependencies -RUN apk add --update \ - build-base \ - g++ \ - libffi-dev \ - mariadb-dev \ - jpeg-dev \ - zlib-dev +FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 AS builder + +# Install requirements +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + bzip2 \ + gcc \ + default-libmysqlclient-dev \ + libssl-dev \ + && rm -rf /var/lib/apt/lists/* # Add requirements -ADD app/requirements.txt /requirements.txt +ADD requirements.* / -# Install Python packages -RUN pip install -r /requirements.txt +# Build Python wheels with hash checking +RUN pip install -U wheel \ + && pip wheel -r /requirements.txt \ + --wheel-dir=/root/wheels -FROM hmsdbmitc/dbmisvc:3.6-alpine +FROM hmsdbmitc/dbmisvc:debian11-slim-python3.10-0.5.0 -RUN apk add --no-cache --update \ - bash \ - nginx \ - curl \ - openssl \ - jq \ - mariadb-connector-c \ - jpeg-dev \ - zlib-dev \ - && rm -rf /var/cache/apk/* +# Copy Python wheels from builder +COPY --from=builder /root/wheels /root/wheels -# Copy pip packages from builder -COPY --from=builder /root/.cache /root/.cache +# Install requirements +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + default-libmysqlclient-dev \ + libmagic1 \ + && rm -rf /var/lib/apt/lists/* -# Add requirements -ADD app/requirements.txt /requirements.txt +# Add requirements files +ADD requirements.* / -# Install Python packages -RUN pip install -r /requirements.txt +# Install Python packages from wheels +RUN pip install --no-index \ + --find-links=/root/wheels \ + --force-reinstall \ + # Use requirements without hashes to allow using wheels. + # For some reason the hashes of the wheels change between stages + # and Pip errors out on the mismatches. + -r /requirements.in + +# Setup entry scripts +ADD docker-entrypoint-init.d/* /docker-entrypoint-init.d/ # Copy app source COPY /app /app @@ -70,4 +79,8 @@ ENV DBMI_APP_STATIC_ROOT=/app/assets # Healthchecks ENV DBMI_HEALTHCHECK=true ENV DBMI_HEALTHCHECK_PATH=/healthcheck -ENV DBMI_APP_HEALTHCHECK_PATH=/healthcheck \ No newline at end of file +ENV DBMI_APP_HEALTHCHECK_PATH=/healthcheck + +# File proxy +ENV DBMI_FILE_PROXY=true +ENV DBMI_FILE_PROXY_PATH=/proxy diff --git a/app/contact/urls.py b/app/contact/urls.py index c43bd766..013fa408 100644 --- a/app/contact/urls.py +++ b/app/contact/urls.py @@ -1,8 +1,8 @@ -from django.conf.urls import url +from django.urls import re_path from .views import contact_form app_name = 'contact' urlpatterns = ( - url(r'^(?P[^/]+)/?$', contact_form, name='contact_form'), - url(r'^', contact_form, name='contact_form'), + re_path(r'^(?P[^/]+)/?$', contact_form, name='contact_form'), + re_path(r'^', contact_form, name='contact_form'), ) diff --git a/app/contact/views.py b/app/contact/views.py index 9e3afd82..d36cd60f 100644 --- a/app/contact/views.py +++ b/app/contact/views.py @@ -1,10 +1,11 @@ import logging -from pyauth0jwt.auth0authenticate import public_user_auth_and_jwt +from hypatio.auth0authenticate import public_user_auth_and_jwt from contact.forms import ContactForm from projects.models import DataProject +from manage.views import is_ajax from django.http import HttpResponse, HttpResponseRedirect from django.urls import reverse @@ -57,7 +58,7 @@ def contact_form(request, project_key=None): extra=context) # Check how the request was made. - if request.is_ajax(): + if is_ajax(request): return HttpResponse('SUCCESS', status=200) if success else HttpResponse('ERROR', status=500) else: if success: @@ -65,16 +66,22 @@ def contact_form(request, project_key=None): messages.success(request, 'Thanks, your message has been submitted!') else: messages.error(request, 'An unexpected error occurred, please try again') - return HttpResponseRedirect(reverse('dashboard:dashboard')) + return HttpResponseRedirect(reverse( + 'projects:view-project', + kwargs={'project_key': form.cleaned_data['project']} + )) else: logger.error("[HYPATIO][ERROR][contact_form] Form is invalid! - " + str(request.user.id)) # Check how the request was made. - if request.is_ajax(): + if is_ajax(request): return HttpResponse('INVALID', status=500) else: messages.error(request, 'An unexpected error occurred, please try again') - return HttpResponseRedirect(reverse('dashboard:dashboard')) + return HttpResponseRedirect(reverse( + 'projects:view-project', + kwargs={'project_key': form.cleaned_data['project']} + )) # If a GET (or any other method) we'll create a blank form. initial = {} @@ -114,7 +121,9 @@ def email_send(subject=None, recipients=None, email_template=None, extra=None): msg.attach_alternative(msg_html, "text/html") msg.send() except Exception as ex: - print(ex) + logger.exception(ex, exc_info=True, extra={ + 'email': email_template, 'extra': extra + }) sent_without_error = False logger.debug("[HYPATIO][DEBUG][email_send] E-Mail Status - " + str(sent_without_error)) diff --git a/app/hypatio/auth0authenticate.py b/app/hypatio/auth0authenticate.py new file mode 100644 index 00000000..8e65a49c --- /dev/null +++ b/app/hypatio/auth0authenticate.py @@ -0,0 +1,277 @@ +from furl import furl +import json +import base64 +import logging +import requests +from functools import wraps + +from django.contrib.auth.models import User +from django.contrib import auth as django_auth +from django.contrib.auth import login +from django.conf import settings +from django.shortcuts import redirect +from django.contrib.auth import logout +from django.core.exceptions import PermissionDenied +from dbmi_client.settings import dbmi_settings +from dbmi_client.settings import dbmi_settings +from dbmi_client.authn import validate_request, login_redirect_url + +logger = logging.getLogger(__name__) + + +def jwt_and_manage(item): + ''' + Decorator that accepts an item string that is used to retrieve + permissions from SciAuthZ. The current user must have a valid JWT + and also have 'MANAGE' permissions for the passed item. + :param item: The SciAuthZ item string + :type item: str + :return: function + ''' + + def real_decorator(function): + + @wraps(function) + def wrap(request, *args, **kwargs): + + # Validates the JWT and returns its payload if valid. + jwt_payload = validate_request(request) + + # User has a valid JWT from SciAuth + if jwt_payload is not None: + + content = None + try: + # Get the email + email = jwt_payload.get('email') + + # Confirm user is a manager of the given project + permissions_url = sciauthz_permission_url(item, email) + response = requests.get(permissions_url, headers=sciauthz_headers(request)) + content = response.content + response.raise_for_status() + + # Parse permissions + user_permissions = response.json() + + if user_permissions is not None and 'results' in user_permissions: + for perm in user_permissions['results']: + if perm['permission'] == "MANAGE": + return function(request, *args, **kwargs) + + # Possibly store these elsewhere for records + logger.warning('{} Failed MANAGE permission on {}'.format(email, item), + extra={'jwt': jwt_payload, 'authorizations': response.json()}) + + except requests.HTTPError as e: + logger.exception('Checking permissions error: {}'.format(e), exc_info=True, + extra={'jwt': jwt_payload, 'content': content}) + + except Exception as e: + logger.exception('Checking permissions error: {}'.format(e), exc_info=True, + extra={'jwt': jwt_payload}) + + # Forbid access as a default measure + raise PermissionDenied + + else: + logger.debug('Missing/invalid JWT, sending to login') + return logout_redirect(request) + + return wrap + + return real_decorator + + +def public_user_auth_and_jwt(function): + + @wraps(function) + def wrap(request, *args, **kwargs): + """ + Here we see if the user is logged in but let them stay on the page if they aren't. + """ + + # Validates the JWT and returns its payload if valid. + jwt_payload = validate_request(request) + + # If user is logged in, make sure they have a valid JWT + if request.user.is_authenticated and jwt_payload is None: + logger.debug('User ' + request.user.email + ' is authenticated but does not have a valid JWT. Logging them out.') + return logout_redirect(request) + + # User has a JWT session open but not a Django session. Try to start a Django session and continue the request. + if not request.user.is_authenticated and jwt_payload is not None: + jwt_login(request, jwt_payload) + + return function(request, *args, **kwargs) + + return wrap + + +def user_auth_and_jwt(function): + ''' + Decorator to verify both the JWT as well as the session + of the current user in order to control access to the + given method or view. JWT email is also compared to the + Django user's email and must match. Redirects back to + authentication server if not all conditions are satisfied. + :param function: The protected method + :return: decorator + ''' + @wraps(function) + def wrap(request, *args, **kwargs): + + # Validates the JWT and returns its payload if valid. + jwt_payload = validate_request(request) + + # User is both logged into this app and via JWT. + if request.user.is_authenticated and jwt_payload is not None: + + # Ensure the email matches (without case sensitivity) + if request.user.username.lower() != jwt_payload['email'].lower(): + logger.debug('Django and JWT email mismatch! Log them out and redirect to log back in') + return logout_redirect(request) + + return function(request, *args, **kwargs) + # User has a JWT session open but not a Django session. Start a Django session and continue the request. + elif not request.user.is_authenticated and jwt_payload is not None: + if jwt_login(request, jwt_payload): + return function(request, *args, **kwargs) + else: + return logout_redirect(request) + # User doesn't pass muster, throw them to the login app. + else: + return logout_redirect(request) + + return wrap + + +def dbmi_jwt(function): + ''' + Decorator to only check if the current user's JWT is valid + :param function: + :type function: + :return: + :rtype: + ''' + @wraps(function) + def wrap(request, *args, **kwargs): + + # Validates the JWT and returns its payload if valid. + jwt_payload = validate_request(request) + + # User has a valid JWT from SciAuth + if jwt_payload is not None: + return function(request, *args, **kwargs) + + else: + logger.debug('Missing/invalid JWT, sending to login') + return logout_redirect(request) + + return wrap + + +def sciauthz_permission_url(item, email): + ''' + Build and return the SciAuthZ URL to GET against for a user's + item permissions + :param item: The SciAuthZ item to check permissions on + :type item: str + :param email: The email of the user in questions + :type email: str + :return: The URL + :rtype: str + ''' + # Build it + url = furl(settings.PERMISSIONS_URL) + + # Add query + url.query.params.add('item', item) + url.query.params.add('email', email) + + return url.url + + +def sciauthz_headers(request): + ''' + Returns the headers needed to authenticate requests against SciAuthZ + :param request: + :type request: + :return: + :rtype: + ''' + + # Extract JWT token into a string. + jwt_string = request.COOKIES.get("DBMI_JWT", None) + + return {"Authorization": "JWT " + jwt_string, 'Content-Type': 'application/json'} + + +def jwt_login(request, jwt_payload): + """ + The user has a valid JWT but needs to log into this app. Do so here and return the status. + :param request: + :param jwt_payload: String form of the JWT. + :return: + """ + + logger.debug("Logging user in via JWT. Is Authenticated? " + str(request.user.is_authenticated)) + + request.session['profile'] = jwt_payload + + user = django_auth.authenticate(**jwt_payload) + + if user: + login(request, user) + else: + logger.debug("Could not log user in.") + + return request.user.is_authenticated + + +def logout_redirect(request): + """ + This will log a user out and redirect them to log in again via the AuthN server. + :param request: + :return: The response object that takes the user to the login page. 'next' parameter set to bring them back to their intended page. + """ + logout(request) + + # Build the URL + login_url = furl(login_redirect_url(request, next_url=request.build_absolute_uri())) + + # Check for branding + if hasattr(settings, 'SCIAUTH_BRANDING'): + logger.debug('SciAuth branding passed') + + # Encode it and pass it + branding = base64.urlsafe_b64encode(json.dumps(settings.SCIAUTH_BRANDING).encode('utf-8')).decode('utf-8') + login_url.query.params.add('branding', branding) + + # Set the URL and purge cookies + response = redirect(login_url.url) + response.delete_cookie('DBMI_JWT', domain=dbmi_settings.JWT_COOKIE_DOMAIN) + logger.debug('Redirecting to: {}'.format(login_url.url)) + + return response + + +class Auth0Authentication(object): + + def authenticate(self, request, **token_dictionary): + logger.debug("Authenticate User: {}/{}".format(token_dictionary.get('sub'), token_dictionary.get('email'))) + + try: + user = User.objects.get(username=token_dictionary["email"]) + except User.DoesNotExist: + logger.debug("User not found, creating: {}".format(token_dictionary.get('email'))) + + user = User(username=token_dictionary["email"], email=token_dictionary["email"]) + user.save() + return user + + def get_user(self, user_id): + try: + return User.objects.get(pk=user_id) + except User.DoesNotExist: + return None diff --git a/app/hypatio/dbmiauthz_services.py b/app/hypatio/dbmiauthz_services.py index 84b54376..8cf885b1 100644 --- a/app/hypatio/dbmiauthz_services.py +++ b/app/hypatio/dbmiauthz_services.py @@ -2,8 +2,8 @@ from furl import furl import logging -from django.conf import settings from django.core.exceptions import ObjectDoesNotExist +from dbmi_client.settings import dbmi_settings from projects.models import DataProject from projects.models import Participant @@ -13,10 +13,10 @@ class DBMIAuthz: - user_permissions_url = settings.AUTHZ_BASE + "/user_permission/" - create_profile_permission_url = settings.AUTHZ_BASE + "/user_permission/create_registration_permission_record/" - create_view_permission_url = settings.AUTHZ_BASE + "/user_permission/create_item_view_permission_record/" - remove_view_permission_url = settings.AUTHZ_BASE + "/user_permission/remove_item_view_permission_record/" + user_permissions_url = dbmi_settings.AUTHZ_URL + "/user_permission/" + create_profile_permission_url = dbmi_settings.AUTHZ_URL + "/user_permission/create_registration_permission_record/" + create_view_permission_url = dbmi_settings.AUTHZ_URL + "/user_permission/create_item_view_permission_record/" + remove_view_permission_url = dbmi_settings.AUTHZ_URL + "/user_permission/remove_item_view_permission_record/" @classmethod def _permissions_query(cls, request, email=None, item=None, search=None): diff --git a/app/hypatio/file_services.py b/app/hypatio/file_services.py index 0b13621f..e7347288 100644 --- a/app/hypatio/file_services.py +++ b/app/hypatio/file_services.py @@ -259,10 +259,7 @@ def group_name(permission): def _s3_client(): # Get the service client with sigv4 configured - return boto3.client('s3', - aws_access_key_id=settings.S3_AWS_ACCESS_KEY_ID, - aws_secret_access_key=settings.S3_AWS_SECRET_ACCESS_KEY, - config=Config(signature_version='s3v4')) + return boto3.client('s3', config=Config(signature_version='s3v4')) def get_download_url(file_name, expires_in=3600): diff --git a/app/hypatio/middleware.py b/app/hypatio/middleware.py new file mode 100644 index 00000000..99e01630 --- /dev/null +++ b/app/hypatio/middleware.py @@ -0,0 +1,22 @@ +from django.conf import settings +from dbmi_client import environment + +import logging +logger = logging.getLogger(__name__) + + +class XRobotsTagMiddleware(object): + """Adds X-Robots-Tag Based on environment""" + + def __init__(self, get_response): + self.get_response = get_response + + def __call__(self, request): + response = self.get_response(request) + + if hasattr(settings, "X_ROBOTS_TAG") and settings.X_ROBOTS_TAG: + response["X-Robots-Tag"] = ",".join(settings.X_ROBOTS_TAG) + elif environment.get_str("DBMI_ENV") != "prod": + response["X-Robots-Tag"] = "noindex,nofollow" + + return response diff --git a/app/hypatio/sciauthz_services.py b/app/hypatio/sciauthz_services.py index bde7c2ed..780d2a32 100644 --- a/app/hypatio/sciauthz_services.py +++ b/app/hypatio/sciauthz_services.py @@ -5,7 +5,7 @@ import furl import logging -from django.conf import settings +from dbmi_client.settings import dbmi_settings from projects.models import DataProject @@ -16,17 +16,12 @@ class SciAuthZ: JWT_HEADERS = None CURRENT_USER_EMAIL = None - def __init__(self, authz_base, jwt, user_email): + def __init__(self, jwt, user_email): - user_permissions_url = authz_base + "/user_permission/" - create_profile_permission = authz_base + "/user_permission/create_registration_permission_record/" - create_view_permission = authz_base + "/user_permission/create_item_view_permission_record/" - remove_view_permission = authz_base + "/user_permission/remove_item_view_permission_record/" - - self.USER_PERMISSIONS_URL = user_permissions_url - self.CREATE_PROFILE_PERMISSION = create_profile_permission - self.CREATE_ITEM_PERMISSION = create_view_permission - self.REMOVE_ITEM_PERMISSION = remove_view_permission + self.USER_PERMISSIONS_URL = dbmi_settings.AUTHZ_URL + "/user_permission/" + self.CREATE_PROFILE_PERMISSION = dbmi_settings.AUTHZ_URL + "/user_permission/create_registration_permission_record/" + self.CREATE_ITEM_PERMISSION = dbmi_settings.AUTHZ_URL + "/user_permission/create_item_view_permission_record/" + self.REMOVE_ITEM_PERMISSION = dbmi_settings.AUTHZ_URL + "/user_permission/remove_item_view_permission_record/" jwt_headers = {"Authorization": "JWT " + jwt, 'Content-Type': 'application/json'} @@ -45,7 +40,7 @@ def user_has_manage_permission(self, item): permissions_url.query.params.add('search', 'Hypatio,MANAGE') try: - user_permissions = requests.get(permissions_url.url, headers=self.JWT_HEADERS, verify=settings.VERIFY_REQUESTS).json() + user_permissions = requests.get(permissions_url.url, headers=self.JWT_HEADERS).json() except JSONDecodeError: user_permissions = None @@ -74,7 +69,6 @@ def current_user_permissions(self): user_permissions_request = requests.get( authz_url.url, headers=self.JWT_HEADERS, - verify=settings.VERIFY_REQUESTS ).json() # If there are any permissions returned, add them to the list. @@ -108,7 +102,6 @@ def create_profile_permission(self, grantee_email, project): self.CREATE_PROFILE_PERMISSION, headers=modified_headers, data=data, - verify=settings.VERIFY_REQUESTS ) return profile_permission @@ -124,7 +117,7 @@ def create_view_permission(self, project, grantee_email): "item": 'Hypatio.' + project } - view_permission = requests.post(self.CREATE_ITEM_PERMISSION, headers=modified_headers, data=context, verify=settings.VERIFY_REQUESTS) + view_permission = requests.post(self.CREATE_ITEM_PERMISSION, headers=modified_headers, data=context) return view_permission def remove_view_permission(self, project, grantee_email): @@ -138,7 +131,7 @@ def remove_view_permission(self, project, grantee_email): "item": 'Hypatio.' + project } - view_permission = requests.post(self.REMOVE_ITEM_PERMISSION, headers=modified_headers, data=context, verify=settings.VERIFY_REQUESTS) + view_permission = requests.post(self.REMOVE_ITEM_PERMISSION, headers=modified_headers, data=context) return view_permission def user_has_single_permission(self, permission, value, email=None): @@ -151,7 +144,7 @@ def user_has_single_permission(self, permission, value, email=None): f.args["email"] = email try: - user_permissions = requests.get(f.url, headers=self.JWT_HEADERS, verify=settings.VERIFY_REQUESTS).json() + user_permissions = requests.get(f.url, headers=self.JWT_HEADERS).json() except JSONDecodeError: logger.debug("[SCIAUTHZ][user_has_single_permission] - No Valid permissions returned.") return False @@ -217,7 +210,6 @@ def get_all_view_permissions_for_project(self, project): user_permissions_request = requests.get( authz_url, headers=self.JWT_HEADERS, - verify=settings.VERIFY_REQUESTS ).json() # If there are any permissions returned, add them to the list. diff --git a/app/hypatio/scireg_services.py b/app/hypatio/scireg_services.py index 587231e7..93e1c5e2 100644 --- a/app/hypatio/scireg_services.py +++ b/app/hypatio/scireg_services.py @@ -4,21 +4,20 @@ from furl import furl from json import JSONDecodeError -from django.conf import settings - +from dbmi_client.settings import dbmi_settings import logging logger = logging.getLogger(__name__) -VERIFY_SSL = True - +# Set the base API URL for registration related queries +DBMI_REG_API_URL = furl(dbmi_settings.REG_URL) / "api" / "register" def build_headers_with_jwt(user_jwt): return {"Authorization": "JWT " + user_jwt, 'Content-Type': 'application/json'} def send_confirmation_email(user_jwt, current_uri): - send_confirm_email_url = settings.SCIREG_REGISTRATION_URL + 'send_confirmation_email/' + send_confirm_email_url = (DBMI_REG_API_URL / 'send_confirmation_email/').url logger.debug("[HYPATIO][DEBUG][send_confirmation_email] - Sending user confirmation e-mail to " + send_confirm_email_url) @@ -36,7 +35,7 @@ def get_user_email_confirmation_status(user_jwt): Returns True or False. """ - response = requests.get(settings.SCIREG_REGISTRATION_URL, headers=build_headers_with_jwt(user_jwt)) + response = requests.get(DBMI_REG_API_URL.url, headers=build_headers_with_jwt(user_jwt)) try: email_status = response.json()['results'][0]['email_confirmed'] @@ -50,7 +49,7 @@ def get_user_email_confirmation_status(user_jwt): def get_current_user_profile(user_jwt): - f = furl(settings.SCIREG_REGISTRATION_URL) + f = furl(DBMI_REG_API_URL.url) try: profile = requests.get(f.url, headers=build_headers_with_jwt(user_jwt)).json() @@ -63,7 +62,7 @@ def get_current_user_profile(user_jwt): def get_user_profile(user_jwt, email_of_profile, project_key): - f = furl(settings.SCIREG_REGISTRATION_URL) + f = furl(DBMI_REG_API_URL.url) f.args["email"] = email_of_profile f.args["project"] = 'Hypatio.' + project_key @@ -82,7 +81,7 @@ def get_distinct_countries_participating(user_jwt, participants, project_key): containing the unique countries of these participants and a count for each. """ - url = settings.SCIREG_REGISTRATION_URL + 'get_countries/' + url = (DBMI_REG_API_URL / 'get_countries/').url # From a QuerySet of participants, get a list of their emails emails = [participant.user.email for participant in participants] @@ -108,7 +107,7 @@ def get_names(user_jwt, participants, project_key): containing the first and last names of each participant. """ - url = settings.SCIREG_REGISTRATION_URL + 'get_names/' + url = (DBMI_REG_API_URL.url / 'get_names/').url # From a QuerySet of participants, get a list of their emails emails = list(participants.values_list('user__email', flat=True)) diff --git a/app/hypatio/settings.py b/app/hypatio/settings.py index d23a97d0..fdd870c4 100644 --- a/app/hypatio/settings.py +++ b/app/hypatio/settings.py @@ -12,11 +12,10 @@ import os import sys +import logging from os.path import normpath, join, dirname, abspath -from django.utils.crypto import get_random_string - -chars = 'abcdefghijklmnopqrstuvwxyz0123456789!@#$%^&*(-_=+)' +from dbmi_client import environment # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -25,14 +24,14 @@ # See https://docs.djangoproject.com/en/1.10/howto/deployment/checklist/ # SECURITY WARNING: keep the secret key used in production secret! -SECRET_KEY = os.environ.get("SECRET_KEY", get_random_string(50, chars)) +SECRET_KEY = environment.get_str("SECRET_KEY", required=True) # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = os.environ.get("DJANGO_DEBUG", False) +DEBUG = environment.get_bool("DJANGO_DEBUG", default=False) PROJECT = 'hypatio' -ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS").split(',') +ALLOWED_HOSTS = environment.get_list("ALLOWED_HOSTS", required=True) # Application definition @@ -48,24 +47,27 @@ 'jquery', 'bootstrap3', 'contact', + 'manage', 'django_countries', 'profile', 'projects', - 'pyauth0jwt', 'health_check', 'raven.contrib.django.raven_compat', 'bootstrap_datepicker_plus', + 'storages', + 'django_jsonfield_backport', + 'django_q', ] -MIDDLEWARE_CLASSES = [ +MIDDLEWARE = [ 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', - 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'hypatio.middleware.XRobotsTagMiddleware', ] ROOT_URLCONF = 'hypatio.urls' @@ -73,7 +75,10 @@ TEMPLATES = [ { 'BACKEND': 'django.template.backends.django.DjangoTemplates', - 'DIRS': [normpath(join(BASE_DIR, 'templates'))], + 'DIRS': [ + normpath(join(BASE_DIR, 'templates')), + normpath(join(dirname(dirname(abspath(__file__))), 'assets')), + ], 'APP_DIRS': True, 'OPTIONS': { 'context_processors': [ @@ -96,10 +101,10 @@ 'default': { 'ENGINE': 'django.db.backends.mysql', 'NAME': 'hypatio', - 'USER': os.environ.get("MYSQL_USERNAME"), - 'PASSWORD': os.environ.get("MYSQL_PASSWORD"), - 'HOST': os.environ.get("MYSQL_HOST"), - 'PORT': os.environ.get("MYSQL_PORT"), + 'USER': environment.get_str("MYSQL_USERNAME", default='hypatio'), + 'PASSWORD': environment.get_str("MYSQL_PASSWORD", required=True), + 'HOST': environment.get_str("MYSQL_HOST", required=True), + 'PORT': environment.get_str("MYSQL_PORT", '3306'), } } @@ -121,26 +126,9 @@ }, ] -SITE_URL = os.environ.get("SITE_URL") - -AUTH0_DOMAIN = os.environ.get("AUTH0_DOMAIN") -AUTH0_CLIENT_ID_LIST = os.environ.get("AUTH0_CLIENT_ID_LIST","").split(",") -AUTH0_SECRET = os.environ.get("AUTH0_SECRET") -AUTH0_SUCCESS_URL = os.environ.get("AUTH0_SUCCESS_URL") -AUTH0_LOGOUT_URL = os.environ.get("AUTH0_LOGOUT_URL","") - -AUTHENTICATION_BACKENDS = ['pyauth0jwt.auth0authenticate.Auth0Authentication', 'django.contrib.auth.backends.ModelBackend'] - -AUTHENTICATION_LOGIN_URL = os.environ.get("ACCOUNT_SERVER_URL") -ACCOUNT_SERVER_URL = os.environ.get("ACCOUNT_SERVER_URL") -SCIREG_SERVER_URL = os.environ.get("SCIREG_SERVER_URL", "") -AUTHZ_BASE = os.environ.get("AUTHZ_BASE", "") +SITE_URL = environment.get_str("SITE_URL", required=True) -USER_PERMISSIONS_URL = AUTHZ_BASE + "/user_permission/" - -SCIREG_REGISTRATION_URL = SCIREG_SERVER_URL + "/api/register/" - -COOKIE_DOMAIN = os.environ.get("COOKIE_DOMAIN") +AUTHENTICATION_BACKENDS = ['hypatio.auth0authenticate.Auth0Authentication', 'django.contrib.auth.backends.ModelBackend'] SSL_SETTING = "https" VERIFY_REQUESTS = True @@ -148,16 +136,16 @@ CONTACT_FORM_RECIPIENTS="dbmi_tech_core@hms.harvard.edu" DEFAULT_FROM_EMAIL="dbmi_tech_core@hms.harvard.edu" -RECAPTCHA_KEY = os.environ.get('RECAPTCHA_KEY') -RECAPTCHA_CLIENT_ID = os.environ.get('RECAPTCHA_CLIENT_ID') - -EMAIL_CONFIRM_SUCCESS_URL = os.environ.get('EMAIL_CONFIRM_SUCCESS_URL') +RECAPTCHA_KEY = environment.get_str('RECAPTCHA_KEY', required=True) +RECAPTCHA_CLIENT_ID = environment.get_str('RECAPTCHA_CLIENT_ID', required=True) ########## # S3 Configurations -S3_AWS_ACCESS_KEY_ID = os.environ.get('S3_AWS_ACCESS_KEY_ID') -S3_AWS_SECRET_ACCESS_KEY = os.environ.get('S3_AWS_SECRET_ACCESS_KEY') -S3_BUCKET = os.environ.get('S3_BUCKET') +S3_BUCKET = environment.get_str('S3_BUCKET', required=True) + +DEFAULT_FILE_STORAGE = 'storages.backends.s3boto3.S3Boto3Storage' +AWS_STORAGE_BUCKET_NAME = environment.get_str('S3_BUCKET', required=True) +AWS_LOCATION = 'upload' ########## @@ -213,27 +201,83 @@ ########## -EMAIL_BACKEND = os.environ.get("EMAIL_BACKEND", "django_smtp_ssl.SSLEmailBackend") +##################################################################################### +# DBMI Client Configurations +##################################################################################### + +DBMI_CLIENT_CONFIG = { + 'CLIENT': 'hypatio', + 'ENVIRONMENT': environment.get_str('DBMI_ENV', required=True), + 'ENABLE_LOGGING': True, + 'LOG_LEVEL': environment.get_int('DBMI_LOG_LEVEL', default=logging.WARNING), + + # AuthZ + 'AUTHZ_ADMIN_GROUP': 'hypatio-admins', + 'AUTHZ_ADMIN_PERMISSION': 'ADMIN', + 'JWT_COOKIE_DOMAIN': environment.get_str('COOKIE_DOMAIN', required=True), + 'AUTHN_TITLE': 'DBMI Portal', + + # Set auth configurations + 'AUTH_CLIENTS': environment.get_dict('AUTH_CLIENTS', required=True), + + # Fileservice + 'FILESERVICE_URL': environment.get_str('FILESERVICE_API_URL', required=True), + 'FILESERVICE_GROUP': environment.get_str('FILESERVICE_GROUP', required=True), + 'FILESERVICE_BUCKETS': [environment.get_str('FILESERVICE_AWS_BUCKET', required=True)], + 'FILESERVICE_TOKEN': environment.get_str('FILESERVICE_SERVICE_TOKEN', required=True), + + # Misc + 'DRF_OBJECT_OWNER_KEY': 'email', +} + +##################################################################################### + +##################################################################################### +# Email Configurations +##################################################################################### + +EMAIL_BACKEND = environment.get_str("EMAIL_BACKEND", "django_smtp_ssl.SSLEmailBackend") EMAIL_USE_SSL = EMAIL_BACKEND == 'django_smtp_ssl.SSLEmailBackend' -EMAIL_HOST = os.environ.get("EMAIL_HOST") -EMAIL_HOST_USER = os.environ.get("EMAIL_HOST_USER") -EMAIL_HOST_PASSWORD = os.environ.get("EMAIL_HOST_PASSWORD") -EMAIL_PORT = os.environ.get("EMAIL_PORT") +EMAIL_HOST = environment.get_str("EMAIL_HOST", required=True) +EMAIL_HOST_USER = environment.get_str("EMAIL_HOST_USER", required=not DEBUG) +EMAIL_HOST_PASSWORD = environment.get_str("EMAIL_HOST_PASSWORD", required=EMAIL_HOST_USER is not None) +EMAIL_PORT = environment.get_str("EMAIL_PORT", required=True) +##################################################################################### ##################################################################################### # FileService Configurations ##################################################################################### -FILESERVICE_API_URL = os.environ.get('FILESERVICE_API_URL') -FILESERVICE_GROUP = os.environ.get('FILESERVICE_GROUP') -FILESERVICE_AWS_BUCKET = os.environ.get('FILESERVICE_AWS_BUCKET') +FILESERVICE_API_URL = environment.get_str('FILESERVICE_API_URL', required=True) +FILESERVICE_GROUP = environment.get_str('FILESERVICE_GROUP', required=True) +FILESERVICE_AWS_BUCKET = environment.get_str('FILESERVICE_AWS_BUCKET', required=True) FILESERVICE_SERVICE_ACCOUNT = 'hypatio' -FILESERVICE_SERVICE_TOKEN = os.environ.get('FILESERVICE_SERVICE_TOKEN') +FILESERVICE_SERVICE_TOKEN = environment.get_str('FILESERVICE_SERVICE_TOKEN', required=True) FILESERVICE_AUTH_HEADER_PREFIX = 'Token' ##################################################################################### +##################################################################################### +# Django-Q settings +##################################################################################### + +Q_CLUSTER = { + 'name': 'hypatio', + 'workers': 8, + 'recycle': 500, + 'timeout': 18000, + 'compress': True, + 'save_limit': 250, + 'queue_limit': 500, + 'cpu_affinity': 1, + 'retry': 20000, + 'label': 'Hypatio Tasks', + 'orm': 'default', + 'guard_cycle': 60, +} + +##################################################################################### LOGGING = { 'version': 1, @@ -294,14 +338,9 @@ } RAVEN_CONFIG = { - 'dsn': os.environ.get("RAVEN_URL", ""), + 'dsn': environment.get_str("RAVEN_URL", required=True), # If you are using git, you can also automatically configure the # release based on the git info. 'release': '1', 'site': 'HYPATIO' } - -try: - from .local_settings import * -except ImportError: - pass diff --git a/app/hypatio/urls.py b/app/hypatio/urls.py index 93e08fad..3679f37a 100644 --- a/app/hypatio/urls.py +++ b/app/hypatio/urls.py @@ -1,5 +1,5 @@ -from django.conf.urls import include -from django.conf.urls import url +from django.urls import include +from django.urls import re_path from django.contrib import admin from hypatio.views import index @@ -9,14 +9,14 @@ urlpatterns = [ - url(r'^admin/', admin.site.urls), - url(r'^contact/', include('contact.urls', namespace='contact')), - url(r'^manage/', include('manage.urls', namespace='manage')), - url(r'^projects/', include('projects.urls', namespace='projects')), - url(r'^profile/', include('profile.urls', namespace='profile')), - url(r'^data-sets/?$', list_data_projects, name='data-sets'), - url(r'^data-challenges/?$', list_data_challenges, name='data-challenges'), - url(r'^software-projects/?$', list_software_projects, name='software-projects'), - url(r'^healthcheck/?', include('health_check.urls')), - url(r'^', index, name='index'), + re_path(r'^admin/', admin.site.urls), + re_path(r'^contact/', include('contact.urls', namespace='contact')), + re_path(r'^manage/', include('manage.urls', namespace='manage')), + re_path(r'^projects/', include('projects.urls', namespace='projects')), + re_path(r'^profile/', include('profile.urls', namespace='profile')), + re_path(r'^data-sets/?$', list_data_projects, name='data-sets'), + re_path(r'^data-challenges/?$', list_data_challenges, name='data-challenges'), + re_path(r'^software-projects/?$', list_software_projects, name='software-projects'), + re_path(r'^healthcheck/?', include('health_check.urls')), + re_path(r'^', index, name='index'), ] diff --git a/app/hypatio/views.py b/app/hypatio/views.py index 30e68720..6da9267e 100644 --- a/app/hypatio/views.py +++ b/app/hypatio/views.py @@ -1,6 +1,6 @@ from django.shortcuts import render -from pyauth0jwt.auth0authenticate import public_user_auth_and_jwt +from hypatio.auth0authenticate import public_user_auth_and_jwt @public_user_auth_and_jwt diff --git a/app/manage/admin.py b/app/manage/admin.py new file mode 100644 index 00000000..18a46e24 --- /dev/null +++ b/app/manage/admin.py @@ -0,0 +1,11 @@ +from django.contrib import admin + +from manage.models import ChallengeTaskSubmissionExport + + +class ChallengeTaskSubmissionExportAdmin(admin.ModelAdmin): + list_display = ('data_project', 'requester', 'request_date', 'uuid', ) + list_filter = ('data_project', 'requester', ) + + +admin.site.register(ChallengeTaskSubmissionExport, ChallengeTaskSubmissionExportAdmin) diff --git a/app/manage/api.py b/app/manage/api.py index bed8f0aa..d55539bb 100644 --- a/app/manage/api.py +++ b/app/manage/api.py @@ -5,15 +5,21 @@ import shutil import uuid import zipfile +import magic +from urllib.parse import urlparse +import urllib +from django_q.tasks import async_task +from dbmi_client import fileservice from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist from django.http import HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, redirect from django.template.loader import render_to_string +from django.core.files.storage import default_storage -from pyauth0jwt.auth0authenticate import user_auth_and_jwt +from hypatio.auth0authenticate import user_auth_and_jwt from contact.views import email_send from hypatio.sciauthz_services import SciAuthZ @@ -24,6 +30,7 @@ from manage.utils import zip_submission_file from projects.templatetags import projects_extras +from manage.models import ChallengeTaskSubmissionExport from projects.models import AgreementForm from projects.models import ChallengeTaskSubmission from projects.models import DataProject @@ -32,6 +39,8 @@ from projects.models import SignedAgreementForm from projects.models import Team from projects.models import TeamComment +from projects.serializers import HostedFileSerializer, HostedFileDownloadSerializer +from projects.models import AGREEMENT_FORM_TYPE_MODEL, AGREEMENT_FORM_TYPE_FILE # Get an instance of a logger logger = logging.getLogger(__name__) @@ -48,7 +57,7 @@ def set_dataproject_registration_status(request): project_key = request.POST.get("project_key") project = get_object_or_404(DataProject, project_key=project_key) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -92,7 +101,7 @@ def set_dataproject_visible_status(request): project_key = request.POST.get("project_key") project = get_object_or_404(DataProject, project_key=project_key) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -136,7 +145,7 @@ def set_dataproject_details(request): project_key = request.POST.get("project_key") project = get_object_or_404(DataProject, project_key=project_key) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -191,7 +200,7 @@ def get_static_agreement_form_html(request): project_key = request.GET.get("project-key") project = get_object_or_404(DataProject, project_key=project_key) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -210,10 +219,22 @@ def get_static_agreement_form_html(request): except ObjectDoesNotExist: return HttpResponse("Error: form not found.", status=404) - if agreement_form.form_file_path is None or agreement_form.form_file_path == "": - return HttpResponse("Error: form file path is missing.", status=400) + if agreement_form.type == AGREEMENT_FORM_TYPE_MODEL: + if agreement_form.content is None or agreement_form.content == "": + return HttpResponse("Error: form content is missing.", status=400) + + form_contents = agreement_form.content + elif agreement_form.type == AGREEMENT_FORM_TYPE_FILE: + if agreement_form.content is None or agreement_form.content == "": + return HttpResponse("Error: form content is missing.", status=400) + + form_contents = agreement_form.upload + else: + if agreement_form.form_file_path is None or agreement_form.form_file_path == "": + return HttpResponse("Error: form file path is missing.", status=400) + + form_contents = projects_extras.get_html_form_file_contents(agreement_form.form_file_path) - form_contents = projects_extras.get_html_form_file_contents(agreement_form.form_file_path) return HttpResponse(form_contents) @user_auth_and_jwt @@ -229,7 +250,7 @@ def get_hosted_file_edit_form(request): project_key = request.GET.get("project-key") project = get_object_or_404(DataProject, project_key=project_key) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -272,12 +293,12 @@ def get_hosted_file_logs(request): project_key = request.GET.get("project-key") project = get_object_or_404(DataProject, project_key=project_key) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: logger.debug( - '[HYPATIO][DEBUG][get_static_agreement_form_html] User {email} does not have MANAGE permissions for item {project_key}.'.format( + '[HYPATIO][DEBUG][get_hosted_file_logs] User {email} does not have MANAGE permissions for item {project_key}.'.format( email=user.email, project_key=project_key ) @@ -287,15 +308,30 @@ def get_hosted_file_logs(request): hosted_file_uuid = request.GET.get("hosted-file-uuid") try: - hosted_file = HostedFile.objects.get(project=project, uuid=hosted_file_uuid).select_related() + hosted_file = HostedFile.objects.get(project=project, uuid=hosted_file_uuid) except ObjectDoesNotExist: + logger.debug( + '[HYPATIO][DEBUG][get_hosted_file_logs] User {email} does not have MANAGE permissions for item {project_key}.'.format( + email=user.email, + project_key=project_key + ) + ) return HttpResponse("Error: file not found.", status=404) + except Exception as e: + logger.exception( + '[HYPATIO][EXCEPTION][get_hosted_file_logs] Could not perform fetch for ' + 'file download logs: {e}.'.format(e=e), exc_info=True, extra={ + 'user': user.email, 'project': project_key, 'hosted_file': hosted_file_uuid + } + ) + return HttpResponse("Error: fetch failed with error", status=500) response_html = render_to_string( 'manage/hosted-file-logs.html', context={ - 'downloads': hosted_file.hostedfiledownload_set, - 'file': hosted_file + 'downloads': [HostedFileDownloadSerializer(downloads).data for downloads in + hosted_file.hostedfiledownload_set.all().order_by('download_date')], + 'file': HostedFileSerializer(hosted_file).data, }, request=request ) @@ -314,7 +350,7 @@ def process_hosted_file_edit_form_submission(request): project_id = request.POST.get("project") project = get_object_or_404(DataProject, id=project_id) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -357,7 +393,7 @@ def download_signed_form(request): user_jwt = request.COOKIES.get("DBMI_JWT", None) project = signed_form.project - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -369,14 +405,64 @@ def download_signed_form(request): ) return HttpResponse("Error: permissions.", status=403) + # Set details of the file download affected_user = signed_form.user date_as_string = datetime.strftime(signed_form.date_signed, "%Y%m%d-%H%M") - filename = affected_user.email + '-' + signed_form.agreement_form.short_name + '-' + date_as_string + '.txt' - response = HttpResponse(signed_form.agreement_text, content_type='text/plain') - response['Content-Disposition'] = 'attachment; filename={0}'.format(filename) + # Check type + if signed_form.agreement_form.type == AGREEMENT_FORM_TYPE_FILE: + + # Download the file + file = signed_form.upload.open(mode="rb").read() + + # Determine MIME + mime = magic.from_buffer(file, mime=True) + + # Get the signed URL for the file in S3 + filename = signed_form.upload.name + response = HttpResponse(file, content_type=mime) + response['Content-Disposition'] = f'attachment; filename={filename}' + + else: + + filename = affected_user.email + '-' + signed_form.agreement_form.short_name + '-' + date_as_string + '.txt' + response = HttpResponse(signed_form.agreement_text, content_type='text/plain') + response['Content-Disposition'] = f'attachment; filename={filename}' + return response +@user_auth_and_jwt +def get_signed_form_status(request): + """ + An HTTP POST endpoint for fetching a signed form's status. + """ + form_id = request.POST.get("form_id") + + logger.debug(f'[get_signed_form_status] {request.user.email} -> {form_id}') + + # Get the form + signed_form = get_object_or_404(SignedAgreementForm, id=form_id) + + # Confirm permissions on project form was signed for + user = request.user + user_jwt = request.COOKIES.get("DBMI_JWT", None) + project = signed_form.project + + sciauthz = SciAuthZ(user_jwt, user.email) + is_manager = sciauthz.user_has_manage_permission(project.project_key) + + if not is_manager: + logger.debug( + '[HYPATIO][DEBUG][change_signed_form_status] User {email} does not have MANAGE permissions for item {project_key}.'.format( + email=user.email, + project_key=project.project_key + ) + ) + return HttpResponse("Error: permissions.", status=403) + + return HttpResponse(signed_form.status, status=200) + + @user_auth_and_jwt def change_signed_form_status(request): """ @@ -396,7 +482,7 @@ def change_signed_form_status(request): user_jwt = request.COOKIES.get("DBMI_JWT", None) project = signed_form.project - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -442,9 +528,13 @@ def change_signed_form_status(request): team.save() for member in team.participant_set.all(): - sciauthz = SciAuthZ(settings.AUTHZ_BASE, request.COOKIES.get("DBMI_JWT", None), request.user.email) + sciauthz = SciAuthZ(request.COOKIES.get("DBMI_JWT", None), request.user.email) sciauthz.remove_view_permission(signed_form.project.project_key, member.user.email) + # Remove their VIEW permission + member.permission = None + member.save() + logger.debug('[HYPATIO][change_signed_form_status] Emailing the whole team that their status has been moved to Ready because someone has a pending form') # Send an email notification to the team @@ -483,7 +573,7 @@ def save_team_comment(request): user = request.user user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -519,7 +609,7 @@ def set_team_status(request): user = request.user user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) logger.debug( @@ -562,7 +652,7 @@ def set_team_status(request): # If setting to Active, grant each team member access permissions. if status == "active": for member in team.participant_set.all(): - sciauthz = SciAuthZ(settings.AUTHZ_BASE, request.COOKIES.get("DBMI_JWT", None), request.user.email) + sciauthz = SciAuthZ(request.COOKIES.get("DBMI_JWT", None), request.user.email) sciauthz.create_view_permission(project_key, member.user.email) # Add permission to Participant @@ -572,7 +662,7 @@ def set_team_status(request): # If setting to Deactivated, revoke each team member's permissions. elif status == "deactivated": for member in team.participant_set.all(): - sciauthz = SciAuthZ(settings.AUTHZ_BASE, request.COOKIES.get("DBMI_JWT", None), request.user.email) + sciauthz = SciAuthZ(request.COOKIES.get("DBMI_JWT", None), request.user.email) sciauthz.remove_view_permission(project_key, member.user.email) # Remove permission from Participant @@ -617,7 +707,7 @@ def delete_team(request): user = request.user user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -635,9 +725,13 @@ def delete_team(request): # First revoke all VIEW permissions for member in team.participant_set.all(): - sciauthz = SciAuthZ(settings.AUTHZ_BASE, request.COOKIES.get("DBMI_JWT", None), request.user.email) + sciauthz = SciAuthZ(request.COOKIES.get("DBMI_JWT", None), request.user.email) sciauthz.remove_view_permission(project_key, member.user.email) + # Remove permission from Participant + member.permission = None + member.save() + logger.debug('[HYPATIO][delete_team] Sending a notification to team members.') # Then send a notification to the team members @@ -675,7 +769,7 @@ def download_team_submissions(request, project_key, team_leader_email): # Check permissions in SciAuthZ. user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -697,7 +791,7 @@ def download_team_submissions(request, project_key, team_leader_email): # For each submission, create a zip file and add the path to the list of zip files. for submission in submissions: - zip_file_path = zip_submission_file(submission, request) + zip_file_path = zip_submission_file(submission, request.user.email, request) zipped_submissions_paths.append(zip_file_path) # Create a directory to store the final encompassing zip file. @@ -740,7 +834,7 @@ def download_submission(request, fileservice_uuid): # Check permissions in SciAuthZ. user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -748,7 +842,7 @@ def download_submission(request, fileservice_uuid): return HttpResponse("You do not have access to download this file.", status=403) # Download the submission file from fileservice and zip it up with the info json. - zip_file_path = zip_submission_file(submission, request) + zip_file_path = zip_submission_file(submission, request.user.email, request) zip_file_name = os.path.basename(zip_file_path) # Prepare the zip file to be served. @@ -761,6 +855,70 @@ def download_submission(request, fileservice_uuid): return response + +@user_auth_and_jwt +def export_submissions(request, project_key): + """ + An HTTP GET endpoint that allows a user to download a ChallengeTaskSubmission's + file from AWS/fileservice. + """ + + if request.method == "GET": + + # Check permissions in SciAuthZ. + user_jwt = request.COOKIES.get("DBMI_JWT", None) + sciauthz = SciAuthZ(user_jwt, request.user.email) + is_manager = sciauthz.user_has_manage_permission(project_key) + + if not is_manager: + logger.debug("[download_team_submissions] - No Access for user " + request.user.email) + return HttpResponse("You do not have access to download this file.", status=403) + + project = get_object_or_404(DataProject, project_key=project_key) + + # Run the task + async_task('manage.tasks.export_task_submissions', project.id, request.user.email) + + # Prepare the zip file to be served. + return HttpResponse(status=201) + + +@user_auth_and_jwt +def download_submissions_export(request, project_key, fileservice_uuid): + """ + An HTTP GET endpoint that allows a user to download a ChallengeTask's + submissions export file from AWS/fileservice. + """ + if request.method == "GET": + + # Check permissions in SciAuthZ. + user_jwt = request.COOKIES.get("DBMI_JWT", None) + sciauthz = SciAuthZ(user_jwt, request.user.email) + is_manager = sciauthz.user_has_manage_permission(project_key) + + if not is_manager: + logger.debug("[download_submissions_export] - No Access for user " + request.user.email) + return HttpResponse("You do not have access to download this file.", status=403) + + # Get filename + filename = f"{project_key}_export_{fileservice_uuid}.zip" + + # Get the url + url = fileservice.get_archivefile_download_url(fileservice_uuid) + + # Prepare the parts + protocol = urlparse(url).scheme + path = urllib.parse.quote_plus(url.replace(protocol + '://', '')) + + # Let NGINX handle it + response = HttpResponse() + response['X-Accel-Redirect'] = '/proxy/' + protocol + '/' + path + response['Content-Disposition'] = 'attachment; filename="{}"'.format(filename) + + logger.debug(f'Sending user to S3 proxy: {response["X-Accel-Redirect"]}') + + return response + @user_auth_and_jwt def host_submission(request, fileservice_uuid): """ @@ -776,7 +934,7 @@ def host_submission(request, fileservice_uuid): submission = get_object_or_404(ChallengeTaskSubmission, uuid=fileservice_uuid) project = submission.challenge_task.data_project - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -828,7 +986,7 @@ def host_submission(request, fileservice_uuid): user = request.user user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project.project_key) if not is_manager: @@ -883,7 +1041,7 @@ def download_email_list(request): # Check Permissions in SciAuthZ user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -955,7 +1113,7 @@ def grant_view_permission(request, project_key, user_email): # Check Permissions in SciAuthZ user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project_key) logger.debug( @@ -1024,7 +1182,7 @@ def remove_view_permission(request, project_key, user_email): # Check Permissions in SciAuthZ user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project_key) logger.debug( @@ -1074,7 +1232,7 @@ def sync_view_permissions(request, project_key): # Check Permissions in SciAuthZ user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project_key) logger.debug( @@ -1106,10 +1264,4 @@ def sync_view_permissions(request, project_key): participant.permission = 'VIEW' participant.save() - elif participant.user.email.lower() not in permitted_emails: - - # Clear it - participant.permission = None - participant.save() - return HttpResponse(status=200) diff --git a/app/manage/apps.py b/app/manage/apps.py index 5a55842e..f935cb63 100644 --- a/app/manage/apps.py +++ b/app/manage/apps.py @@ -3,3 +3,4 @@ class ManageConfig(AppConfig): name = 'manage' + default_auto_field = 'django.db.models.BigAutoField' diff --git a/app/manage/forms.py b/app/manage/forms.py index c92240e2..4d142c45 100644 --- a/app/manage/forms.py +++ b/app/manage/forms.py @@ -3,8 +3,10 @@ from bootstrap_datepicker_plus import DateTimePickerInput from dal import autocomplete -from projects.models import DataProject +from projects.models import AgreementForm, DataProject from projects.models import HostedFile +from projects.models import Team +from projects.models import AGREEMENT_FORM_TYPE_FILE # TODO Convert all other manual forms into Django forms # ... @@ -55,3 +57,25 @@ class Meta: 'hostedfileset': autocomplete.ModelSelect2(url='projects:hostedfileset-autocomplete', forward=['project'], attrs={'class': 'form-control form-control-select2'}) } +class NotificationForm(forms.Form): + """ + Determines the fields that will appear. + """ + project = forms.ModelChoiceField(queryset=DataProject.objects.all(), widget=forms.HiddenInput) + message = forms.CharField(label='Message', required=True, widget=forms.Textarea) + team = forms.ModelChoiceField(queryset=Team.objects.all(), widget=forms.HiddenInput) + + +class UploadSignedAgreementFormForm(forms.Form): + agreement_form = forms.ModelChoiceField(queryset=AgreementForm.objects.filter(type=AGREEMENT_FORM_TYPE_FILE, internal=True), widget=forms.Select(attrs={'class': 'form-control'})) + project_key = forms.CharField(label='Project Key', max_length=128, required=True, widget=forms.HiddenInput()) + participant = forms.CharField(label='Participant', max_length=128, required=True, widget=forms.HiddenInput()) + signed_agreement_form = forms.FileField(label="Signed Agreement Form PDF", required=True) + + def __init__(self, *args, **kwargs): + project_key = kwargs.pop('project_key', None) + super(UploadSignedAgreementFormForm, self).__init__(*args, **kwargs) + + # Limit agreement form choices to those related to the passed project + if project_key: + self.fields['agreement_form'].queryset = DataProject.objects.get(project_key=project_key).agreement_forms.all() diff --git a/app/manage/migrations/0001_initial.py b/app/manage/migrations/0001_initial.py new file mode 100644 index 00000000..7d40c5ac --- /dev/null +++ b/app/manage/migrations/0001_initial.py @@ -0,0 +1,30 @@ +# Generated by Django 2.2.28 on 2022-06-02 14:35 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('projects', '0092_auto_20220517_1520'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='ChallengeTaskSubmissionExport', + fields=[ + ('request_date', models.DateTimeField(auto_now_add=True)), + ('uuid', models.UUIDField(default=None, primary_key=True, serialize=False, unique=True)), + ('location', models.CharField(blank=True, default=None, max_length=12, null=True)), + ('challenge_task_submissions', models.ManyToManyField(to='projects.ChallengeTaskSubmission')), + ('challenge_tasks', models.ManyToManyField(to='projects.ChallengeTask')), + ('data_project', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='projects.DataProject')), + ('requester', models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/app/manage/migrations/__init__.py b/app/manage/migrations/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/manage/models.py b/app/manage/models.py index 7658df9a..778b0308 100644 --- a/app/manage/models.py +++ b/app/manage/models.py @@ -1,2 +1,23 @@ from django.db import models -from django.contrib.auth.models import User \ No newline at end of file +from django.contrib.auth.models import User + +from projects.models import DataProject +from projects.models import ChallengeTask +from projects.models import ChallengeTaskSubmission + + +class ChallengeTaskSubmissionExport(models.Model): + """ + Captures the files that are generated as an export for admins. + """ + + data_project = models.ForeignKey(DataProject, on_delete=models.PROTECT) + challenge_tasks = models.ManyToManyField(ChallengeTask) + challenge_task_submissions = models.ManyToManyField(ChallengeTaskSubmission) + requester = models.ForeignKey(User, on_delete=models.PROTECT) + request_date = models.DateTimeField(auto_now_add=True) + uuid = models.UUIDField(null=False, unique=True, primary_key=True, default=None) + location = models.CharField(max_length=12, default=None, blank=True, null=True) + + def __str__(self): + return '%s' % (self.uuid) diff --git a/app/manage/tasks.py b/app/manage/tasks.py new file mode 100644 index 00000000..50fbb8dd --- /dev/null +++ b/app/manage/tasks.py @@ -0,0 +1,206 @@ +import uuid +import os +import shutil +import json +import requests +import tempfile +from datetime import datetime +from django_q.tasks import Chain, async_task +from django.conf import settings +from django.contrib.auth.models import User + +from projects.models import DataProject +from projects.models import ChallengeTaskSubmission +from projects.models import ChallengeTaskSubmissionDownload +from manage.models import ChallengeTaskSubmissionExport +from dbmi_client import fileservice +from manage.api import zip_submission_file +from contact.views import email_send + +import logging +logger = logging.getLogger(__name__) + + +def export_task_submissions(project_id, requester): + """ + This method fetches all current submissions for a challenge task + and prepares an export of all files along with associated metadata. + :param project_id: The ID of the DataProject to export submissions for + :type project_id: str + :param requester: The email of the admin requesting the export + :type requester: str + :return: Whether the operation succeeded or not + :rtype: bool + """ + try: + # Get project and challenge task + project = DataProject.objects.get(id=project_id) + + # Get all submissions made by this team for this project. + submissions = ChallengeTaskSubmission.objects.filter( + challenge_task__in=project.challengetask_set.all(), + deleted=False + ) + + # Create a temporary directory + export_uuid = export_location = None + with tempfile.TemporaryDirectory() as directory: + + # Create directory to put submissions in + submissions_directory_path = os.path.join(directory, f"{project.project_key}_submissions_{datetime.now().isoformat()}") + + # For each submission, create a directory for the file and its metadata file + submission_file_response = None + for submission in submissions: + try: + # Set the name of the containing directory + submission_directory_path = os.path.join(submissions_directory_path, f"{submission.participant.user.email}_{submission.uuid}") + + # Create a record of the user downloading the file. + ChallengeTaskSubmissionDownload.objects.create( + user=User.objects.get(email=requester), + submission=submission + ) + + # Create a temporary directory to hold the files specific to this submission that need to be zipped together. + if not os.path.exists(submission_directory_path): + os.makedirs(submission_directory_path) + + # Create a json file with the submission info string. + info_file_name = "submission_info.json" + with open(os.path.join(submission_directory_path, info_file_name), mode="w") as f: + f.write(submission.submission_info) + + # Determine filename + try: + submission_file_name = json.loads(submission.submission_info).get("filename") + if not submission_file_name: + + # Check fileservice + submission_file_name = fileservice.get_archivefile(submission.uuid)["filename"] + except Exception as e: + logger.exception( + f"Could not determine filename for submission", + exc_info=True, + extra={ + "submission": submission, + "archivefile_uuid": submission.uuid, + "submission_info": submission.submission_info, + } + ) + + # Use a default filename + submission_file_name = "submission_file.zip" + + # Get the submission file's byte contents from S3. + submission_file_download_url = fileservice.get_archivefile_proxy_url(uuid=submission.uuid) + headers = {"Authorization": f"{settings.FILESERVICE_AUTH_HEADER_PREFIX} {settings.FILESERVICE_SERVICE_TOKEN}"} + with requests.get(submission_file_download_url, headers=headers, stream=True) as submission_file_response: + submission_file_response.raise_for_status() + + # Write the submission file's bytes to a zip file. + with open(os.path.join(submission_directory_path, submission_file_name), mode="wb") as f: + shutil.copyfileobj(submission_file_response.raw, f) + + except requests.exceptions.HTTPError as e: + logger.exception( + f"{project.project_key}: Could not download submission '{submission.uuid}': {e}", + extra={ + "submission": submission, + "archivefile_uuid": submission.uuid, + "response": submission_file_response.content, + "status_code": submission_file_response.status_code + }) + + except Exception as e: + logger.exception( + f"{project.project_key}: Could not export submission '{submission.uuid}': {e}", + exc_info=True + ) + + # Set the archive name + archive_basename = f"{project.project_key}_submissions" + + # Archive the directory + archive_path = shutil.make_archive(archive_basename, "zip", submissions_directory_path) + + # Perform the request to upload the file + with open(archive_path, "rb") as file: + + # Build upload request + response = None + files = {"file": file} + try: + # Create the file in Fileservice + metadata = { + "project": project.project_key, + "type": "export", + } + tags = ["hypatio", "export", "submissions", project.project_key, requester] + export_uuid, upload_data = fileservice.create_archivefile_upload(os.path.basename(archive_path), metadata, tags) + + # Get the location + export_location = upload_data["locationid"] + + # Upload to S3 + response = requests.post(upload_data["post"]["url"], data=upload_data["post"]["fields"], files=files) + response.raise_for_status() + + # Mark the upload as complete + fileservice.uploaded_archivefile(export_uuid, export_location) + + except KeyError as e: + logger.error( + f'{project.project_key}: Failed export post generation: {upload_data}', + exc_info=True + ) + raise e + + except requests.exceptions.HTTPError as e: + logger.exception( + f'{project.project_key}: Failed export upload: {upload_data}', + extra={ + "response": response.content, + "status_code": response.status_code + } + ) + raise e + + except Exception as e: + logger.exception( + f"{project.project_key}: Could not export submissions: {e}", + exc_info=True, + ) + raise e + + # Create the model entry for the export + export = ChallengeTaskSubmissionExport.objects.create( + data_project=project, + requester=User.objects.get(email=requester), + uuid=export_uuid, + location=export_location, + ) + + # Set many to many fields + export.challenge_tasks.set(project.challengetask_set.all()) + export.challenge_task_submissions.set(submissions) + export.save() + + # Notify requester + email_send( + subject='DBMI Portal - Challenge Task Submissions Export', + recipients=[requester], + email_template='email_submissions_export_notification', + extra={"site_url": settings.SITE_URL, "project": project} + ) + + except Exception as e: + logger.exception( + f"Export challenge task submissions error: {e}", + exc_info=True, + extra={ + "project_id": project_id, + "requester": requester, + } + ) + raise e diff --git a/app/manage/urls.py b/app/manage/urls.py index 821cf8a3..6c3f3238 100644 --- a/app/manage/urls.py +++ b/app/manage/urls.py @@ -1,10 +1,12 @@ -from django.conf.urls import url +from django.urls import re_path from manage.apps import ManageConfig from manage.views import DataProjectListManageView from manage.views import DataProjectManageView from manage.views import manage_team from manage.views import ProjectParticipants +from manage.views import team_notification +from manage.views import UploadSignedAgreementFormView from manage.api import set_dataproject_details from manage.api import set_dataproject_registration_status @@ -14,6 +16,7 @@ from manage.api import process_hosted_file_edit_form_submission from manage.api import download_signed_form from manage.api import change_signed_form_status +from manage.api import get_signed_form_status from manage.api import save_team_comment from manage.api import set_team_status from manage.api import delete_team @@ -25,31 +28,38 @@ from manage.api import grant_view_permission from manage.api import remove_view_permission from manage.api import sync_view_permissions +from manage.api import export_submissions +from manage.api import download_submissions_export app_name = ManageConfig.name urlpatterns = [ - url(r'^$', DataProjectListManageView.as_view(), name='manage-projects'), - url(r'^download-email-list/?$', download_email_list, name='download-email-list'), - url(r'^set-dataproject-details/?$', set_dataproject_details, name='set-dataproject-details'), - url(r'^set-dataproject-registration-status/?$', set_dataproject_registration_status, name='set-dataproject-registration-status'), - url(r'^set-dataproject-visible-status/?$', set_dataproject_visible_status, name='set-dataproject-visible-status'), - url(r'^get-static-agreement-form-html/?$', get_static_agreement_form_html, name='get-static-agreement-form-html'), - url(r'^get-hosted-file-edit-form/?$', get_hosted_file_edit_form, name='get-hosted-file-edit-form'), - url(r'^get-hosted-file-logs/?$', get_hosted_file_logs, name='get-hosted-file-logs'), - url(r'^process-hosted-file-edit-form-submission/?$', process_hosted_file_edit_form_submission, name='process-hosted-file-edit-form-submission'), - url(r'^download-signed-form/?$', download_signed_form, name='download-signed-form'), - url(r'^change-signed-form-status/?$', change_signed_form_status, name='change-signed-form-status'), - url(r'^save-team-comment/?$', save_team_comment, name='save-team-comment'), - url(r'^set-team-status/?$', set_team_status, name='set-team-status'), - url(r'^delete-team/?$', delete_team, name='delete-team'), - url(r'^download-team-submissions/(?P[^/]+)/(?P[^/]+)/?$', download_team_submissions, name='download-team-submissions'), - url(r'^download-submission/(?P[^/]+)/?$', download_submission, name='download-submission'), - url(r'^host-submission/(?P[^/]+)/?$', host_submission, name='host-submission'), - url(r'^sync-view-permissions/(?P[^/]+)/?$', sync_view_permissions, name='sync-view-permissions'), - url(r'^grant-view-permission/(?P[^/]+)/(?P[^/]+)/?$', grant_view_permission, name='grant-view-permission'), - url(r'^remove-view-permission/(?P[^/]+)/(?P[^/]+)/?$', remove_view_permission, name='remove-view-permission'), - url(r'^get-project-participants/(?P[^/]+)/?$', ProjectParticipants.as_view(), name='get-project-participants'), - url(r'^(?P[^/]+)/?$', DataProjectManageView.as_view(), name='manage-project'), - url(r'^(?P[^/]+)/(?P[^/]+)/?$', manage_team, name='manage-team'), + re_path(r'^$', DataProjectListManageView.as_view(), name='manage-projects'), + re_path(r'^download-email-list/?$', download_email_list, name='download-email-list'), + re_path(r'^set-dataproject-details/?$', set_dataproject_details, name='set-dataproject-details'), + re_path(r'^set-dataproject-registration-status/?$', set_dataproject_registration_status, name='set-dataproject-registration-status'), + re_path(r'^set-dataproject-visible-status/?$', set_dataproject_visible_status, name='set-dataproject-visible-status'), + re_path(r'^get-static-agreement-form-html/?$', get_static_agreement_form_html, name='get-static-agreement-form-html'), + re_path(r'^get-hosted-file-edit-form/?$', get_hosted_file_edit_form, name='get-hosted-file-edit-form'), + re_path(r'^get-hosted-file-logs/?$', get_hosted_file_logs, name='get-hosted-file-logs'), + re_path(r'^process-hosted-file-edit-form-submission/?$', process_hosted_file_edit_form_submission, name='process-hosted-file-edit-form-submission'), + re_path(r'^download-signed-form/?$', download_signed_form, name='download-signed-form'), + re_path(r'^get-signed-form-status/?$', get_signed_form_status, name='get-signed-form-status'), + re_path(r'^change-signed-form-status/?$', change_signed_form_status, name='change-signed-form-status'), + re_path(r'^save-team-comment/?$', save_team_comment, name='save-team-comment'), + re_path(r'^set-team-status/?$', set_team_status, name='set-team-status'), + re_path(r'^delete-team/?$', delete_team, name='delete-team'), + re_path(r'^team-notification/?$', team_notification, name='team-notification'), + re_path(r'^download-team-submissions/(?P[^/]+)/(?P[^/]+)/?$', download_team_submissions, name='download-team-submissions'), + re_path(r'^download-submission/(?P[^/]+)/?$', download_submission, name='download-submission'), + re_path(r'^export-submissions/(?P[^/]+)/?$', export_submissions, name='export-submissions'), + re_path(r'^download-submissions-export/(?P[^/]+)/(?P[^/]+)/?$', download_submissions_export, name='download-submissions-export'), + re_path(r'^host-submission/(?P[^/]+)/?$', host_submission, name='host-submission'), + re_path(r'^sync-view-permissions/(?P[^/]+)/?$', sync_view_permissions, name='sync-view-permissions'), + re_path(r'^grant-view-permission/(?P[^/]+)/(?P[^/]+)/?$', grant_view_permission, name='grant-view-permission'), + re_path(r'^remove-view-permission/(?P[^/]+)/(?P[^/]+)/?$', remove_view_permission, name='remove-view-permission'), + re_path(r'^get-project-participants/(?P[^/]+)/?$', ProjectParticipants.as_view(), name='get-project-participants'), + re_path(r'^upload-signed-agreement-form/(?P[^/]+)/(?P[^/]+)/?$', UploadSignedAgreementFormView.as_view(), name='upload-signed-agreement-form'), + re_path(r'^(?P[^/]+)/?$', DataProjectManageView.as_view(), name='manage-project'), + re_path(r'^(?P[^/]+)/(?P[^/]+)/?$', manage_team, name='manage-team'), ] diff --git a/app/manage/utils.py b/app/manage/utils.py index d03e37da..1e5f04fb 100644 --- a/app/manage/utils.py +++ b/app/manage/utils.py @@ -4,26 +4,35 @@ import uuid import zipfile import requests +from django.conf import settings +from django.contrib.auth.models import User +from dbmi_client import fileservice -from hypatio import file_services as fileservice - +from hypatio import file_services from projects.models import ChallengeTaskSubmissionDownload # Get an instance of a logger logger = logging.getLogger(__name__) -def zip_submission_file(submission, request): +def zip_submission_file(submission, requester, request=None): """ Creates a zip file containing a ChallengeTaskSubmission's file and the info json. The submission file is pulled from fileservice. - Returns the path to the zip file. + :param submission: The submission object to zip + :type submission: ChallengeTaskSubmission + :param requester: The email of the admin requesting the export + :type requester: str + :param request: The current request, if any + :type: request: HttpRequest + :returns: Returns the path to the zip file. + :rtype: str """ # Create a record of the user downloading the file. - download_record = ChallengeTaskSubmissionDownload.objects.create( - user=request.user, + ChallengeTaskSubmissionDownload.objects.create( + user=User.objects.get(email=requester), submission=submission ) @@ -38,8 +47,11 @@ def zip_submission_file(submission, request): f.write(submission.submission_info) # Get the submission file's byte contents from S3. - submission_file_download_url = fileservice.get_fileservice_download_url(request, submission.uuid) - submission_file_request = requests.get(submission_file_download_url) + submission_file_download_url = fileservice.get_archivefile_proxy_url(uuid=submission.uuid) + + # Use token in headers + headers = {"Authorization": f"{settings.FILESERVICE_AUTH_HEADER_PREFIX} {settings.FILESERVICE_SERVICE_TOKEN}"} + submission_file_request = requests.get(submission_file_download_url, headers=headers) # Write the submission file's bytes to a zip file. submission_file_name = "submission_file.zip" diff --git a/app/manage/views.py b/app/manage/views.py index f7151ed8..64a98cb7 100644 --- a/app/manage/views.py +++ b/app/manage/views.py @@ -1,23 +1,32 @@ import logging - -from pyauth0jwt.auth0authenticate import user_auth_and_jwt +from datetime import datetime +from hypatio.auth0authenticate import user_auth_and_jwt from django.conf import settings from django.contrib.auth.models import User from django.core.exceptions import ObjectDoesNotExist from django.db.models import Count from django.db.models import F -from django.http import HttpResponse, JsonResponse +from django.db.models import Q +from django.http import HttpResponse, JsonResponse, HttpResponseRedirect +from django.contrib import messages from django.shortcuts import render from django.utils.decorators import method_decorator from django.views.generic import TemplateView from django.views.generic.base import View from django.core.paginator import Paginator +from django.urls import reverse +from django.core.mail import EmailMultiAlternatives +from django.template.loader import render_to_string +from dbmi_client import fileservice from hypatio.sciauthz_services import SciAuthZ from hypatio.scireg_services import get_user_profile, get_distinct_countries_participating -from projects.models import ChallengeTaskSubmission +from manage.forms import NotificationForm +from manage.models import ChallengeTaskSubmissionExport +from manage.forms import UploadSignedAgreementFormForm +from projects.models import AgreementForm, ChallengeTaskSubmission from projects.models import DataProject from projects.models import Participant from projects.models import Team @@ -25,10 +34,17 @@ from projects.models import SignedAgreementForm from projects.models import HostedFile from projects.models import HostedFileDownload +from projects.models import SIGNED_FORM_APPROVED # Get an instance of a logger logger = logging.getLogger(__name__) + +def is_ajax(request): + # Returns whether a request is a vanilla ajax request or not + return request.META.get('HTTP_X_REQUESTED_WITH') == 'XMLHttpRequest' + + @method_decorator(user_auth_and_jwt, name='dispatch') class DataProjectListManageView(TemplateView): """ @@ -56,7 +72,7 @@ def get_context_data(self, **kwargs): context = super(DataProjectListManageView, self).get_context_data(**kwargs) user_jwt = self.request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, self.request.user.email) + sciauthz = SciAuthZ(user_jwt, self.request.user.email) projects_managed = sciauthz.get_projects_managed_by_user() context['projects'] = projects_managed @@ -89,7 +105,7 @@ def dispatch(self, request, *args, **kwargs): user_jwt = request.COOKIES.get("DBMI_JWT", None) - self.sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + self.sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = self.sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -140,6 +156,9 @@ def get_context_data(self, **kwargs): teams = [] for team in self.project.team_set.all(): + # Allow hiding of teams + team_hidden = False + team_downloads = 0 team_uploads = 0 @@ -154,13 +173,29 @@ def get_context_data(self, **kwargs): except ObjectDoesNotExist: team_uploads += 0 - teams.append({ - 'team_leader': team.team_leader.email, - 'member_count': team.participant_set.all().count(), - 'status': team.status, - 'downloads': team_downloads, - 'submissions': team_uploads, - }) + # If this is a project that is using shared teams, determine if this team should be hidden or not + # This is required since shared teams don't implicitly have all forms completed. + if self.project.hide_incomplete_teams and self.project.teams_source and not team_hidden: + + # Get all related signed agreement forms + signed_agreement_forms = SignedAgreementForm.objects.filter( + Q(user=participant.user, agreement_form__in=self.project.agreement_forms.all()) & + (Q(project=self.project) | Q(project__shares_agreement_forms=True)) + ) + + # Compare number of agreement forms + if len(signed_agreement_forms) < len(self.project.agreement_forms.all()): + logger.debug(f"{self.project.project_key}/{team.id}/{participant.user.email}: {len(signed_agreement_forms)} SignedAgreementForms: HIDDEN") + team_hidden = True + + if not team_hidden: + teams.append({ + 'team_leader': team.team_leader.email, + 'member_count': team.participant_set.all().count(), + 'status': team.status, + 'downloads': team_downloads, + 'submissions': team_uploads, + }) context['teams'] = teams @@ -187,6 +222,15 @@ def get_context_data(self, **kwargs): deleted=False ) + # Collect all submissions made for tasks related to this project. + for export in ChallengeTaskSubmissionExport.objects.filter( + data_project=self.project, + ).order_by("-request_date"): + export.download_url = fileservice.get_archivefile_proxy_url(uuid=export.uuid) + + # Add it to context + context.setdefault('submissions_exports', []).append(export) + context['num_required_forms'] = self.project.agreement_forms.count() # Get information about what files there are for this project. @@ -263,14 +307,27 @@ def get(self, request, project_key, *args, **kwargs): else: sort_order = ['user__email'] if order_direction == 'asc' else ['-user__email'] - # Paginate participants - query_set = project.participant_set.filter(user__email__icontains=search).order_by(*sort_order).all() \ - if search else project.participant_set.order_by(*sort_order).all() - paginator = Paginator(query_set, length) + # Get list of SignedAgreementForms for this project so we can hide Participants that have yet to complete at + # least one of the required forms + ready_users = [ + s.user for s in SignedAgreementForm.objects.filter( + Q(agreement_form__in=project.agreement_forms.all()) & + (Q(project=project) | Q(project__shares_agreement_forms=True)) + ).select_related("user") + ] + logger.debug(f"{project.project_key}: {len(ready_users)} Ready Participants") + + # Set queryset + query_set = project.participant_set.filter(user__in=ready_users).order_by(*sort_order) + + # Setup paginator + paginator = Paginator( + query_set.filter(user__email__icontains=search) if search else query_set, + length + ) # Determine page index (1-index) from DT parameters page = start / length + 1 - logger.debug('Participant page: {}'.format(page)) participant_page = paginator.page(page) participants = [] @@ -291,11 +348,24 @@ def get(self, request, project_key, *args, **kwargs): # For each of the available agreement forms for this project, display only latest version completed by the user for agreement_form in project.agreement_forms.all(): - signed_form = SignedAgreementForm.objects.filter( - user__email=participant.user.email, - project=project, - agreement_form=agreement_form - ).last() + + # Check if this project uses shared agreement forms + if project.shares_agreement_forms: + + # Fetch without a specific project + signed_form = SignedAgreementForm.objects.filter( + user__email=participant.user.email, + agreement_form=agreement_form, + ).last() + + else: + + # Fetch only for this project + signed_form = SignedAgreementForm.objects.filter( + user__email=participant.user.email, + project=project, + agreement_form=agreement_form + ).last() if signed_form is not None: signed_agreement_forms.append(signed_form) @@ -311,7 +381,8 @@ def get(self, request, project_key, *args, **kwargs): { 'status': f.status, 'id': f.id, - 'name': f.agreement_form.short_name + 'name': f.agreement_form.short_name, + 'project': f.project.project_key, } for f in signed_agreement_forms ], { @@ -334,7 +405,7 @@ def get(self, request, project_key, *args, **kwargs): # Build DataTables response data data = { 'draw': draw, - 'recordsTotal': project.participant_set.count(), + 'recordsTotal': query_set.count(), 'recordsFiltered': paginator.count, 'data': participants, 'error': None, @@ -343,6 +414,128 @@ def get(self, request, project_key, *args, **kwargs): return JsonResponse(data=data) +@user_auth_and_jwt +def team_notification(request, project_key=None): + """ + Manages sending notifications to team leaders + + :param request: The current HTTP request + :type request: HttpRequest + """ + # If this is a POST request we need to process the form data. + if request.method == 'POST': + logger.debug(f"Team notification: POST") + + # Process the form. + form = NotificationForm(request.POST) + if form.is_valid(): + + # Get the project + project = form.cleaned_data['project'] + team = form.cleaned_data['team'] + + # Form the context. + context = { + 'administrator_message': form.cleaned_data['message'], + 'project': project, + 'team': team, + 'site_url': settings.SITE_URL + } + + # Send it out. + email_template='team_notification' + subject='DBMI Portal - Team Notification' + + # Render templates + msg_html = render_to_string('email/email_team_notification.html', context) + msg_plain = render_to_string('email/email_team_notification.txt', context) + + try: + msg = EmailMultiAlternatives(subject=subject, + body=msg_plain, + from_email=settings.DEFAULT_FROM_EMAIL, + to=[team.team_leader.email]) + msg.attach_alternative(msg_html, "text/html") + msg.send() + + # Handle outcome + if is_ajax(request): + return HttpResponse('SUCCESS', status=200) + else: + # Set a message. + messages.success(request, 'Thanks, your notification has been sent!') + + except Exception as ex: + logger.exception(ex, exc_info=True, extra={ + 'email': email_template, 'extra': context + }) + + # Check how the request was made. + if is_ajax(request): + return HttpResponse('ERROR', status=500) + else: + messages.error(request, 'An unexpected error occurred, please try again') + + # Send them back + return HttpResponseRedirect(reverse( + 'projects:view-project', + kwargs={'project_key': form.cleaned_data['project']} + )) + else: + logger.error(f"Invalid team notification form", extra={ + 'request': request, 'errors': form.errors.as_json(), + }) + + # Check how the request was made. + if is_ajax(request): + return HttpResponse(form.errors.as_json(), status=500) + else: + messages.error(request, 'The form was invalid, please try again') + return HttpResponseRedirect(reverse( + 'projects:view-project', + kwargs={'project_key': form.cleaned_data['project']} + )) + + logger.debug(f"Team notification: GET") + + # If a GET (or any other method) we'll create a blank form. + initial = {} + + # If a project key was supplied and it matches a real project, pre-populate the form with it. + try: + if project_key: + data_project = DataProject.objects.get(project_key=project_key) + else: + data_project = DataProject.objects.get(id=request.GET["project"]) + + initial['project'] = data_project + except ObjectDoesNotExist: + logger.exception(f"Could not determine project", exc_info=True, extra={ + 'request': request, + }) + if is_ajax(request): + return HttpResponse('The project could not be determined, cannot send message.', status=500) + else: + messages.error(request, 'The project could not be determined, cannot send message.') + + # Get the team + try: + team = Team.objects.get(id=request.GET["team"]) + initial['team'] = team + except ObjectDoesNotExist: + logger.exception(f"Could not determine team leader", exc_info=True, extra={ + 'request': request, + }) + if is_ajax(request): + return HttpResponse('The team leader could not be determined, cannot send message.', status=500) + else: + messages.error(request, 'The team leader could not be determined, cannot send message.') + + # Generate and render the form. + form = NotificationForm(initial=initial) + return render(request, 'manage/notification.html', {'notification_form': form}) + + @user_auth_and_jwt def manage_team(request, project_key, team_leader, template_name='manage/team.html'): """ @@ -352,7 +545,7 @@ def manage_team(request, project_key, team_leader, template_name='manage/team.ht user = request.user user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, user.email) + sciauthz = SciAuthZ(user_jwt, user.email) is_manager = sciauthz.user_has_manage_permission(project_key) if not is_manager: @@ -386,19 +579,31 @@ def manage_team(request, project_key, team_leader, template_name='manage/team.ht else: user_info = None - # Make a request to DBMIAuthZ to check for this person's permissions. - access_granted = sciauthz.user_has_single_permission(project_key, "VIEW", email) + # Check if this participant has access + access_granted = member.permission == "VIEW" signed_agreement_forms = [] signed_accepted_agreement_forms = 0 # For each of the available agreement forms for this project, display only latest version completed by the user for agreement_form in project.agreement_forms.all(): - signed_form = SignedAgreementForm.objects.filter( - user__email=email, - project=project, - agreement_form=agreement_form - ).last() + + # If this project accepts agreement forms from other projects, check those + if project.shares_agreement_forms: + + # Fetch without a specific project + signed_form = SignedAgreementForm.objects.filter( + user__email=email, + agreement_form=agreement_form, + ).last() + + else: + # Fetch only for the current project + signed_form = SignedAgreementForm.objects.filter( + user__email=email, + project=project, + agreement_form=agreement_form + ).last() if signed_form is not None: signed_agreement_forms.append(signed_form) @@ -407,6 +612,15 @@ def manage_team(request, project_key, team_leader, template_name='manage/team.ht team_accepted_forms += 1 signed_accepted_agreement_forms += 1 + # Add internal signed agreement forms + for signed_agreement_form in SignedAgreementForm.objects.filter( + agreement_form__internal=True, + user__email=email, + project=project): + + # Add it + signed_agreement_forms.append(signed_agreement_form) + team_member_details.append({ 'email': email, 'user_info': user_info, @@ -446,3 +660,101 @@ def manage_team(request, project_key, team_leader, template_name='manage/team.ht } return render(request, template_name, context=context) + + +@method_decorator([user_auth_and_jwt], name='dispatch') +class UploadSignedAgreementFormView(View): + """ + View to upload signed agreement forms for participants. + + * Requires token authentication. + * Only admin users are able to access this view. + """ + def get(self, request, project_key, user_email, *args, **kwargs): + """ + Return the upload form template + """ + user = request.user + user_jwt = request.COOKIES.get("DBMI_JWT", None) + + sciauthz = SciAuthZ(user_jwt, user.email) + is_manager = sciauthz.user_has_manage_permission(project_key) + + if not is_manager: + logger.debug('User {email} does not have MANAGE permissions for item {project_key}.'.format( + email=user.email, + project_key=project_key + )) + return HttpResponse(403) + + # Return file upload form + form = UploadSignedAgreementFormForm(initial={ + "project_key": project_key, + "participant": user_email, + }) + + # Set context + context = { + "form": form, + "project_key": project_key, + "user_email": user_email, + } + + # Render html + return render(request, "manage/upload-signed-agreement-form.html", context) + + def post(self, request, project_key, user_email, *args, **kwargs): + """ + Process the form + """ + user = request.user + user_jwt = request.COOKIES.get("DBMI_JWT", None) + + sciauthz = SciAuthZ(user_jwt, user.email) + is_manager = sciauthz.user_has_manage_permission(project_key) + + if not is_manager: + logger.debug('User {email} does not have MANAGE permissions for item {project_key}.'.format( + email=user.email, + project_key=project_key + )) + return HttpResponse(403) + + # Assembles the form and run validation. + form = UploadSignedAgreementFormForm(data=request.POST, files=request.FILES) + if not form.is_valid(): + logger.warning('Form failed: {}'.format(form.errors.as_json())) + return HttpResponse(status=400) + + logger.debug(f"[upload_signed_agreement_form] Data -> {form.cleaned_data}") + + signed_agreement_form = form.cleaned_data['signed_agreement_form'] + agreement_form = form.cleaned_data['agreement_form'] + project_key = form.cleaned_data['project_key'] + participant_email = form.cleaned_data['participant'] + + project = DataProject.objects.get(project_key=project_key) + participant = Participant.objects.get(project=project, user__email=participant_email) + + signed_agreement_form = SignedAgreementForm( + user=participant.user, + agreement_form=agreement_form, + project=project, + date_signed=datetime.now(), + upload=signed_agreement_form, + status=SIGNED_FORM_APPROVED, + ) + signed_agreement_form.save() + + # Create the response. + response = HttpResponse(status=201) + + # Setup the script run. + response['X-IC-Script'] = "notify('{}', '{}', 'glyphicon glyphicon-{}');".format( + "success", "Signed agreement form successfully uploaded", "thumbs-up" + ) + + # Close the modal + response['X-IC-Script'] += "$('#page-modal').modal('hide');" + + return response diff --git a/app/profile/urls.py b/app/profile/urls.py index b8aa55f4..b5ac42b7 100644 --- a/app/profile/urls.py +++ b/app/profile/urls.py @@ -1,14 +1,14 @@ -from django.conf.urls import url +from django.urls import re_path from profile.views import profile from profile.views import send_confirmation_email_view from profile.views import signout from profile.views import update_profile - +app_name = 'profile' urlpatterns = [ - url(r'^$', profile, name='profile'), - url(r'^send_confirmation_email/?$', send_confirmation_email_view, name='send_confirmation_email'), - url(r'^update/?$', update_profile, name='update'), - url(r'^signout/?$', signout, name='signout'), -] \ No newline at end of file + re_path(r'^$', profile, name='profile'), + re_path(r'^send_confirmation_email/?$', send_confirmation_email_view, name='send_confirmation_email'), + re_path(r'^update/?$', update_profile, name='update'), + re_path(r'^signout/?$', signout, name='signout'), +] diff --git a/app/profile/views.py b/app/profile/views.py index 5f675623..14e50186 100644 --- a/app/profile/views.py +++ b/app/profile/views.py @@ -2,20 +2,22 @@ import logging import requests -from pyauth0jwt.auth0authenticate import user_auth_and_jwt -from pyauth0jwt.auth0authenticate import validate_request as validate_jwt -from pyauth0jwt.auth0authenticate import logout_redirect - from django.conf import settings from django.contrib import messages from django.contrib.auth import logout from django.http import HttpResponse from django.shortcuts import redirect from django.shortcuts import render +from django.urls import reverse +from furl import furl +from dbmi_client.settings import dbmi_settings +from dbmi_client.authn import logout_redirect_url from hypatio import scireg_services - -from .forms import RegistrationForm +from hypatio.auth0authenticate import user_auth_and_jwt +from hypatio.auth0authenticate import validate_request as validate_jwt +from hypatio.auth0authenticate import logout_redirect +from profile.forms import RegistrationForm # Get an instance of a logger logger = logging.getLogger(__name__) @@ -24,8 +26,7 @@ @user_auth_and_jwt def signout(request): logout(request) - response = redirect(settings.AUTH0_LOGOUT_URL) - response.delete_cookie('DBMI_JWT', domain=settings.COOKIE_DOMAIN) + response = redirect(logout_redirect_url(request, request.build_absolute_uri(reverse("index")))) return response @@ -51,12 +52,13 @@ def update_profile(request): logger.debug('[HYPATIO][DEBUG] Profile form fields submitted: ' + json.dumps(registration_form.cleaned_data)) # Create a new registration with a POST + url = furl(dbmi_settings.REG_URL) / "api" / "register" if registration_form.cleaned_data['id'] == "": - requests.post(settings.SCIREG_REGISTRATION_URL, headers=jwt_headers, data=json.dumps(registration_form.cleaned_data), verify=settings.VERIFY_REQUESTS) + requests.post((url / "/").url, headers=jwt_headers, data=json.dumps(registration_form.cleaned_data)) # Update an existing registration with a PUT to the specific ID else: - registration_url = settings.SCIREG_REGISTRATION_URL + registration_form.cleaned_data['id'] + '/' - requests.put(registration_url, headers=jwt_headers, data=json.dumps(registration_form.cleaned_data), verify=settings.VERIFY_REQUESTS) + url.path.segments.extend([registration_form.cleaned_data['id'], ""]) + requests.put(url.url, headers=jwt_headers, data=json.dumps(registration_form.cleaned_data)) return HttpResponse(200) else: @@ -74,7 +76,8 @@ def profile(request, template_name='profile/profile.html'): jwt_headers = {"Authorization": "JWT " + user_jwt, 'Content-Type': 'application/json'} # Query SciReg to get the user's information - registration_info = requests.get(settings.SCIREG_REGISTRATION_URL, headers=jwt_headers, verify=settings.VERIFY_REQUESTS).json() + url = furl(dbmi_settings.REG_URL) / "api" / "register" / "/" + registration_info = requests.get(url.url, headers=jwt_headers).json() logger.debug('[HYPATIO][DEBUG] Registration info ' + json.dumps(registration_info)) diff --git a/app/projects/admin.py b/app/projects/admin.py index 07c40efc..e8fe6c49 100644 --- a/app/projects/admin.py +++ b/app/projects/admin.py @@ -1,4 +1,6 @@ from django.contrib import admin +from django.urls import reverse +from django.utils.html import escape, mark_safe from projects.models import DataProject from projects.models import AgreementForm @@ -12,6 +14,12 @@ from projects.models import ChallengeTask from projects.models import ChallengeTaskSubmission from projects.models import ChallengeTaskSubmissionDownload +from projects.models import NLPDUASignedAgreementFormFields +from projects.models import NLPWHYSignedAgreementFormFields +from projects.models import DUASignedAgreementFormFields +from projects.models import ROCSignedAgreementFormFields +from projects.models import MAYOSignedAgreementFormFields +from projects.models import MIMIC3SignedAgreementFormFields class DataProjectAdmin(admin.ModelAdmin): list_display = ('name', 'project_key', 'informational_only', 'registration_open', 'requires_authorization', 'is_challenge', 'order') @@ -62,6 +70,67 @@ class ChallengeTaskSubmissionDownloadAdmin(admin.ModelAdmin): list_display = ('user', 'submission', 'download_date') search_fields = ('user__email', ) + +class SignedAgreementFormFieldsAdmin(admin.ModelAdmin): + def get_user(self, obj): + return obj.signed_agreement_form.user.email + get_user.short_description = 'User' + get_user.admin_order_field = 'signed_agreement_form__user__email' + + def get_status(self, obj): + return obj.signed_agreement_form.status + get_status.short_description = 'Status' + get_status.admin_order_field = 'signed_agreement_form__status' + + def signed_agreement_form_link(self, obj): + link = reverse("admin:projects_signedagreementform_change", args=[obj.signed_agreement_form.id]) + return mark_safe(f'{escape(obj.signed_agreement_form.__str__())}') + + signed_agreement_form_link.short_description = 'Signed Agreement Form' + signed_agreement_form_link.admin_order_field = 'signed agreement form' + + list_display = ( + 'get_user', + 'get_status', + 'signed_agreement_form_link' + ) + search_fields = ( + 'signed_agreement_form__user__email', + 'signed_agreement_form__agreement_form__project', + 'signed_agreement_form__agreement_form__short_name', + 'signed_agreement_form', + ) + readonly_fields = ( + 'signed_agreement_form', + 'created', + 'modified' + ) + + +class NLPDUASignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): + pass + + +class NLPWHYSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): + pass + + +class DUASignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): + pass + + +class ROCSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): + pass + + +class MAYOSignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): + pass + + +class MIMIC3SignedAgreementFormFieldsAdmin(SignedAgreementFormFieldsAdmin): + pass + + admin.site.register(DataProject, DataProjectAdmin) admin.site.register(AgreementForm, AgreementformAdmin) admin.site.register(SignedAgreementForm, SignedagreementformAdmin) @@ -74,3 +143,11 @@ class ChallengeTaskSubmissionDownloadAdmin(admin.ModelAdmin): admin.site.register(ChallengeTask, ChallengeTaskAdmin) admin.site.register(ChallengeTaskSubmission, ChallengeTaskSubmissionAdmin) admin.site.register(ChallengeTaskSubmissionDownload, ChallengeTaskSubmissionDownloadAdmin) + + +admin.site.register(NLPDUASignedAgreementFormFields, NLPDUASignedAgreementFormFieldsAdmin) +admin.site.register(NLPWHYSignedAgreementFormFields, NLPWHYSignedAgreementFormFieldsAdmin) +admin.site.register(DUASignedAgreementFormFields, DUASignedAgreementFormFieldsAdmin) +admin.site.register(ROCSignedAgreementFormFields, ROCSignedAgreementFormFieldsAdmin) +admin.site.register(MAYOSignedAgreementFormFields, MAYOSignedAgreementFormFieldsAdmin) +admin.site.register(MIMIC3SignedAgreementFormFields, MIMIC3SignedAgreementFormFieldsAdmin) diff --git a/app/projects/api.py b/app/projects/api.py index affd75e2..024342c6 100644 --- a/app/projects/api.py +++ b/app/projects/api.py @@ -3,7 +3,7 @@ import json import logging -from pyauth0jwt.auth0authenticate import user_auth_and_jwt +from hypatio.auth0authenticate import user_auth_and_jwt from django.conf import settings from django.contrib import messages @@ -37,7 +37,7 @@ from projects.models import Team from projects.models import SIGNED_FORM_REJECTED from projects.models import HostedFileSet - +from projects import models logger = logging.getLogger(__name__) @@ -45,7 +45,7 @@ class HostedFileSetAutocomplete(autocomplete.Select2QuerySetView): def get_queryset(self): # Don't forget to filter out results depending on the visitor ! - if not self.request.user.is_authenticated(): + if not self.request.user.is_authenticated: return HostedFileSet.objects.none() queryset = HostedFileSet.objects.all() @@ -214,7 +214,7 @@ def leave_team(request): # TODO user does not have permissions to remove their view permission (whether or not it exists) # Remove VIEW permissions on the DataProject - # sciauthz = SciAuthZ(settings.AUTHZ_BASE, request.COOKIES.get("DBMI_JWT", None), request.user.email) + # sciauthz = SciAuthZ(request.COOKIES.get("DBMI_JWT", None), request.user.email) # sciauthz.remove_view_permission(project_key, request.user.email) # TODO remove team leader's scireg permissions @@ -296,7 +296,7 @@ def join_team(request): extra=context) # Create record to allow leader access to profile. - sciauthz = SciAuthZ(settings.AUTHZ_BASE, request.COOKIES.get("DBMI_JWT", None), request.user.email) + sciauthz = SciAuthZ(request.COOKIES.get("DBMI_JWT", None), request.user.email) sciauthz.create_profile_permission(team_leader, project_key) return redirect('/projects/' + request.POST.get('project_key') + '/') @@ -396,17 +396,32 @@ def upload_challengetasksubmission_file(request): # If the project requires authorization to access, check for permissions before allowing submission if project.requires_authorization: - # Check that user has permissions to be submitting files for this project. - user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + # Get their permission for this project + has_permission = False + try: + participant = Participant.objects.get(user=request.user, project=project) + has_permission = participant.permission == "VIEW" + if not has_permission: + logger.debug(f"[{project_key}][{request.user.email}] No VIEW access for user") + except ObjectDoesNotExist as e: + logger.exception(f"Participant does not exist", exc_info=False, extra={ + "request": request, "project": project_key, "user": request.user, + }) + + # Check AuthZ + if not has_permission: + logger.warning( + f"[{project_key}][{request.user.email}] Local permission " + f"does not exist, checking DBMI AuthZ for " + ) - if not sciauthz.user_has_single_permission(project_key, "VIEW", request.user.email): - logger.debug("[upload_challengetasksubmission_file] - No Access for user " + request.user.email) - return HttpResponse("You do not have access to upload this file.", status=403) + # Check that user has permissions to be submitting files for this project. + user_jwt = request.COOKIES.get("DBMI_JWT", None) + sciauthz = SciAuthZ(user_jwt, request.user.email) - if filename.split(".")[-1] != "zip": - logger.error('Not a zip file.') - return HttpResponse("Only .zip files are accepted", status=400) + if not sciauthz.user_has_single_permission(project_key, "VIEW", request.user.email): + logger.warning(f"[{project_key}][{request.user.email}] No Access") + return HttpResponse("You do not have access to upload this file.", status=403) try: task = ChallengeTask.objects.get(id=task_id) @@ -486,7 +501,8 @@ def upload_challengetasksubmission_file(request): participant=participant, uuid=data['uuid'], location=data['location'], - submission_info=submission_info_json + submission_info=submission_info_json, + file_type=task.submission_file_type, ) # Send an email notification to the submitters. @@ -524,7 +540,7 @@ def delete_challengetasksubmission(request): # Check that user has permissions to be viewing files for this project. user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) submission_uuid = request.POST.get('submission_uuid') submission = ChallengeTaskSubmission.objects.get(uuid=submission_uuid) @@ -632,6 +648,69 @@ def save_signed_agreement_form(request): ) signed_agreement_form.save() + # Persist fields to JSON field on object + try: + # Set fields that we do not need to persist here + exclusions = [ + "csrfmiddlewaretoken", "project_key", "agreement_form_id", + "agreement_text" + ] + + # Save form fields + fields = {k:v for k, v in request.POST.items() if k.lower() not in exclusions} + signed_agreement_form.fields = fields + + # Save + signed_agreement_form.save() + + except Exception as e: + logger.exception( + f"HYP/Projects/API: Fields error: {e}", + exc_info=True, + extra={"form": agreement_form.short_name, "fields": request.POST,} + ) + + # TODO: The following behavior should be removed as soon as it is possible + + # Create a row for storing fields + model_name = f"{agreement_form.short_name.upper()}SignedAgreementFormFields" + if not hasattr(models, model_name): + logger.error( + f"HYP/Projects/API: Cannot persist fields for signed agreement " + f"form: {agreement_form.short_name.upper()}" + ) + + else: + try: + # Create the object + model_class = getattr(models, model_name) + signed_agreement_form_fields = model_class( + signed_agreement_form=signed_agreement_form + ) + + # Save form fields + for key, data in request.POST.items(): + + # Replace dashes with underscore + _field = key.replace("-", "_") + + # Check if field on model + if hasattr(signed_agreement_form_fields, _field): + + # Set it + setattr(signed_agreement_form_fields, _field, data) + + else: + logger.warning(f"HYP/Projects/API: '{model_name}' unhandled field: '{_field}'") + + # Save + signed_agreement_form_fields.save() + except Exception as e: + logger.exception( + f"HYP/Projects/API: Fields error: {e}", + exc_info=True, + extra={"form": agreement_form.short_name, "model": model_name}) + return HttpResponse(status=200) @user_auth_and_jwt @@ -720,3 +799,46 @@ def submit_user_permission_request(request): logger.exception(e) return HttpResponse(200) + + + +@user_auth_and_jwt +def upload_signed_agreement_form(request): + """ + An HTTP POST endpoint that takes the contents of an agreement form that a + user has submitted and saves it to the database. + """ + logger.debug(f"[upload_signed_agreement_form]: POST -> {request.POST}") + logger.debug(f"[upload_signed_agreement_form]: FILES -> {request.FILES}") + + upload = request.FILES['upload'] + agreement_form_id = request.POST['agreement_form_id'] + project_key = request.POST['project_key'] + agreement_text = request.POST['agreement_text'] + + agreement_form = AgreementForm.objects.get(id=agreement_form_id) + project = DataProject.objects.get(project_key=project_key) + + # Only create a new record if one does not already exist in a state other than Rejected. + existing_signed_form = SignedAgreementForm.objects.filter( + user=request.user, + agreement_form=agreement_form, + project=project, + ).exclude( + status=SIGNED_FORM_REJECTED + ) + + if existing_signed_form.exists(): + logger.debug('%s already has signed the agreement form "%s" for project "%s".', request.user.email, agreement_form.name, project.project_key) + return HttpResponse(status=400) + + signed_agreement_form = SignedAgreementForm( + user=request.user, + agreement_form=agreement_form, + project=project, + date_signed=datetime.now(), + upload=upload + ) + signed_agreement_form.save() + + return HttpResponse(status=200) diff --git a/app/projects/apps.py b/app/projects/apps.py index 5c4365f3..8f92c096 100644 --- a/app/projects/apps.py +++ b/app/projects/apps.py @@ -21,10 +21,14 @@ def check_fileservice(sender, **kwargs): class ProjectsConfig(AppConfig): name = 'projects' + default_auto_field = 'django.db.models.BigAutoField' def ready(self): """ Run any one-time only startup routines here """ + # Import signals + import projects.signals + # Check Fileservice groups once post_migrate.connect(check_fileservice, sender=self) diff --git a/app/projects/management/__init__.py b/app/projects/management/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/projects/management/commands/__init__.py b/app/projects/management/commands/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/app/projects/management/commands/list_no_submissions.py b/app/projects/management/commands/list_no_submissions.py new file mode 100644 index 00000000..fda57296 --- /dev/null +++ b/app/projects/management/commands/list_no_submissions.py @@ -0,0 +1,147 @@ +import json +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from dbmi_client import fileservice + +from projects.models import DataProject +from projects.models import Team, TEAM_ACTIVE, TEAM_DEACTIVATED +from projects.models import Participant +from projects.models import HostedFile +from projects.models import HostedFileDownload + +from contact.views import email_send + +import logging +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'List teams/participants that downloaded files for a project but did not provide submissions' + + def add_arguments(self, parser): + # Positional arguments + parser.add_argument('project_key', type=str) + + # Optional arguments + parser.add_argument('-r', '--recipient', type=str, help='The recipient for operation report', ) + + def email_report(self, project_key, recipient, report, *args, **options): + """ + Sends a report of the results of the operation. + """ + # Set context + context={ + "operation": self.help, + "project": project_key, + "message": report, + } + + # Send it out. + success = email_send( + subject=f'DBMI Portal - Operation Report', + recipients=[recipient], + email_template='email_operation_report', + extra=context + ) + if success: + self.stdout.write(self.style.SUCCESS(f"Report sent to: {recipient}")) + else: + self.stdout.write(self.style.ERROR(f"Report failed to send to: {recipient}")) + + def handle(self, *args, **options): + + # Ensure it exists + if not DataProject.objects.filter(project_key=options['project_key']).exists(): + raise CommandError(f'Project with key "{options["project_key"]}" does not exist') + + # Get the objects + project = DataProject.objects.get(project_key=options['project_key']) + + # Determine if a team-based challenge or not + if project.has_teams: + + # Fetch teams + teams = Team.objects.filter(data_project=project) + + # Filter out teams without submissions + teams_without_submissions = [t for t in teams if not t.get_submissions()] + + # Get all hosted files for the project + hosted_files = HostedFile.objects.filter(project=project) + + # Track teams with no submissions but did download hosted files + teams_with_access_and_no_submissions = [] + + # Find teams with a user that downloaded any of the files for the project + for team in teams_without_submissions: + + # Iterate participants + for participant in team.participant_set.all(): + + # Find downloads + if HostedFileDownload.objects.filter(hosted_file__in=hosted_files, user=participant.user): + + # Add them + teams_with_access_and_no_submissions.append(team) + break + + # Build report object + report = { + "total_teams_with_downloads_and_no_submissions": len(teams_with_access_and_no_submissions), + "teams_with_downloads_and_no_submissions": [ + {"team_leader": team.team_leader.email, "team_id": team.id} + for team in teams_with_access_and_no_submissions + ], + "total_active_teams": len(teams), + } + + # Output + self.stdout.write(self.style.SUCCESS(json.dumps(report))) + + # Check for recipient + if options['recipient']: + + # Send it + self.email_report(options['project_key'], options['recipient'], json.dumps(report)) + + else: + + # Get all participants + participants = Participant.objects.filter(project=project) + + # Filter out teams without submissions + participants_without_submissions = [p for p in participants if not p.get_submissions()] + + # Get all hosted files for the project + hosted_files = HostedFile.objects.filter(project=project) + + # Track teams with no submissions but did download hosted files + participants_with_access_and_no_submissions = [] + + # Find teams with a user that downloaded any of the files for the project + for participant in participants_without_submissions: + + # Find downloads + if HostedFileDownload.objects.filter(hosted_file__in=hosted_files, user=participant.user): + + # Add them + participants_with_access_and_no_submissions.append(participant) + + # Build report object + report = { + "total_participants_with_downloads_and_no_submissions": len(participants_with_access_and_no_submissions), + "participants_with_downloads_and_no_submissions": [ + {"email": participant.user.email, "participant_id": participant.id} + for participant in participants_with_access_and_no_submissions + ], + "total_active_participants": len(participants), + } + + # Output + self.stdout.write(self.style.SUCCESS(json.dumps(report))) + + # Check for recipient + if options['recipient']: + + # Send it + self.email_report(options['project_key'], options['recipient'], json.dumps(report)) diff --git a/app/projects/management/commands/n2c2-2022-t2-revoke.py b/app/projects/management/commands/n2c2-2022-t2-revoke.py new file mode 100644 index 00000000..0912d926 --- /dev/null +++ b/app/projects/management/commands/n2c2-2022-t2-revoke.py @@ -0,0 +1,98 @@ +import json + +from django.conf import settings +from django.core.management.base import BaseCommand + +from projects.models import DataProject +from projects.models import Team, TEAM_ACTIVE, TEAM_DEACTIVATED +from contact.views import email_send + +import logging +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Revoke access for teams that have not submitted for any subtasks' + + def add_arguments(self, parser): + + # Optional arguments + parser.add_argument('-r', '--recipient', type=str, help='The recipient for operation report', ) + parser.add_argument('-c', '--commit', action='store_true', help='Commit revocations for participants/teams', ) + + def email_report(self, project_key, recipient, report, *args, **options): + """ + Sends a report of the results of the operation. + """ + # Set context + context={ + "operation": self.help, + "project": project_key, + "message": report, + } + + # Send it out. + success = email_send( + subject=f'DBMI Portal - Operation Report', + recipients=[recipient], + email_template='email_operation_report', + extra=context + ) + if success: + self.stdout.write(self.style.SUCCESS(f"Report sent to: {recipient}")) + else: + self.stdout.write(self.style.ERROR(f"Report failed to send to: {recipient}")) + + + def handle(self, *args, **options): + + # Get the objects + project = DataProject.objects.get(project_key="n2c2-2022-t2") + + # Collect teams without submissions + teams_without_submissions = [] + + # Fetch teams + teams = Team.objects.filter(data_project=project, status=TEAM_ACTIVE) + + # Iterate teams + for team in teams: + + # Get all sub teams + sub_teams = Team.objects.filter(source=team) + + # Check if any have submissions + has_submissions = next((t for t in sub_teams if t.get_submissions()), None) + + # Add to the list if nothing + if not has_submissions: + teams_without_submissions.append(team) + + # Iterate the list + for team in teams_without_submissions: + + # Check if only listing + if options['commit']: + + # Revoke access for the team + team.status = TEAM_DEACTIVATED + team.save() + + # Build report object + report = { + "total_revoked_teams": len(teams_without_submissions), + "revoked_teams": [ + {"team_leader": team.team_leader.email, "team_id": team.id} + for team in teams_without_submissions + ], + "total_active_teams": len(teams), + } + + # Output + self.stdout.write(self.style.SUCCESS(json.dumps(report))) + + # Check for recipient + if options['recipient']: + + # Send it + self.email_report("n2c2-2022-t2", options['recipient'], json.dumps(report)) diff --git a/app/projects/management/commands/revoke_access_no_submissions.py b/app/projects/management/commands/revoke_access_no_submissions.py new file mode 100644 index 00000000..6a3ee173 --- /dev/null +++ b/app/projects/management/commands/revoke_access_no_submissions.py @@ -0,0 +1,132 @@ +import json + +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError + +from projects.models import DataProject +from projects.models import Team, TEAM_ACTIVE, TEAM_DEACTIVATED +from projects.models import Participant +from contact.views import email_send + +import logging +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Revoke access for individuals/teams without submissions' + + def add_arguments(self, parser): + # Positional arguments + parser.add_argument('project_key', type=str) + + # Optional arguments + parser.add_argument('-r', '--recipient', type=str, help='The recipient for operation report', ) + parser.add_argument('-c', '--commit', action='store_true', help='Commit revocations for participants/teams', ) + + def email_report(self, project_key, recipient, report, *args, **options): + """ + Sends a report of the results of the operation. + """ + # Set context + context={ + "operation": self.help, + "project": project_key, + "message": report, + } + + # Send it out. + success = email_send( + subject=f'DBMI Portal - Operation Report', + recipients=[recipient], + email_template='email_operation_report', + extra=context + ) + if success: + self.stdout.write(self.style.SUCCESS(f"Report sent to: {recipient}")) + else: + self.stdout.write(self.style.ERROR(f"Report failed to send to: {recipient}")) + + + def handle(self, *args, **options): + + # Ensure it exists + if not DataProject.objects.filter(project_key=options['project_key']).exists(): + raise CommandError(f'Project with key "{options["project_key"]}" does not exist') + + # Get the objects + project = DataProject.objects.get(project_key=options['project_key']) + + # Determine if a team-based challenge or not + if project.has_teams: + + # Fetch teams + teams = Team.objects.filter(data_project=project, status=TEAM_ACTIVE) + + # Filter out teams without submissions + teams_without_submissions = [t for t in teams if not t.get_submissions()] + + # Iterate the list + for team in teams_without_submissions: + + # Check if only listing + if options['commit']: + + # Revoke access for the team + team.status = TEAM_DEACTIVATED + team.save() + + # Build report object + report = { + "total_revoked_teams": len(teams_without_submissions), + "revoked_teams": [ + {"team_leader": team.team_leader.email, "team_id": team.id} + for team in teams_without_submissions + ], + "total_active_teams": len(teams), + } + + # Output + self.stdout.write(self.style.SUCCESS(json.dumps(report))) + + # Check for recipient + if options['recipient']: + + # Send it + self.email_report(options['project_key'], options['recipient'], json.dumps(report)) + + else: + + # Get all participants + participants = Participant.objects.filter(project=project, permission="VIEW") + + # Filter out teams without submissions + participants_without_submissions = [p for p in participants if not p.get_submissions()] + + # Get all participants + for participant in participants_without_submissions: + + # Check if only listing + if options['commit']: + + # Revoke access for the participant + participant.permission = None + participant.save() + + # Build report object + report = { + "total_revoked_participants": len(participants_without_submissions), + "revoked_participants": [ + {"email": participant.user.email, "participant_id": participant.id} + for participant in participants_without_submissions + ], + "total_active_participants": len(participants), + } + + # Output + self.stdout.write(self.style.SUCCESS(json.dumps(report))) + + # Check for recipient + if options['recipient']: + + # Send it + self.email_report(options['project_key'], options['recipient'], json.dumps(report)) diff --git a/app/projects/management/commands/signed_agreement_form_export.py b/app/projects/management/commands/signed_agreement_form_export.py new file mode 100644 index 00000000..40d4e046 --- /dev/null +++ b/app/projects/management/commands/signed_agreement_form_export.py @@ -0,0 +1,144 @@ +import sys +import tempfile +import requests +import shutil +import boto3 +import os +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from dbmi_client import fileservice + +from projects.models import DataProject +from projects.models import SignedAgreementForm +from projects.models import AgreementForm + +import logging +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = 'Export signed agreement forms' + + def add_arguments(self, parser): + # Positional arguments + parser.add_argument('project_key', type=str) + parser.add_argument('agreement_form', type=str) + parser.add_argument('status', type=str, default='A') + + def handle(self, *args, **options): + + # Ensure it exists + if not DataProject.objects.filter(project_key=options['project_key']).exists(): + raise CommandError(f'Project with key "{options["project_key"]}" does not exist') + if not AgreementForm.objects.filter(short_name=options['agreement_form']).exists(): + raise CommandError(f'Agreement Form with name "{options["agreement_form"]}" does not exist') + + # Get the objects + project = DataProject.objects.get(project_key=options['project_key']) + agreement_form = AgreementForm.objects.get(short_name=options['agreement_form']) + signed_agreement_forms = SignedAgreementForm.objects.filter( + project=project, + agreement_form=agreement_form, + status=options["status"], + ) + + # Ensure we've got Qualtrics surveys + if not signed_agreement_forms: + self.stdout.write( + f'{project.project_key}/{agreement_form.name}: Does not have any signed agreement forms' + ) + return + + export_uuid = export_location = export_url = None + try: + # Create a temporary directory + with tempfile.TemporaryDirectory() as directory: + + # Set the archive name + archive_root = os.path.join(directory, "signed_agreement_forms") + os.makedirs(archive_root) + archive_basename = f"{project.project_key}_{agreement_form.short_name}" + + # Create boto client + s3 = boto3.client("s3") + + # Download DUAs + for signed_agreement_form in signed_agreement_forms: + + try: + # Set the key + key = os.path.join(settings.AWS_LOCATION, signed_agreement_form.upload.name) + + # Download the file + s3.download_file(settings.AWS_STORAGE_BUCKET_NAME, key, os.path.join(archive_root, signed_agreement_form.upload.name)) + + except Exception as e: + self.stdout.write(self.style.ERROR( + f"Error: Could not download signed agreement form file: {e}" + )) + + # Archive the directory + archive_path = shutil.make_archive(archive_basename, "zip", archive_root) + logger.debug(f"Export archive: {archive_path}") + + # Perform the request to upload the file + with open(archive_path, "rb") as file: + + # Build upload request + response = None + files = {"file": file} + try: + # Create the file in Fileservice + metadata = { + "project": project.project_key, + "agreement_form": agreement_form.short_name, + "type": "export", + } + tags = ["hypatio", "export", "signed-agreement-forms", project.project_key, ] + export_uuid, upload_data = fileservice.create_archivefile_upload(os.path.basename(archive_path), metadata, tags) + + # Get the location + export_location = upload_data["locationid"] + + # Upload to S3 + response = requests.post(upload_data["post"]["url"], data=upload_data["post"]["fields"], files=files) + response.raise_for_status() + + # Mark the upload as complete + fileservice.uploaded_archivefile(export_uuid, export_location) + + # Get the download URL + export_url = fileservice.get_archivefile_download_url(export_uuid) + + except KeyError as e: + logger.error( + f'{project.project_key}: Failed export post generation: {upload_data}', + exc_info=True + ) + raise e + + except requests.exceptions.HTTPError as e: + logger.exception( + f'{project.project_key}: Failed export upload: {upload_data}', + extra={ + "response": response.content, + "status_code": response.status_code + } + ) + raise e + + except Exception as e: + logger.exception( + f"{project.project_key}: Could not export submissions: {e}", + exc_info=True, + ) + raise e + + # Return export UUID + self.stdout.write(f"Export: {export_url}") + + except Exception as e: + self.stdout.write(self.style.ERROR( + f"Error: {e}" + )) + sys.exit(1) diff --git a/app/projects/migrations/0079_duasignedagreementformfields_mayosignedagreementformfields_nlpduasignedagreementformfields_nlpwhysig.py b/app/projects/migrations/0079_duasignedagreementformfields_mayosignedagreementformfields_nlpduasignedagreementformfields_nlpwhysig.py new file mode 100644 index 00000000..212c8269 --- /dev/null +++ b/app/projects/migrations/0079_duasignedagreementformfields_mayosignedagreementformfields_nlpduasignedagreementformfields_nlpwhysig.py @@ -0,0 +1,162 @@ +# Generated by Django 2.2.15 on 2021-03-08 15:14 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0078_participant_permission'), + ] + + operations = [ + migrations.CreateModel( + name='ROCSignedAgreementFormFields', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('day', models.CharField(blank=True, max_length=2, null=True)), + ('month', models.CharField(blank=True, max_length=20, null=True)), + ('year', models.CharField(blank=True, max_length=4, null=True)), + ('e_signature', models.CharField(blank=True, max_length=255, null=True)), + ('organization', models.CharField(blank=True, max_length=255, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('signed_agreement_form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.SignedAgreementForm')), + ], + options={ + 'verbose_name': 'ROC signed agreement form fields', + 'verbose_name_plural': 'ROC signed agreement form fields', + }, + ), + migrations.CreateModel( + name='NLPWHYSignedAgreementFormFields', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('day', models.CharField(blank=True, max_length=2, null=True)), + ('month', models.CharField(blank=True, max_length=20, null=True)), + ('year', models.CharField(blank=True, max_length=4, null=True)), + ('research_use', models.TextField(blank=True, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('signed_agreement_form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.SignedAgreementForm')), + ], + options={ + 'verbose_name': 'NLP Research Purpose signed agreement form fields', + 'verbose_name_plural': 'NLP Research Purpose signed agreement form fields', + }, + ), + migrations.CreateModel( + name='NLPDUASignedAgreementFormFields', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('day', models.CharField(blank=True, max_length=2, null=True)), + ('month', models.CharField(blank=True, max_length=20, null=True)), + ('year', models.CharField(blank=True, max_length=4, null=True)), + ('form_type', models.CharField(blank=True, max_length=255, null=True)), + ('data_user', models.CharField(blank=True, max_length=255, null=True)), + ('individual_name', models.CharField(blank=True, max_length=255, null=True)), + ('individual_professional_title', models.CharField(blank=True, max_length=255, null=True)), + ('individual_address_1', models.TextField(blank=True, null=True)), + ('individual_address_2', models.TextField(blank=True, null=True)), + ('individual_address_city', models.CharField(blank=True, max_length=255, null=True)), + ('individual_address_state', models.CharField(blank=True, max_length=255, null=True)), + ('individual_address_zip', models.CharField(blank=True, max_length=255, null=True)), + ('individual_address_country', models.CharField(blank=True, max_length=255, null=True)), + ('individual_phone', models.CharField(blank=True, max_length=255, null=True)), + ('individual_fax', models.CharField(blank=True, max_length=255, null=True)), + ('individual_email', models.CharField(blank=True, max_length=255, null=True)), + ('corporation_place_of_business', models.CharField(blank=True, max_length=255, null=True)), + ('corporation_contact_name', models.CharField(blank=True, max_length=255, null=True)), + ('corporation_phone', models.CharField(blank=True, max_length=255, null=True)), + ('corporation_fax', models.CharField(blank=True, max_length=255, null=True)), + ('corporation_email', models.CharField(blank=True, max_length=255, null=True)), + ('research_team_person_1', models.CharField(blank=True, max_length=1024, null=True)), + ('research_team_person_2', models.CharField(blank=True, max_length=1024, null=True)), + ('research_team_person_3', models.CharField(blank=True, max_length=1024, null=True)), + ('research_team_person_4', models.CharField(blank=True, max_length=1024, null=True)), + ('data_user_signature', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_name', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_title', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_address_1', models.TextField(blank=True, null=True)), + ('data_user_address_2', models.TextField(blank=True, null=True)), + ('data_user_address_city', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_address_state', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_address_zip', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_address_country', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_date', models.CharField(blank=True, max_length=255, null=True)), + ('registrant_is', models.CharField(blank=True, max_length=255, null=True)), + ('commercial_registrant_is', models.CharField(blank=True, max_length=255, null=True)), + ('data_user_acknowledge', models.CharField(blank=True, max_length=3, null=True)), + ('partners_name', models.CharField(blank=True, max_length=255, null=True)), + ('partners_title', models.CharField(blank=True, max_length=255, null=True)), + ('partners_address', models.TextField(blank=True, null=True)), + ('partners_date', models.CharField(blank=True, max_length=255, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('signed_agreement_form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.SignedAgreementForm')), + ], + options={ + 'verbose_name': 'NLP DUA signed agreement form fields', + 'verbose_name_plural': 'NLP DUA signed agreement form fields', + }, + ), + migrations.CreateModel( + name='MAYOSignedAgreementFormFields', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('day', models.CharField(blank=True, max_length=2, null=True)), + ('month', models.CharField(blank=True, max_length=20, null=True)), + ('year', models.CharField(blank=True, max_length=4, null=True)), + ('institution', models.CharField(blank=True, max_length=255, null=True)), + ('pi_name', models.CharField(blank=True, max_length=1024, null=True)), + ('i_agree', models.CharField(blank=True, max_length=3, null=True)), + ('recipient_institution', models.CharField(blank=True, max_length=1024, null=True)), + ('recipient_by', models.CharField(blank=True, max_length=255, null=True)), + ('recipient_its', models.CharField(blank=True, max_length=255, null=True)), + ('recipient_attn', models.CharField(blank=True, max_length=255, null=True)), + ('recipient_phone', models.CharField(blank=True, max_length=255, null=True)), + ('recipient_fax', models.CharField(blank=True, max_length=1024, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('signed_agreement_form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.SignedAgreementForm')), + ], + options={ + 'verbose_name': 'Mayo DUA signed agreement form fields', + 'verbose_name_plural': 'Mayo DUA signed agreement form fields', + }, + ), + migrations.CreateModel( + name='DUASignedAgreementFormFields', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('day', models.CharField(blank=True, max_length=2, null=True)), + ('month', models.CharField(blank=True, max_length=20, null=True)), + ('year', models.CharField(blank=True, max_length=4, null=True)), + ('person_name', models.CharField(blank=True, max_length=1024, null=True)), + ('institution', models.CharField(blank=True, max_length=255, null=True)), + ('address', models.TextField(blank=True, null=True)), + ('city', models.CharField(blank=True, max_length=255, null=True)), + ('state', models.CharField(blank=True, max_length=255, null=True)), + ('zip', models.CharField(blank=True, max_length=255, null=True)), + ('country', models.CharField(blank=True, max_length=255, null=True)), + ('person_phone', models.CharField(blank=True, max_length=255, null=True)), + ('person_email', models.CharField(blank=True, max_length=255, null=True)), + ('place_of_business', models.CharField(blank=True, max_length=255, null=True)), + ('contact_name', models.CharField(blank=True, max_length=1024, null=True)), + ('business_phone', models.CharField(blank=True, max_length=255, null=True)), + ('business_email', models.CharField(blank=True, max_length=255, null=True)), + ('electronic_signature', models.CharField(blank=True, max_length=255, null=True)), + ('professional_title', models.CharField(blank=True, max_length=255, null=True)), + ('date', models.CharField(blank=True, max_length=255, null=True)), + ('i_agree', models.CharField(blank=True, max_length=10, null=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('signed_agreement_form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.SignedAgreementForm')), + ], + options={ + 'verbose_name': 'DUA signed agreement form fields', + 'verbose_name_plural': 'DUA signed agreement form fields', + }, + ), + ] diff --git a/app/projects/migrations/0080_auto_20211130_2115.py b/app/projects/migrations/0080_auto_20211130_2115.py new file mode 100644 index 00000000..b425eca6 --- /dev/null +++ b/app/projects/migrations/0080_auto_20211130_2115.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-11-30 21:15 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0079_duasignedagreementformfields_mayosignedagreementformfields_nlpduasignedagreementformfields_nlpwhysig'), + ] + + operations = [ + migrations.AlterField( + model_name='agreementform', + name='short_name', + field=models.CharField(max_length=16), + ), + ] diff --git a/app/projects/migrations/0081_agreementform_priority.py b/app/projects/migrations/0081_agreementform_priority.py new file mode 100644 index 00000000..7f01ec23 --- /dev/null +++ b/app/projects/migrations/0081_agreementform_priority.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.24 on 2021-11-30 22:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0080_auto_20211130_2115'), + ] + + operations = [ + migrations.AddField( + model_name='agreementform', + name='order', + field=models.IntegerField(default=50), + ), + ] diff --git a/app/projects/migrations/0082_auto_20211201_1436.py b/app/projects/migrations/0082_auto_20211201_1436.py new file mode 100644 index 00000000..456dc44b --- /dev/null +++ b/app/projects/migrations/0082_auto_20211201_1436.py @@ -0,0 +1,28 @@ +# Generated by Django 2.2.24 on 2021-12-01 14:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0081_agreementform_priority'), + ] + + operations = [ + migrations.AddField( + model_name='agreementform', + name='content', + field=models.TextField(blank=True, help_text="If Agreement Form type is set to 'MODEL', the HTML set here will be rendered for the user", null=True), + ), + migrations.AlterField( + model_name='agreementform', + name='order', + field=models.IntegerField(default=50, help_text='Indicate an order (lowest number = first listing) for how the Agreement Forms should be listed during registration workflows.'), + ), + migrations.AlterField( + model_name='agreementform', + name='type', + field=models.CharField(blank=True, choices=[('STATIC', 'STATIC'), ('EXTERNAL_LINK', 'EXTERNAL LINK'), ('MODEL', 'MODEL')], max_length=50, null=True), + ), + ] diff --git a/app/projects/migrations/0083_signedagreementform_upload.py b/app/projects/migrations/0083_signedagreementform_upload.py new file mode 100644 index 00000000..8f0cb468 --- /dev/null +++ b/app/projects/migrations/0083_signedagreementform_upload.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.25 on 2021-12-08 18:49 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0082_auto_20211201_1436'), + ] + + operations = [ + migrations.AddField( + model_name='signedagreementform', + name='upload', + field=models.FileField(blank=True, null=True, upload_to=''), + ), + migrations.AlterField( + model_name='agreementform', + name='type', + field=models.CharField(blank=True, choices=[('STATIC', 'STATIC'), ('EXTERNAL_LINK', 'EXTERNAL LINK'), ('MODEL', 'MODEL'), ('FILE', 'FILE')], max_length=50, null=True), + ), + ] diff --git a/app/projects/migrations/0084_auto_20211210_1724.py b/app/projects/migrations/0084_auto_20211210_1724.py new file mode 100644 index 00000000..c50af5d1 --- /dev/null +++ b/app/projects/migrations/0084_auto_20211210_1724.py @@ -0,0 +1,39 @@ +# Generated by Django 2.2.25 on 2021-12-10 17:24 + +from django.db import migrations, models +import django.db.models.deletion +import projects.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0083_signedagreementform_upload'), + ] + + operations = [ + migrations.AlterField( + model_name='agreementform', + name='type', + field=models.CharField(blank=True, choices=[('STATIC', 'STATIC'), ('EXTERNAL_LINK', 'EXTERNAL LINK'), ('MODEL', 'MODEL'), ('FILE', 'FILE')], max_length=50, null=True), + ), + migrations.AlterField( + model_name='signedagreementform', + name='upload', + field=models.FileField(blank=True, null=True, upload_to='', validators=[projects.models.validate_pdf_file]), + ), + migrations.CreateModel( + name='MIMIC3SignedAgreementFormFields', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('email', models.CharField(max_length=255)), + ('created', models.DateTimeField(auto_now_add=True)), + ('modified', models.DateTimeField(auto_now=True)), + ('signed_agreement_form', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='projects.SignedAgreementForm')), + ], + options={ + 'verbose_name': 'MIMIC3 Signed Agreement Form Fields', + 'verbose_name_plural': 'MIMIC3 Signed Agreement Forms Fields', + }, + ), + ] diff --git a/app/projects/migrations/0085_auto_20211210_1743.py b/app/projects/migrations/0085_auto_20211210_1743.py new file mode 100644 index 00000000..9fd3b7f0 --- /dev/null +++ b/app/projects/migrations/0085_auto_20211210_1743.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.25 on 2021-12-10 17:43 + +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0084_auto_20211210_1724'), + ] + + operations = [ + migrations.AlterModelOptions( + name='signedagreementform', + options={'verbose_name': 'Signed Agreement Form', 'verbose_name_plural': 'Signed Agreement Forms'}, + ), + migrations.AddField( + model_name='signedagreementform', + name='created', + field=models.DateTimeField(auto_now_add=True, default=django.utils.timezone.now), + preserve_default=False, + ), + migrations.AddField( + model_name='signedagreementform', + name='modified', + field=models.DateTimeField(auto_now=True), + ), + ] diff --git a/app/projects/migrations/0086_auto_20211212_1445.py b/app/projects/migrations/0086_auto_20211212_1445.py new file mode 100644 index 00000000..a86497be --- /dev/null +++ b/app/projects/migrations/0086_auto_20211212_1445.py @@ -0,0 +1,29 @@ +# Generated by Django 2.2.25 on 2021-12-12 14:45 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0085_auto_20211210_1743'), + ] + + operations = [ + migrations.AddField( + model_name='dataproject', + name='shares_teams', + field=models.BooleanField(default=False), + ), + migrations.AddField( + model_name='dataproject', + name='teams_source', + field=models.ForeignKey(blank=True, help_text='Set this to a Data Project from which teams should be imported for use in this Data Project. Only Data Projects that are configured to share will be available.', limit_choices_to={'has_teams': True, 'shares_teams': True}, null=True, on_delete=django.db.models.deletion.PROTECT, to='projects.DataProject'), + ), + migrations.AddField( + model_name='team', + name='source', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='projects.Team'), + ), + ] diff --git a/app/projects/migrations/0087_dataproject_shares_agreement_forms.py b/app/projects/migrations/0087_dataproject_shares_agreement_forms.py new file mode 100644 index 00000000..6f6ba29b --- /dev/null +++ b/app/projects/migrations/0087_dataproject_shares_agreement_forms.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.25 on 2021-12-17 23:36 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0086_auto_20211212_1445'), + ] + + operations = [ + migrations.AddField( + model_name='dataproject', + name='shares_agreement_forms', + field=models.BooleanField(default=False), + ), + ] diff --git a/app/projects/migrations/0088_signedagreementform_fields.py b/app/projects/migrations/0088_signedagreementform_fields.py new file mode 100644 index 00000000..92e60d0c --- /dev/null +++ b/app/projects/migrations/0088_signedagreementform_fields.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.26 on 2022-01-31 20:35 + +from django.db import migrations +import django.db.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0087_dataproject_shares_agreement_forms'), + ] + + operations = [ + migrations.AddField( + model_name='signedagreementform', + name='fields', + field=django.db.models.JSONField(blank=True, null=True), + ), + ] diff --git a/app/projects/migrations/0089_dataproject_teams_source_message.py b/app/projects/migrations/0089_dataproject_teams_source_message.py new file mode 100644 index 00000000..b1ea9fd7 --- /dev/null +++ b/app/projects/migrations/0089_dataproject_teams_source_message.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.27 on 2022-03-15 15:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0088_signedagreementform_fields'), + ] + + operations = [ + migrations.AddField( + model_name='dataproject', + name='teams_source_message', + field=models.TextField(blank=True, default='Teams approved there will be automatically added to this project but will need still need approval for this project.', null=True, verbose_name='Teams Source Message'), + ), + ] diff --git a/app/projects/migrations/0090_auto_20220426_2139.py b/app/projects/migrations/0090_auto_20220426_2139.py new file mode 100644 index 00000000..dca01460 --- /dev/null +++ b/app/projects/migrations/0090_auto_20220426_2139.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.27 on 2022-04-26 21:39 + +from django.db import migrations, models +import projects.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0089_dataproject_teams_source_message'), + ] + + operations = [ + migrations.AlterField( + model_name='signedagreementform', + name='upload', + field=models.FileField(blank=True, null=True, upload_to=projects.models.signed_agreement_form_path, validators=[projects.models.validate_pdf_file]), + ), + ] diff --git a/app/projects/migrations/0091_auto_20220512_2129.py b/app/projects/migrations/0091_auto_20220512_2129.py new file mode 100644 index 00000000..61d97bc6 --- /dev/null +++ b/app/projects/migrations/0091_auto_20220512_2129.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2022-05-12 21:29 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0090_auto_20220426_2139'), + ] + + operations = [ + migrations.AlterField( + model_name='signedagreementform', + name='agreement_text', + field=models.TextField(blank=True, null=True), + ), + ] diff --git a/app/projects/migrations/0092_auto_20220517_1520.py b/app/projects/migrations/0092_auto_20220517_1520.py new file mode 100644 index 00000000..75d0f008 --- /dev/null +++ b/app/projects/migrations/0092_auto_20220517_1520.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.28 on 2022-05-17 15:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0091_auto_20220512_2129'), + ] + + operations = [ + migrations.AddField( + model_name='dataproject', + name='hide_incomplete_teams', + field=models.BooleanField(default=False, help_text='Shared teams that have one or more participants with incomplete project requirements will be hidden from the teams list'), + ), + migrations.AddField( + model_name='dataproject', + name='hosted_submissions', + field=models.BooleanField(default=False, help_text='Project administrators will be able to host user submissions for download by other users'), + ), + migrations.AlterField( + model_name='dataproject', + name='shares_teams', + field=models.BooleanField(default=False, help_text='Teams formed for this project will be automatically added to projects which use this as a team source. Teams must be approved and activated before they will be added to other projects.'), + ), + migrations.AlterField( + model_name='dataproject', + name='teams_source', + field=models.ForeignKey(blank=True, help_text='Set this to a Data Project from which approved and activated teams should be imported for use in this Data Project. Only Data Projects that are configured to share will be available.', limit_choices_to={'has_teams': True, 'shares_teams': True}, null=True, on_delete=django.db.models.deletion.PROTECT, to='projects.DataProject'), + ), + ] diff --git a/app/projects/migrations/0093_auto_20220810_1356.py b/app/projects/migrations/0093_auto_20220810_1356.py new file mode 100644 index 00000000..6f89090e --- /dev/null +++ b/app/projects/migrations/0093_auto_20220810_1356.py @@ -0,0 +1,23 @@ +# Generated by Django 2.2.28 on 2022-08-10 13:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0092_auto_20220517_1520'), + ] + + operations = [ + migrations.AddField( + model_name='challengetask', + name='submission_file_type', + field=models.CharField(choices=[('zip', 'ZIP'), ('pdf', 'PDF')], default='zip', max_length=15), + ), + migrations.AddField( + model_name='challengetasksubmission', + name='file_type', + field=models.CharField(choices=[('zip', 'ZIP'), ('pdf', 'PDF')], default='zip', max_length=15), + ), + ] diff --git a/app/projects/migrations/0094_agreementform_internal.py b/app/projects/migrations/0094_agreementform_internal.py new file mode 100644 index 00000000..72872214 --- /dev/null +++ b/app/projects/migrations/0094_agreementform_internal.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.28 on 2022-08-11 17:57 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0093_auto_20220810_1356'), + ] + + operations = [ + migrations.AddField( + model_name='agreementform', + name='internal', + field=models.BooleanField(default=False, help_text='Internal agreement forms are never presented to participants and are only submitted by administrators on behalf of participants'), + ), + ] diff --git a/app/projects/migrations/0095_alter_agreementform_id_alter_challengetask_id_and_more.py b/app/projects/migrations/0095_alter_agreementform_id_alter_challengetask_id_and_more.py new file mode 100644 index 00000000..2034cdb3 --- /dev/null +++ b/app/projects/migrations/0095_alter_agreementform_id_alter_challengetask_id_and_more.py @@ -0,0 +1,103 @@ +# Generated by Django 4.1.1 on 2022-09-29 15:05 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('projects', '0094_agreementform_internal'), + ] + + operations = [ + migrations.AlterField( + model_name='agreementform', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='challengetask', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='challengetasksubmissiondownload', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='dataproject', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='duasignedagreementformfields', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='hostedfile', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='hostedfiledownload', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='hostedfileset', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='institution', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='mayosignedagreementformfields', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='mimic3signedagreementformfields', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='nlpduasignedagreementformfields', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='nlpwhysignedagreementformfields', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='participant', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='rocsignedagreementformfields', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='signedagreementform', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='team', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + migrations.AlterField( + model_name='teamcomment', + name='id', + field=models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID'), + ), + ] diff --git a/app/projects/models.py b/app/projects/models.py index 0ddff820..e3b8cae7 100644 --- a/app/projects/models.py +++ b/app/projects/models.py @@ -1,8 +1,12 @@ import uuid +from datetime import datetime +from django.conf import settings from django.db import models from django.contrib.auth.models import User from django.core.exceptions import ValidationError +from django.db.models import JSONField +from django.core.files.uploadedfile import UploadedFile TEAM_PENDING = 'Pending' TEAM_READY = 'Ready' @@ -28,12 +32,28 @@ AGREEMENT_FORM_TYPE_STATIC = 'STATIC' AGREEMENT_FORM_TYPE_EXTERNAL_LINK = 'EXTERNAL_LINK' +AGREEMENT_FORM_TYPE_MODEL = 'MODEL' +AGREEMENT_FORM_TYPE_FILE = 'FILE' AGREEMENT_FORM_TYPE = ( (AGREEMENT_FORM_TYPE_STATIC, 'STATIC'), - (AGREEMENT_FORM_TYPE_EXTERNAL_LINK, 'EXTERNAL LINK') + (AGREEMENT_FORM_TYPE_EXTERNAL_LINK, 'EXTERNAL LINK'), + (AGREEMENT_FORM_TYPE_MODEL, 'MODEL'), + (AGREEMENT_FORM_TYPE_FILE, 'FILE'), ) +FILE_TYPE_ZIP = "zip" +FILE_TYPE_PDF = "pdf" + +FILES_TYPES = ( + (FILE_TYPE_ZIP, "ZIP"), + (FILE_TYPE_PDF, "PDF"), +) + +FILES_CONTENT_TYPES = { + FILE_TYPE_ZIP: "application/zip", + FILE_TYPE_PDF: "application/pdf", +} def get_agreement_form_upload_path(instance, filename): @@ -80,21 +100,28 @@ class AgreementForm(models.Model): """ name = models.CharField(max_length=100, blank=False, null=False, verbose_name="name") - short_name = models.CharField(max_length=6, blank=False, null=False) + short_name = models.CharField(max_length=16, blank=False, null=False) description = models.TextField(blank=True) created = models.DateTimeField(auto_now_add=True) form_file_path = models.CharField(max_length=300, blank=True, null=True) external_link = models.CharField(max_length=300, blank=True, null=True) type = models.CharField(max_length=50, choices=AGREEMENT_FORM_TYPE, blank=True, null=True) + order = models.IntegerField(default=50, help_text="Indicate an order (lowest number = first listing) for how the Agreement Forms should be listed during registration workflows.") + content = models.TextField(blank=True, null=True, help_text="If Agreement Form type is set to 'MODEL', the HTML set here will be rendered for the user") + internal = models.BooleanField(default=False, help_text="Internal agreement forms are never presented to participants and are only submitted by administrators on behalf of participants") def __str__(self): return '%s' % (self.name) def clean(self): - if self.type == AGREEMENT_FORM_TYPE_EXTERNAL_LINK and self.form_file_path is not None: - raise ValidationError("An external link form should not have the form file path field populated.") - if self.type != AGREEMENT_FORM_TYPE_EXTERNAL_LINK and self.external_link is not None: - raise ValidationError("If the form type is not an external link, the external link field should not be populated.") + if (self.type == AGREEMENT_FORM_TYPE_STATIC or not self.type) and not self.form_file_path: + raise ValidationError("If the form type is static, the file path field should be populated.") + if self.type == AGREEMENT_FORM_TYPE_EXTERNAL_LINK and not self.external_link: + raise ValidationError("If the form type is external link, the external link field should be populated.") + if self.type == AGREEMENT_FORM_TYPE_MODEL and not self.content: + raise ValidationError("If the form type is model, the content field should be populated with the agreement form's HTML.") + if self.type == AGREEMENT_FORM_TYPE_FILE and not self.content and not self.form_file_path: + raise ValidationError("If the form type is file, the content field should be populated with the agreement form's HTML.") class DataProject(models.Model): @@ -119,6 +146,7 @@ class DataProject(models.Model): # Which forms users need to sign before accessing any data. agreement_forms = models.ManyToManyField(AgreementForm, blank=True, related_name='data_project_agreement_forms') + shares_agreement_forms = models.BooleanField(default=False, blank=False, null=False) # Various tags for what a project may be, influencing where the project is listed and the functionality on its page. is_dataset = models.BooleanField(default=False, blank=False, null=False) @@ -128,6 +156,25 @@ class DataProject(models.Model): # Set whether users need to form teams before accessing data. has_teams = models.BooleanField(default=False, blank=False, null=False) + # Set whether submissions can be hosted for download or not. + hosted_submissions = models.BooleanField(default=False, blank=False, null=False, help_text="Project administrators will be able to host user submissions for download by other users") + + # Set whether to hide incomplete shared teams or not. Incomplete means one or more participants on a shared team + # have incomplete and/or denied DUAs or required steps. + hide_incomplete_teams = models.BooleanField(default=False, blank=False, null=False, help_text="Shared teams that have one or more participants with incomplete project requirements will be hidden from the teams list") + + # Set whether the teams created for this project can be used by other challenges + shares_teams = models.BooleanField(default=False, blank=False, null=False, help_text="Teams formed for this project will be automatically added to projects which use this as a team source. Teams must be approved and activated before they will be added to other projects.") + teams_source = models.ForeignKey( + to="DataProject", + on_delete=models.PROTECT, + blank=True, + null=True, + limit_choices_to={"shares_teams": True, "has_teams": True}, + help_text="Set this to a Data Project from which approved and activated teams should be imported for use in this Data Project. Only Data Projects that are configured to share will be available." + ) + teams_source_message = models.TextField(default="Teams approved there will be automatically added to this project but will need still need approval for this project.", blank=True, null=True, verbose_name="Teams Source Message") + show_jwt = models.BooleanField(default=False, blank=False, null=False) order = models.IntegerField(blank=True, null=True, help_text="Indicate an order (lowest number = highest order) for how the DataProjects should be listed.") @@ -148,6 +195,25 @@ def clean(self): if self.is_challenge and self.is_dataset: raise ValidationError('At this time, a challenge should not also be marked as software or dataset.') + if not self.has_teams and self.shares_teams: + raise ValidationError('A Project cannot share teams if it does not itself use teams') + + if self.teams_source and self.shares_teams: + raise ValidationError('A Project cannot share teams if it is using shared teams from another project') + + +def validate_pdf_file(value): + """ + Ensures only a file with a content type of PDF can be persisted + """ + if type(value.file) is UploadedFile and value.file.content_type != 'application/pdf': + raise ValidationError('Only PDF files can be uploaded') + + +def signed_agreement_form_path(instance, filename): + # file will be uploaded to AWS_LOCATION/__ + return f'{instance.user.email}_{datetime.now().isoformat()}_{filename}' + class SignedAgreementForm(models.Model): """ @@ -158,10 +224,191 @@ class SignedAgreementForm(models.Model): agreement_form = models.ForeignKey(AgreementForm, on_delete=models.PROTECT) project = models.ForeignKey(DataProject, on_delete=models.PROTECT) date_signed = models.DateTimeField(auto_now_add=True) - agreement_text = models.TextField(blank=False) + agreement_text = models.TextField(null=True, blank=True) status = models.CharField(max_length=1, null=False, blank=False, default='P', choices=SIGNED_FORM_STATUSES) + upload = models.FileField(null=True, blank=True, validators=[validate_pdf_file], upload_to=signed_agreement_form_path) + fields = JSONField(null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'Signed Agreement Form' + verbose_name_plural = 'Signed Agreement Forms' + + +class MIMIC3SignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + email = models.CharField(max_length=255) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + + class Meta: + verbose_name = 'MIMIC3 Signed Agreement Form Fields' + verbose_name_plural = 'MIMIC3 Signed Agreement Forms Fields' +class ROCSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # N2C2-t1 ROC + e_signature = models.CharField(max_length=255, null=True, blank=True) + organization = models.CharField(max_length=255, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'ROC signed agreement form fields' + verbose_name_plural = 'ROC signed agreement form fields' + +class DUASignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # N2C2-T1 DUA + person_name = models.CharField(max_length=1024, null=True, blank=True) + institution = models.CharField(max_length=255, null=True, blank=True) + address = models.TextField(null=True, blank=True) + city = models.CharField(max_length=255, null=True, blank=True) + state = models.CharField(max_length=255, null=True, blank=True) + zip = models.CharField(max_length=255, null=True, blank=True) + country = models.CharField(max_length=255, null=True, blank=True) + person_phone = models.CharField(max_length=255, null=True, blank=True) + person_email = models.CharField(max_length=255, null=True, blank=True) + place_of_business = models.CharField(max_length=255, null=True, blank=True) + contact_name = models.CharField(max_length=1024, null=True, blank=True) + business_phone = models.CharField(max_length=255, null=True, blank=True) + business_email = models.CharField(max_length=255, null=True, blank=True) + electronic_signature = models.CharField(max_length=255, null=True, blank=True) + professional_title = models.CharField(max_length=255, null=True, blank=True) + date = models.CharField(max_length=255, null=True, blank=True) + i_agree = models.CharField(max_length=10, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'DUA signed agreement form fields' + verbose_name_plural = 'DUA signed agreement form fields' + +class MAYOSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # Mayo DUA + institution = models.CharField(max_length=255, null=True, blank=True) + pi_name = models.CharField(max_length=1024, null=True, blank=True) + i_agree = models.CharField(max_length=3, null=True, blank=True) + recipient_institution = models.CharField(max_length=1024, null=True, blank=True) + recipient_by = models.CharField(max_length=255, null=True, blank=True) + recipient_its = models.CharField(max_length=255, null=True, blank=True) + recipient_attn = models.CharField(max_length=255, null=True, blank=True) + recipient_phone = models.CharField(max_length=255, null=True, blank=True) + recipient_fax = models.CharField(max_length=1024, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'Mayo DUA signed agreement form fields' + verbose_name_plural = 'Mayo DUA signed agreement form fields' + +class NLPWHYSignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # NLP Research Purpose + research_use = models.TextField(null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'NLP Research Purpose signed agreement form fields' + verbose_name_plural = 'NLP Research Purpose signed agreement form fields' + +class NLPDUASignedAgreementFormFields(models.Model): + + signed_agreement_form = models.ForeignKey(SignedAgreementForm, on_delete=models.CASCADE) + + # All DUAs + day = models.CharField(max_length=2, null=True, blank=True) + month = models.CharField(max_length=20, null=True, blank=True) + year = models.CharField(max_length=4, null=True, blank=True) + + # NLP DUA + form_type = models.CharField(max_length=255, null=True, blank=True) + data_user = models.CharField(max_length=255, null=True, blank=True) + individual_name = models.CharField(max_length=255, null=True, blank=True) + individual_professional_title = models.CharField(max_length=255, null=True, blank=True) + individual_address_1 = models.TextField(null=True, blank=True) + individual_address_2 = models.TextField(null=True, blank=True) + individual_address_city = models.CharField(max_length=255, null=True, blank=True) + individual_address_state = models.CharField(max_length=255, null=True, blank=True) + individual_address_zip = models.CharField(max_length=255, null=True, blank=True) + individual_address_country = models.CharField(max_length=255, null=True, blank=True) + individual_phone = models.CharField(max_length=255, null=True, blank=True) + individual_fax = models.CharField(max_length=255, null=True, blank=True) + individual_email = models.CharField(max_length=255, null=True, blank=True) + corporation_place_of_business = models.CharField(max_length=255, null=True, blank=True) + corporation_contact_name = models.CharField(max_length=255, null=True, blank=True) + corporation_phone = models.CharField(max_length=255, null=True, blank=True) + corporation_fax = models.CharField(max_length=255, null=True, blank=True) + corporation_email = models.CharField(max_length=255, null=True, blank=True) + research_team_person_1 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_2 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_3 = models.CharField(max_length=1024, null=True, blank=True) + research_team_person_4 = models.CharField(max_length=1024, null=True, blank=True) + data_user_signature = models.CharField(max_length=255, null=True, blank=True) + data_user_name = models.CharField(max_length=255, null=True, blank=True) + data_user_title = models.CharField(max_length=255, null=True, blank=True) + data_user_address_1 = models.TextField(null=True, blank=True) + data_user_address_2 = models.TextField(null=True, blank=True) + data_user_address_city = models.CharField(max_length=255, null=True, blank=True) + data_user_address_state = models.CharField(max_length=255, null=True, blank=True) + data_user_address_zip = models.CharField(max_length=255, null=True, blank=True) + data_user_address_country = models.CharField(max_length=255, null=True, blank=True) + data_user_date = models.CharField(max_length=255, null=True, blank=True) + registrant_is = models.CharField(max_length=255, null=True, blank=True) + commercial_registrant_is = models.CharField(max_length=255, null=True, blank=True) + data_user_acknowledge = models.CharField(max_length=3, null=True, blank=True) + partners_name = models.CharField(max_length=255, null=True, blank=True) + partners_title = models.CharField(max_length=255, null=True, blank=True) + partners_address = models.TextField(null=True, blank=True) + partners_date = models.CharField(max_length=255, null=True, blank=True) + + # Meta + created = models.DateTimeField(auto_now_add=True) + modified = models.DateTimeField(auto_now=True) + class Meta: + verbose_name = 'NLP DUA signed agreement form fields' + verbose_name_plural = 'NLP DUA signed agreement form fields' + class Team(models.Model): """ This model describes a team of participants that are competing in a data challenge. @@ -170,6 +417,7 @@ class Team(models.Model): team_leader = models.ForeignKey(User, on_delete=models.PROTECT) data_project = models.ForeignKey(DataProject, on_delete=models.CASCADE) status = models.CharField(max_length=30, choices=TEAM_STATUS, default='Pending') + source = models.ForeignKey("Team", null=True, blank=True, on_delete=models.CASCADE) class Meta: unique_together = ('team_leader', 'data_project',) @@ -332,6 +580,9 @@ class ChallengeTask(models.Model): # Should supervisors be notified of submissions of this task notify_supervisors_of_submissions = models.BooleanField(default=False, blank=False, null=False, help_text="Sends a notification to any emails listed in the project's supervisors field.") + # The content type to restrict file uploads to + submission_file_type = models.CharField(max_length=15, default=FILE_TYPE_ZIP, choices=FILES_TYPES) + def __str__(self): return '%s: %s' % (self.data_project.project_key, self.title) @@ -339,6 +590,10 @@ def clean(self): if self.opened_time is not None and self.closed_time is not None and (self.opened_time > self.closed_time or self.closed_time < self.opened_time): raise ValidationError("Closed time must be a datetime after opened time") + @property + def submission_file_content_type(self): + return FILES_CONTENT_TYPES[self.submission_file_type] + class ChallengeTaskSubmission(models.Model): """ @@ -355,6 +610,7 @@ class ChallengeTaskSubmission(models.Model): location = models.CharField(max_length=12, default=None, blank=True, null=True) submission_info = models.TextField(default=None, blank=True, null=True) deleted = models.BooleanField(default=False) + file_type = models.CharField(max_length=15, default=FILE_TYPE_ZIP, choices=FILES_TYPES) def __str__(self): return '%s' % (self.uuid) diff --git a/app/projects/panels.py b/app/projects/panels.py index cdc82522..c3e4b000 100644 --- a/app/projects/panels.py +++ b/app/projects/panels.py @@ -50,3 +50,13 @@ class DataProjectActionablePanel(DataProjectPanel): def __init__(self, title, bootstrap_color, template, additional_context=None): super().__init__(title, bootstrap_color, template, additional_context) + +class DataProjectSharedTeamsPanel(DataProjectPanel): + """ + This class holds information needed to display panels on the DataProject + page that do have actions. + """ + + def __init__(self, title, bootstrap_color, template, status, additional_context=None): + super().__init__(title, bootstrap_color, template, additional_context) + self.status = status diff --git a/app/projects/serializers.py b/app/projects/serializers.py new file mode 100644 index 00000000..48169fa0 --- /dev/null +++ b/app/projects/serializers.py @@ -0,0 +1,36 @@ +from rest_framework import serializers + +from projects.models import HostedFileDownload, HostedFile, ChallengeTaskSubmission + + +class ChallengeTaskSubmissionSerializer(serializers.ModelSerializer): + upload_date = serializers.SerializerMethodField(source='*') + + class Meta: + model = ChallengeTaskSubmission + fields = '__all__' + + def get_upload_date(self, obj): + return obj.upload_date.isoformat() + + +class HostedFileSerializer(serializers.ModelSerializer): + + class Meta: + model = HostedFile + fields = '__all__' + + +class HostedFileDownloadSerializer(serializers.ModelSerializer): + download_date = serializers.SerializerMethodField(source='*') + user = serializers.SerializerMethodField(source='*') + + class Meta: + model = HostedFileDownload + fields = ['user', 'download_date'] + + def get_user(self, obj): + return obj.user.email + + def get_download_date(self, obj): + return obj.download_date.isoformat() diff --git a/app/projects/signals.py b/app/projects/signals.py new file mode 100644 index 00000000..0e8f0972 --- /dev/null +++ b/app/projects/signals.py @@ -0,0 +1,118 @@ +from django.db.models.signals import post_save +from django.dispatch import receiver +from django.db import transaction + +from projects.models import DataProject +from projects.models import Team +from projects.models import Participant +from projects.models import TEAM_ACTIVE, TEAM_DEACTIVATED, TEAM_READY + +import logging +logger = logging.getLogger(__name__) + + +@receiver(post_save, sender=DataProject) +def dataproject_post_save_handler(sender, **kwargs): + """ + This hook listens for DataProjects that have been saved and handles the + syncing of Teams if team sharing has been specified. + """ + instance = kwargs.get("instance") + + # Check if this project requires importing of teams + if instance.teams_source: + logger.debug(f"Project uses teams from: {instance.teams_source}") + + # Sync + sync_teams(instance.teams_source) + +@receiver(post_save, sender=Team) +def team_post_save_handler(sender, **kwargs): + """ + This hook listens for Team modifications and appropriately propogates + properties if this team is a source for other DataProject teams. + """ + instance = kwargs.get("instance") + + # Check if this is a shared team + if instance.data_project.shares_teams: + logger.debug(f"Team is a source for other projects: {instance}") + + # Sync + sync_teams(instance.data_project) + + +def sync_teams(project): + """ + This method accepts a project and performs a sync of the project's teams + and participants. If this project shares teams, all ACTIVE teams and their + corresponding participants will be duplicated for all projects that use + this project as their team source. Existing teams will also be checked for + access revocations if they are unapproved for the source project as that + change is automatically propogated out to sharing projects' teams. + + :param project: The source project + :type project: DataProject + """ + # Load active teams for the source project that are not yet copied + logger.debug("Team Sync: Processing new teams") + for team in project.team_set.filter(status=TEAM_ACTIVE): + + # Iterate projects that use this project's teams + for sharing_project in DataProject.objects.filter(teams_source=project): + + # Check if already created + shared_team = Team.objects.filter(data_project=sharing_project, source=team).first() + if not shared_team: + logger.debug(f"Team Sync: Team/{team} not shared with DataProject/{sharing_project}, creating") + + # Create the team + shared_team = Team( + source=team, + data_project=sharing_project, + team_leader=team.team_leader, + status=TEAM_READY, + ) + shared_team.save() + + else: + logger.debug(f"Team Sync: Team/{team} already shared with DataProject/{sharing_project}") + + # Iterate participants in the source team + for participant in team.participant_set.all(): + + # See if they already exist + shared_participant = Participant.objects.filter( + user=participant.user, + project=sharing_project, + team=shared_team, + ) + + if not shared_participant: + logger.debug(f"Team Sync: Participant/{participant} not shared with DataProject/{sharing_project}, creating") + + # Create a new one + shared_participant = Participant( + user=participant.user, + project=sharing_project, + team=shared_team, + team_wait_on_leader_email=participant.team_wait_on_leader_email, + team_wait_on_leader=participant.team_wait_on_leader, + team_pending=participant.team_pending, + team_approved=participant.team_approved, + ) + shared_participant.save() + + else: + logger.debug(f"Team Sync: Participant/{participant} already shared with DataProject/{sharing_project}") + + # Load deactivated teams for the source project that have been copied + logger.debug("Team Sync: Processing deactivated teams") + for team in project.team_set.filter(status=TEAM_DEACTIVATED): + + # Iterate copied teams + for shared_team in Team.objects.filter(source=team): + + # Set as deactivated + shared_team.status = TEAM_DEACTIVATED + shared_team.save() diff --git a/app/projects/templatetags/projects_extras.py b/app/projects/templatetags/projects_extras.py index dc7952e2..c7403e2d 100644 --- a/app/projects/templatetags/projects_extras.py +++ b/app/projects/templatetags/projects_extras.py @@ -1,12 +1,12 @@ -import os import datetime -import furl +from furl import furl import logging from django import template from django.conf import settings -from django.utils.safestring import mark_safe from django.utils.timezone import utc +from django.template.loader import render_to_string +from dbmi_client.authn import login_redirect_url from hypatio.dbmiauthz_services import DBMIAuthz @@ -17,18 +17,13 @@ @register.filter def get_html_form_file_contents(form_file_path): - - form_path = os.path.join(settings.STATIC_ROOT, form_file_path) - return open(form_path, 'r').read() + return render_to_string(form_file_path) @register.filter def get_login_url(current_uri): # Build the login URL - login_url = furl.furl(settings.ACCOUNT_SERVER_URL) - - # Add the next URL - login_url.args.add('next', current_uri) + login_url = furl(login_redirect_url(None, next_url=current_uri)) # Add project, if any project = getattr(settings, 'PROJECT', None) diff --git a/app/projects/urls.py b/app/projects/urls.py index eeadedd1..d39de224 100644 --- a/app/projects/urls.py +++ b/app/projects/urls.py @@ -1,4 +1,4 @@ -from django.conf.urls import url +from django.urls import re_path from projects.apps import ProjectsConfig from projects.views import list_data_projects @@ -17,25 +17,27 @@ from projects.api import save_signed_agreement_form from projects.api import save_signed_external_agreement_form from projects.api import submit_user_permission_request +from projects.api import upload_signed_agreement_form from projects.api import HostedFileSetAutocomplete app_name = ProjectsConfig.name urlpatterns = [ - url(r'^$', list_data_projects, name='index'), - url(r'^autocomplete/hostedfileset/?$', HostedFileSetAutocomplete.as_view(create_field='title'), name='hostedfileset-autocomplete'), - url(r'^submit_user_permission_request/?$', submit_user_permission_request, name='submit_user_permission_request'), - url(r'^save_signed_agreement_form/?$', save_signed_agreement_form, name='save_signed_agreement_form'), - url(r'^save_signed_external_agreement_form/?$', save_signed_external_agreement_form, name='save_signed_external_agreement_form'), - url(r'^join_team/?$', join_team, name='join_team'), - url(r'^leave_team/?$', leave_team, name='leave_team'), - url(r'^approve_team_join/?$', approve_team_join, name='approve_team_join'), - url(r'^reject_team_join/?$', reject_team_join, name='reject_team_join'), - url(r'^create_team/?$', create_team, name='create_team'), - url(r'^finalize_team/?$', finalize_team, name='finalize_team'), - url(r'^signed_agreement_form/?$', signed_agreement_form, name='signed_agreement_form'), - url(r'^download_dataset/?$', download_dataset, name='download_dataset'), - url(r'^upload_challengetasksubmission_file/?$', upload_challengetasksubmission_file, name="upload_challengetasksubmission_file"), - url(r'^delete_challengetasksubmission/?$', delete_challengetasksubmission, name='delete_challengetasksubmission'), - url(r'^(?P[^/]+)/?$', DataProjectView.as_view(), name="view-project"), + re_path(r'^$', list_data_projects, name='index'), + re_path(r'^autocomplete/hostedfileset/?$', HostedFileSetAutocomplete.as_view(create_field='title'), name='hostedfileset-autocomplete'), + re_path(r'^submit_user_permission_request/?$', submit_user_permission_request, name='submit_user_permission_request'), + re_path(r'^save_signed_agreement_form', save_signed_agreement_form, name='save_signed_agreement_form'), + re_path(r'^save_signed_external_agreement_form', save_signed_external_agreement_form, name='save_signed_external_agreement_form'), + re_path(r'^upload_signed_agreement_form', upload_signed_agreement_form, name='upload_signed_agreement_form'), + re_path(r'^join_team/?$', join_team, name='join_team'), + re_path(r'^leave_team/?$', leave_team, name='leave_team'), + re_path(r'^approve_team_join/?$', approve_team_join, name='approve_team_join'), + re_path(r'^reject_team_join/?$', reject_team_join, name='reject_team_join'), + re_path(r'^create_team/?$', create_team, name='create_team'), + re_path(r'^finalize_team/?$', finalize_team, name='finalize_team'), + re_path(r'^signed_agreement_form/?$', signed_agreement_form, name='signed_agreement_form'), + re_path(r'^download_dataset/?$', download_dataset, name='download_dataset'), + re_path(r'^upload_challengetasksubmission_file/?$', upload_challengetasksubmission_file, name="upload_challengetasksubmission_file"), + re_path(r'^delete_challengetasksubmission/?$', delete_challengetasksubmission, name='delete_challengetasksubmission'), + re_path(r'^(?P[^/]+)/?$', DataProjectView.as_view(), name="view-project"), ] diff --git a/app/projects/views.py b/app/projects/views.py index 9b721763..ed536bef 100644 --- a/app/projects/views.py +++ b/app/projects/views.py @@ -10,24 +10,23 @@ from django.shortcuts import render from django.utils.decorators import method_decorator from django.views.generic import TemplateView + from hypatio.sciauthz_services import SciAuthZ from hypatio.dbmiauthz_services import DBMIAuthz from hypatio.scireg_services import get_current_user_profile from hypatio.scireg_services import get_user_email_confirmation_status - from profile.forms import RegistrationForm - -from pyauth0jwt.auth0authenticate import public_user_auth_and_jwt -from pyauth0jwt.auth0authenticate import user_auth_and_jwt - -from projects.models import AGREEMENT_FORM_TYPE_EXTERNAL_LINK +from hypatio.auth0authenticate import public_user_auth_and_jwt +from hypatio.auth0authenticate import user_auth_and_jwt +from projects.models import AGREEMENT_FORM_TYPE_EXTERNAL_LINK, TEAM_ACTIVE, TEAM_READY from projects.models import AGREEMENT_FORM_TYPE_STATIC +from projects.models import AGREEMENT_FORM_TYPE_MODEL +from projects.models import AGREEMENT_FORM_TYPE_FILE from projects.models import ChallengeTaskSubmission from projects.models import DataProject from projects.models import HostedFile from projects.models import Participant from projects.models import SignedAgreementForm - from projects.panels import SIGNUP_STEP_COMPLETED_STATUS from projects.panels import SIGNUP_STEP_CURRENT_STATUS from projects.panels import SIGNUP_STEP_FUTURE_STATUS @@ -35,6 +34,7 @@ from projects.panels import DataProjectInformationalPanel from projects.panels import DataProjectSignupPanel from projects.panels import DataProjectActionablePanel +from projects.panels import DataProjectSharedTeamsPanel # Get an instance of a logger logger = logging.getLogger(__name__) @@ -47,7 +47,7 @@ def signed_agreement_form(request): signed_agreement_form_id = request.GET['signed_form_id'] user_jwt = request.COOKIES.get("DBMI_JWT", None) - sciauthz = SciAuthZ(settings.AUTHZ_BASE, user_jwt, request.user.email) + sciauthz = SciAuthZ(user_jwt, request.user.email) is_manager = sciauthz.user_has_manage_permission(project_key) project = get_object_or_404(DataProject, project_key=project_key) @@ -58,6 +58,10 @@ def signed_agreement_form(request): except ObjectDoesNotExist: participant = None + # Get fields, if applicable. It sucks that these are hard-coded but until we + # find a better solution and have more time, this is it. + signed_agreement_form_fields = {} + if is_manager or signed_form.user == request.user: template_name = "projects/participate/view-signed-agreement-form.html" filled_out_signed_form = None @@ -67,6 +71,7 @@ def signed_agreement_form(request): "is_manager": is_manager, "signed_form": signed_form, "filled_out_signed_form": filled_out_signed_form, + "signed_agreement_form_fields": signed_agreement_form_fields, "participant": participant}) else: return HttpResponse(403) @@ -140,7 +145,7 @@ def dispatch(self, request, *args, **kwargs): self.user_jwt = request.COOKIES.get("DBMI_JWT", None) # Add the participant to the class instance if available. - if request.user.is_authenticated(): + if request.user.is_authenticated: try: self.participant = Participant.objects.get( user=self.request.user, @@ -180,12 +185,12 @@ def get_context_data(self, **kwargs): return context # Otherwise, users who are not logged in should be prompted to first before proceeding further. - if not self.request.user.is_authenticated() or self.user_jwt is None: + if not self.request.user.is_authenticated or self.user_jwt is None: self.get_unregistered_context(context) return context # Check the users current permissions on this project. - if self.request.user.is_authenticated(): + if self.request.user.is_authenticated: context['has_manage_permissions'] = DBMIAuthz.user_has_manage_permission( request=self.request, project_key=self.project.project_key ) @@ -273,21 +278,29 @@ def get_signup_context(self, context): # SciReg complete profile step. self.setup_panel_complete_profile(context) - # Agreement forms step (if needed). - self.setup_panel_sign_agreement_forms(context) + # Check if this project uses shared teams + if self.project.teams_source and not Participant.objects.filter(user=self.request.user, team__data_project=self.project, team__status=TEAM_READY).exists(): + + # Show panel + self.setup_panel_shared_teams(context) + + else: + + # Agreement forms step (if needed). + self.setup_panel_sign_agreement_forms(context) - # Show JWT step (if needed). - self.setup_panel_show_jwt(context) + # Show JWT step (if needed). + self.setup_panel_show_jwt(context) - # Access request step (if needed). - self.setup_panel_request_access(context) + # Access request step (if needed). + self.setup_panel_request_access(context) - # Team setup step (if needed). - self.setup_panel_team(context) + # Team setup step (if needed). + self.setup_panel_team(context) - # TODO commented out until this is ready. - # Static page that lets user know to wait. - # self.step_pending_review(context) + # TODO commented out until this is ready. + # Static page that lets user know to wait. + # self.step_pending_review(context) return context @@ -303,6 +316,9 @@ def get_participate_context(self, context): # Add a panel for displaying your signed agreement forms (if needed). self.panel_signed_agreement_forms(context) + # Add a panel for projects + self.panel_available_projects(context) + # Add a panel for available downloads. self.panel_available_downloads(context) @@ -317,6 +333,9 @@ def get_manager_context(self, context): otherwise participating in the project. """ + # Add a panel for projects + self.panel_available_projects(context) + # Add a panel for available downloads. self.panel_available_downloads(context) @@ -338,12 +357,15 @@ def get_step_status(self, step_name, step_complete, is_permanent=False): if is_permanent: return SIGNUP_STEP_PERMANENT_STATUS + logger.debug(f"{self.project.project_key}/{step_name}: Completed step") return SIGNUP_STEP_COMPLETED_STATUS if self.current_step is None: self.current_step = step_name + logger.debug(f"{self.project.project_key}/{step_name}: Current step") return SIGNUP_STEP_CURRENT_STATUS + logger.debug(f"{self.project.project_key}/{step_name}: Future step, {self.current_step}: Current step") return SIGNUP_STEP_FUTURE_STATUS def setup_panel_verify_email(self, context): @@ -383,6 +405,9 @@ def setup_panel_complete_profile(self, context): if not profile_complete: registration_form = RegistrationForm(initial=profile_data) + # Log errors + logger.debug(f"{self.project.project_key}/{self.request.user.email}: Registration form errors: {registration_form.errors.as_json()}") + except (KeyError, IndexError): profile_data = None profile_complete = False @@ -418,6 +443,23 @@ def setup_panel_complete_profile(self, context): context['setup_panels'].append(panel) + def setup_panel_shared_teams(self, context): + """ + Builds the context needed for users to be informed of a team sharing setup. This requires + users to register with another project before requesting access to this one. + """ + step_status = self.get_step_status('setup_shared_team', False) + + panel = DataProjectSharedTeamsPanel( + title='Data Project Teams', + bootstrap_color='default', + template='projects/signup/shared-teams.html', + status=step_status, + additional_context={'project': self.project} + ) + + context['setup_panels'].append(panel) + def setup_panel_sign_agreement_forms(self, context): """ Builds the context needed for users to complete any required agreement forms. @@ -429,10 +471,11 @@ def setup_panel_sign_agreement_forms(self, context): if self.project.agreement_forms.count() == 0: return - agreement_forms = self.project.agreement_forms.order_by('-name') + agreement_forms = self.project.agreement_forms.order_by('order', '-name') # Each form will be a separate step. for form in agreement_forms: + logger.debug(f"{self.project.project_key}/{form.short_name}: Checking panel signed agreement form") # Only include Pending or Approved forms when searching. signed_forms = SignedAgreementForm.objects.filter( @@ -441,19 +484,35 @@ def setup_panel_sign_agreement_forms(self, context): agreement_form=form, status__in=["P", "A"] ) + logger.debug(f"{self.project.project_key}/{form.short_name}: Found {len(signed_forms)} signed P/A forms") + + # If this project accepts agreement forms from other projects, check those too + if not signed_forms and self.project.shares_agreement_forms: + + # Fetch without a specific project + signed_forms = SignedAgreementForm.objects.filter( + user=self.request.user, + agreement_form=form, + status__in=["P", "A"] + ) + logger.debug(f"{self.project.project_key}/{form.short_name}: Found {len(signed_forms)} shared signed P/A forms") # If the form has already been signed, then the step should be complete. step_complete = signed_forms.count() > 0 + logger.debug(f"{self.project.project_key}/{form.short_name}: Step is completed: {step_complete}") # If the form lives externally, then the step will be marked as permanent because we cannot tell if it was completed. permanent_step = form.type == AGREEMENT_FORM_TYPE_EXTERNAL_LINK step_status = self.get_step_status(form.short_name, step_complete, permanent_step) + logger.debug(f"{self.project.project_key}/{form.short_name}: Step status: {step_status}") title = 'Form: {name}'.format(name=form.name) - if not form.type or form.type == AGREEMENT_FORM_TYPE_STATIC: + if not form.type or form.type == AGREEMENT_FORM_TYPE_STATIC or form.type == AGREEMENT_FORM_TYPE_MODEL: template = 'projects/signup/sign-agreement-form.html' + elif form.type == AGREEMENT_FORM_TYPE_FILE: + template = 'projects/signup/upload-agreement-form.html' elif form.type == AGREEMENT_FORM_TYPE_EXTERNAL_LINK: template = 'projects/signup/sign-external-agreement-form.html' else: @@ -606,6 +665,26 @@ def panel_signed_agreement_forms(self, context): context['informational_panels'].append(panel) + def panel_available_projects(self, context): + """ + Builds the context needed for a user to be able to view any + related DataProjects to this one via team sharing. + """ + # Check if we should list sub-projects + sub_projects = DataProject.objects.filter(teams_source=self.project) + if not sub_projects: + return + + # List them + panel = DataProjectActionablePanel( + title='Tasks', + bootstrap_color='default', + template='projects/participate/sub-project-listing.html', + additional_context={'sub_projects': sub_projects,} + ) + + context['actionable_panels'].append(panel) + def panel_available_downloads(self, context): """ Builds the context needed for a user to be able to download data sets diff --git a/app/requirements.txt b/app/requirements.txt deleted file mode 100644 index a56466e9..00000000 --- a/app/requirements.txt +++ /dev/null @@ -1,24 +0,0 @@ -boto3 -Django==1.11.20 -django-autocomplete-light==3.5.1 -django-bootstrap3==9.0.0 # TODO delete after removing all template extensions -django-bootstrap-datepicker-plus==3.0.5 -django-countries==5.1.1 -django-health-check==3.7.0 -django-jquery==3.1.0 -django-stronghold==0.2.8 -djangorestframework==3.9.1 -djangorestframework-jwt==1.9.0 -django-smtp-ssl==1.0 -furl==1.0.1 -mock==2.0.0 -mysqlclient==1.3.13 -Pillow==6.2.0 -py-auth0-jwt==0.3.0 -py-auth0-jwt-rest==0.1.2 -python-pstore==0.8 -PyJWT==1.6.1 -PyMySQL==0.7.9 -raven==6.1.0 -requests==2.20.0 -sqlparse==0.2.4 diff --git a/app/static/agreementforms/2022-n2c2-dua.html b/app/static/agreementforms/2022-n2c2-dua.html new file mode 100644 index 00000000..170adbea --- /dev/null +++ b/app/static/agreementforms/2022-n2c2-dua.html @@ -0,0 +1,160 @@ +
+

2022 n2c2 Challenge Data Use and Confidentiality Agreement

+ +

Partners HealthCare System, Inc. on behalf of itself and its affiliates (collectively, "Partners") controls certain patient-level data, in the form of patient discharge summaries, from its clinical information systems, which have been de-identified within the meaning of the HIPAA Privacy Rule (the "Data"). Partners wishes to make the Data, in the form of one or more "Datasets", available to eligible registrants in connection with the National NLP Clinical Challenges (n2c2) Shared task and Workshop (the "Shared task"). Partners is making the Data available solely for the purpose of enabling registrants to conduct research (the "Purpose").

+ +

This Data Use and Confidentiality Agreement explains the terms and conditions of access to the Data by registrants in the Shared task. ANY REGISTRANT WHO WISHES TO ACCESS THE DATA MUST READ THE FOLLOWING TERMS AND AGREE TO THEM BY ENTERING THE INFORMATION REQUESTED BELOW.

+ +
TERMS AND CONDITIONS
+ +

1. Registrant understands and agrees that any Data / Datasets that Partners provides to Registrant are proprietary and confidential to Partners.

+

2. Registrant agrees that Registrant will use the Data / Datasets solely for research and for no other purpose.

+

3. Registrant agrees that Registrant will not attempt to identify or re-identify any individual patient or group of patients from the Data / Datasets.

+

4. Registrant agrees that Registrant will not disclose, disseminate, or otherwise share the Data / Datasets to or with any other person or entity except with persons or entities on Registrant's Shared task team, each of which persons/entities Registrant will inform of the obligations with respect to use and confidentiality of the Data under this Agreement. Registrant shall be responsible for compliance by these persons/entities with the terms of this Agreement and any breach thereof.

+

5. Registrant agrees not to use the name or logo of Partners or any of its affiliates or any of their respective trustees, directors, officers, staff members, employees, students or agents for any purpose without Partners' prior written approval excepting in the course of presentation and or publication deriving from the Shared task wherein the data source is acknowledged.

+

6. All Data / Datasets disclosed pursuant to this Agreement, including without limitation all written and tangible forms thereof, shall be and remain the property of Partners. Upon completion of the Shared task or earlier termination of the Agreement as provided in Section 8 below, Registrant shall cease using the Data / Datasets and shall destroy (or return if so requested by Partners) all of the Data / Datasets received in tangible form, including notes, reports, and other information incorporating the Data / Datasets, and shall keep no copies, except to the extent specifically required by law(s) made know to Partners by Registrant.

+

7. Registrant understands and agrees that Partners may use and further share with third parties any Data / Datasets that have been annotated or otherwise enhanced or modified by Registrant in connection with its participation in the Shared task ("Annotated Data / Datasets") for the purpose of conducting or enabling such third parties to conduct research and development of Natural Language Processing tools and analytics. [Registrant shall not be entitled to any compensation in connection with such use of the Annotated Data / Datasets.]

+

8. Partners may terminate Registrant's access to Data / Datasets under this Agreement for any reason upon written notice to Registrant.

+

9. Registrant's obligations under this Agreement shall survive the expiration or termination of the Agreement.

+

10. Registrant acknowledges that any use or disclosure of the Data / Datasets that is inconsistent with the terms of this Agreement may cause irreparable injury to Partners and agrees that Partners will be entitled to seek injunctive relief with respect to such use and/or disclosure, in addition to seeking any other remedy available at law or in equity.

+

11. This Agreement may be modified or amended only in a writing signed by duly authorized representatives of both Registrant and Partners. This Agreement shall be governed by and construed in accordance with the laws of the Commonwealth of Massachusetts. Any claim or action brought under this Agreement shall be brought in the federal or state courts of Massachusetts.

+ +
+ +
REGISTRANT INFORMATION
+ +
Registrant is either (select one):
+ +
+ + +
+ + + + + +
+ +

BY ENTERING THE INFORMATION REQUESTED IN THIS AGREEMENT AS DIRECTED ABOVE, REGISTRANT AFFIRMS THAT REGISTRANT HAS READ THE TERMS AND CONDITIONS OF ACCESS TO DATA FOR THE COMPETITION AND AGREES TO THEM.

+ +
+ + +
+
+ + +
+
+ + +
+ + +
+ + + + diff --git a/app/static/agreementforms/2022-n2c2-roc.html b/app/static/agreementforms/2022-n2c2-roc.html new file mode 100644 index 00000000..33bea748 --- /dev/null +++ b/app/static/agreementforms/2022-n2c2-roc.html @@ -0,0 +1,57 @@ +
+

2022 National NLP Clinical Challenges (n2c2) Shared Task and Workshop

+ +
Rules of Conduct
+ +

The format of the 2022 n2c2 shared task and the principles which bind the participants of this shared task are as follows:

+

1. In order to support the shared task, n2c2 will provide the participants with data from Partners Healthcare and collaborators.

+

2. The data will be distributed with a data use agreement.

+

3. All members of all teams are required to sign the data use agreement online.

+

4. n2c2 will first release annotated training data. Teams can use this data to develop their systems. The systems are expected to be fully automatic, i.e., no human intervention in their output.

+

5. Evaluation is to be run on held-out test data. Teams are not allowed to train on test data. All development and training must stop before teams download the test data. Teams are expected to generate fully automatic system outputs on the test data. Manual interventions with the test data predictions are not acceptable.

+

6. Gaining access to any portion of the 2022 n2c2 data commits the teams to participate in the evaluation that will be run by n2c2. Teams cannot withdraw from the evaluation after gaining access to the data.

+

7. Gaining access to any portion of the 2022 n2c2 data commits the teams to submit a paper describing their developed system to n2c2.

+

8. Gaining access to any portion of the 2022 n2c2 data commits the teams to present their work in the follow-up workshop to be organized by n2c2 (should their submitted paper be accepted for presentation).

+

9. In return for following through with the commitments outlined in the previous three items, the teams can conduct research and publish papers on the released data one year before anyone else. The 2022 n2c2 data will only be available to the research community-at-large one year after the evaluation so that shared-task participants have ample time to publish their work on the shared-task data.

+ +
+ +

I have read and understood the 2022 n2c2 Shared Task Challenge and Workshop Rules of Conduct.

+ +
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+ + +
+ + diff --git a/app/static/agreementforms/MIMIC DUA Sample.pdf b/app/static/agreementforms/MIMIC DUA Sample.pdf new file mode 100644 index 00000000..d621845e Binary files /dev/null and b/app/static/agreementforms/MIMIC DUA Sample.pdf differ diff --git a/app/static/agreementforms/UW External DUA NLP Challenge.pdf b/app/static/agreementforms/UW External DUA NLP Challenge.pdf new file mode 100644 index 00000000..240febb6 Binary files /dev/null and b/app/static/agreementforms/UW External DUA NLP Challenge.pdf differ diff --git a/app/static/agreementforms/mayo_dua.html b/app/static/agreementforms/mayo_dua.html index 67d54088..e159b157 100644 --- a/app/static/agreementforms/mayo_dua.html +++ b/app/static/agreementforms/mayo_dua.html @@ -4,14 +4,14 @@

MAYO DATA USE AGREEMENT


-

THIS AGREEMENT is made effective the day of , ("EFFECTIVE DATE") by and between (Institution name, denoted as "RECIPIENT") and MAYO CLINIC, a Minnesota nonprofit corporation, on its own behalf and for the benefit of all present and future entities that are legal affiliates of Mayo Clinic, a Minnesota nonprofit corporation (Mayo Clinic and all of its affiliates are herein individually and collectively referred to as "MAYO"). The purpose of this Agreement is to satisfy certain obligations of Mayo under the Health Insurance Portability and Accountability Act of 1996 and its implementing regulations (45 C.F.R. Parts 160-64) ("HIPAA") to ensure the integrity and confidentiality of Protected Health Information exchanged in the form of a Limited Data Set for the protocol "THYME" ("Protocol"). The data has been collected under Mayo Clinic IRB approval: #17-003030: Open Health Natural Language Processing Collaboratory. The PI at (Institution) is and the PI at Mayo Clinic is Dr. H. Liu.

- +

THIS AGREEMENT is made effective the day of , ("EFFECTIVE DATE") by and between (Institution name, denoted as "RECIPIENT") and MAYO CLINIC, a Minnesota nonprofit corporation, on its own behalf and for the benefit of all present and future entities that are legal affiliates of Mayo Clinic, a Minnesota nonprofit corporation (Mayo Clinic and all of its affiliates are herein individually and collectively referred to as "MAYO"). The purpose of this Agreement is to satisfy certain obligations of Mayo under the Health Insurance Portability and Accountability Act of 1996 and its implementing regulations (45 C.F.R. Parts 160-64) ("HIPAA") to ensure the integrity and confidentiality of Protected Health Information exchanged in the form of a Limited Data Set for the protocol "THYME" ("Protocol"). The data has been collected under Mayo Clinic IRB approval: #17-003030: Open Health Natural Language Processing Collaboratory. The PI at (Institution) is and the PI at Mayo Clinic is Dr. H. Liu.

+
- +

In consideration of the foregoing and other good and valuable consideration, the receipt and sufficiency of which are hereby acknowledged, Recipient and Mayo agree as follows:

- +
- +

1 Definitions. Capitalized terms used, but not otherwise defined, in this Agreement shall have the meanings given them in HIPAA. For convenience of reference, the definitions of "Individually Identifiable Health Information," "Limited Data Set," and "Protected Health Information" as of the Effective Date are as follows:

1.1 "Individually Identifiable Health Information" means information that is a subset of health information, including demographic information collected from an individual, and (i) is created or received by a healthcare provider, health plan, employer, or health care clearinghouse; and (ii) relates to the past, present, or future physical or mental health or condition of an individual; the provision of healthcare to an individual; or the past, present, or future payment for the provision of health care to an individual; and (a) that identifies the individual, or (b) with respect to which there is a reasonable basis to believe the information can be used to identify the individual.

@@ -22,7 +22,7 @@

MAYO DATA USE AGREEMENT

2 Applicability of Terms; Conflicts. As of the Effective Date, this Agreement automatically amends all existing agreements between Recipient and Mayo involving the use or disclosure of a Limited Data Set for the Protocol. In the event of any conflict or inconsistency between a provision of this Agreement and a provision of any other agreement between Recipient and Mayo regarding the Protocol, the provision of this Agreement shall control unless: (i) Mayo specifically agrees to the contrary in writing, or (ii) the provision in such other agreement establishes additional rights for Mayo or additional duties for or restrictions on Recipient with respect to a Limited Data Set, in which case the provision of such other agreement will control.

-

3 Obligations and Activities of Recipient

+

3 Obligations and Activities of Recipient

3.1 Non-disclosure: Recipient will not use or disclose a Limited Data Set other than as permitted or required by this Agreement or as Required By Law or as otherwise authorized by Mayo.

@@ -66,12 +66,12 @@

MAYO DATA USE AGREEMENT

-

RECIPIENT:

+

RECIPIENT:



@@ -79,15 +79,15 @@

RECIPIENT:

- +
- +
- +

@@ -98,21 +98,21 @@

Address for notices:

- +
- +
- +
- -

MAYO:

-

MAYO CLINIC

+ +

MAYO:

+

MAYO CLINIC


@@ -144,4 +144,4 @@

MAYO CLINIC

Fax: (507) 284-0929

- \ No newline at end of file + diff --git a/app/static/agreementforms/mimic3-dua.html b/app/static/agreementforms/mimic3-dua.html new file mode 100644 index 00000000..24270a54 --- /dev/null +++ b/app/static/agreementforms/mimic3-dua.html @@ -0,0 +1,6 @@ +

Please submit a PDF demonstrating that you have both of the following:

+
    +
  1. A credentialed account on the PhysioNet website
    2. +
  2. An approved DUA associated with the MIMIC-III Clinical Database (v1.4)
    3. +
+

Please carefully follow the instructions on the n2c2 website to assemble this PDF. Instructions vary depending on whether your PhysioNet account was credentialed before or after 2019.

\ No newline at end of file diff --git a/app/static/agreementforms/n2c2-t1_dua.html b/app/static/agreementforms/n2c2-t1_dua.html index 9e205c59..3e69ca6c 100644 --- a/app/static/agreementforms/n2c2-t1_dua.html +++ b/app/static/agreementforms/n2c2-t1_dua.html @@ -1,5 +1,5 @@
-

2018 n2c2 Challenge Data Use and Confidentiality Agreement

+

{% now 'Y' %} n2c2 Challenge Data Use and Confidentiality Agreement

Partners HealthCare System, Inc. on behalf of itself and its affiliates (collectively, "Partners") controls certain patient-level data, in the form of patient discharge summaries, from its clinical information systems, which have been de-identified within the meaning of the HIPAA Privacy Rule (the "Data"). Partners wishes to make the Data, in the form of one or more "Datasets", available to eligible registrants in connection with the National NLP Clinical Challenges (n2c2) Shared task and Workshop (the "Shared task"). Partners is making the Data available solely for the purpose of enabling registrants to conduct research (the "Purpose").

@@ -41,43 +41,43 @@
Individual/Academic
- +
- +
- +
- +
- +
- +
- +
- +
- +
- +
@@ -86,19 +86,19 @@
Corporation
- +
- +
- +
- +
@@ -108,19 +108,19 @@
Corporation
- +
- +
- +
@@ -129,7 +129,7 @@
Corporation
// When someone clicks what kind of registrant they are, hide the opposite form section $('#registrant-is').change(function() { var selected_value = $("input[name='registrant-is']:checked").val(); - + if (selected_value == "individual") { $('#academic-questions').show(); $('#corporation-questions').hide(); @@ -144,7 +144,7 @@
Corporation
$('#corporation-questions').show(); $("#academic-questions :input").prop('required',false); - $("#academic-questions :input").val(''); + $("#academic-questions :input").val(''); $("#corporation-questions :input").prop('required',true); } }); @@ -157,4 +157,4 @@
Corporation
var today = new Date(); document.getElementById("date").valueAsDate = today; }); - \ No newline at end of file + diff --git a/app/static/agreementforms/n2c2-t1_roc.html b/app/static/agreementforms/n2c2-t1_roc.html index 722f7445..4cade027 100644 --- a/app/static/agreementforms/n2c2-t1_roc.html +++ b/app/static/agreementforms/n2c2-t1_roc.html @@ -1,50 +1,50 @@
-

2019 National NLP Clinical Challenges (n2c2) Shared Task and Workshop

+

{% now 'Y' %} National NLP Clinical Challenges (n2c2) Shared Task and Workshop

Rules of Conduct
-

The format of the 2019 n2c2 shared task and the principles which bind the participants of this shared task are as follows:

+

The format of the {% now 'Y' %} n2c2 shared task and the principles which bind the participants of this shared task are as follows:

1. In order to support the shared task, n2c2 will provide the participants with data from Partners Healthcare and collaborators.

2. The data will be distributed with a data use agreement.

3. All members of all teams are required to sign the data use agreement online.

4. n2c2 will first release annotated training data. Teams can use this data to develop their systems. The systems are expected to be fully automatic, i.e., no human intervention in their output.

5. Evaluation is to be run on held-out test data. Teams are not allowed to train on test data. All development and training must stop before teams download the test data. Teams are expected to generate fully automatic system outputs on the test data. Manual interventions with the test data predictions are not acceptable.

-

6. Gaining access to any portion of the 2019 n2c2 data commits the teams to participate in the evaluation that will be run by n2c2. Teams cannot withdraw from the evaluation after gaining access to the data.

-

7. Gaining access to any portion of the 2019 n2c2 data commits the teams to submit a paper describing their developed system to n2c2.

-

8. Gaining access to any portion of the 2019 n2c2 data commits the teams to present their work in the follow-up workshop to be organized by n2c2 (should their submitted paper be accepted for presentation).

-

9. In return for following through with the commitments outlined in the previous three items, the teams can conduct research and publish papers on the released data one year before anyone else. The 2019 n2c2 data will only be available to the research community-at-large one year after the evaluation so that shared-task participants have ample time to publish their work on the shared-task data.

+

6. Gaining access to any portion of the {% now 'Y' %} n2c2 data commits the teams to participate in the evaluation that will be run by n2c2. Teams cannot withdraw from the evaluation after gaining access to the data.

+

7. Gaining access to any portion of the {% now 'Y' %} n2c2 data commits the teams to submit a paper describing their developed system to n2c2.

+

8. Gaining access to any portion of the {% now 'Y' %} n2c2 data commits the teams to present their work in the follow-up workshop to be organized by n2c2 (should their submitted paper be accepted for presentation).

+

9. In return for following through with the commitments outlined in the previous three items, the teams can conduct research and publish papers on the released data one year before anyone else. The {% now 'Y' %} n2c2 data will only be available to the research community-at-large one year after the evaluation so that shared-task participants have ample time to publish their work on the shared-task data.

-

I have read and understood the 2019 n2c2 Shared Task Challenge and Workshop Rules of Conduct.

+

I have read and understood the {% now 'Y' %} n2c2 Shared Task Challenge and Workshop Rules of Conduct.

- +
- +
- +
- +
- +
- +
@@ -54,4 +54,4 @@
Rules of Conduct
var today = $.datepicker.formatDate('yy-mm-dd', new Date()); $('#date').val(today); }); - \ No newline at end of file + diff --git a/app/static/agreementforms/nlp_dua.html b/app/static/agreementforms/nlp_dua.html index 4fc776fb..7ecf820a 100644 --- a/app/static/agreementforms/nlp_dua.html +++ b/app/static/agreementforms/nlp_dua.html @@ -22,9 +22,9 @@

Partners HealthCare

i2b2 National Center for Biomedical Computing

- +
- +
DATA USE AND CONFIDENTIALITY AGREEMENT
for Academic Organizations
for access to data from the Challenges in Natural Language Processing for Clinical Data Shared Task Competitions
@@ -32,188 +32,219 @@
for access to data from the Challenges in Natural Language Processing for Cl
-

This Data Use and Confidentiality Agreement (the "Agreement") is made as of the day of , by and between Partners HealthCare System, Inc., through its i2b2 National Center for Biomedical Computing, and on behalf of itself and its affiliates (collectively, "Partners") and ("Data User").

+

This Data Use and Confidentiality Agreement (the "Agreement") is made as of the day of , by and between Partners HealthCare System, Inc., through its i2b2 National Center for Biomedical Computing, and on behalf of itself and its affiliates (collectively, "Partners") and ("Data User").


WHEREAS, the Partners i2b2 National Center for Biomedical Computing operates an annual Challenges in Natural Language Processing for Clinical Data Shared Task Competition (the "Competition"); and

- +

WHEREAS, Partners controls certain patient-level data in the form of patient discharge summaries from its clinical information systems, which data have been De-Identified within the meaning of the Health Insurance Portability and Accountability Act of 1996 privacy regulations ("HIPAA") and previously utilized and annotated by participants in the Competition (the "Data"); and

- +

WHEREAS, Partners has an interest in supporting the research and development of Natural Language Processing tools and analytics that may advance the meaningful use of electronic health records; and

WHEREAS, to the extent permitted by its Institutional Review Board and institutional policies, Partners now wishes to make the Data, in the form of one or more "Datasets," available to Data User for the purpose of conducting such independent research and development (the "Purpose"), and Data User wishes to receive the Datasets for this Purpose under the terms and conditions of access set forth herein;

- +

NOW, THEREFORE, in consideration of the mutual promises and covenants set forth below, the parties hereby agree as follows:

- +

1. Data User is either (check one):

- + - +

2. Data User will describe to Partners via the electronic registration process for Data access at https://portal.dbmi.hms.harvard.edu the specific natural language processing research and development use for the Data / Datasets proposed by Data User (the "Specific Purpose"). For avoidance of doubt, permissible uses may include use of the Data / Datasets for evaluation and testing of a natural language processing tool or technology but will not extend to proposals that include or incorporate the Data / Datasets into such product. Partners will provide the Data / Datasets requested by the Data User upon Partners' approval, in its sole discretion, of the Specific Purpose.

- +

3. Any Data/Datasets provided to Data User under this Agreement will be De-Identified within the meaning of HIPAA. Data User agrees that Data User will not attempt to identify or re-identify any individual patient or group of patients from the Data / Datasets.

- +

4. Data User agrees that Data User will use the Data / Datasets solely for the Specific Purpose and for no other purpose.

- +

5. Data User understands and agrees that the Data / Datasets are proprietary and confidential to Partners and agrees that Data User will not disclose, disseminate, or otherwise share the Data / Datasets to or with any other person or entity, including any subcontractor, for any purpose, without the prior written consent of Partners. To the extent Partners agrees in writing to permit such further access, the Data User will ensure that such further recipient of the Data / Datasets agrees in writing to all of the same restrictions, conditions and obligations that apply to Data User with respect to the Data / Datasets, and will make Partners a third-party beneficiary of such agreement.


- +
Persons/Entities on Registrant's Research team
- +
- +
- +
- +

Registrant shall be responsible for compliance by these persons/entities with the terms of this Agreement and any breach thereof.

- +

6. If the Data User determines that it is Required by Law (as that term is defined in the HIPAA privacy regulations) to use or disclose the Data / Datasets other than as provided for in this Agreement, Data User shall provide prompt written notice of such determination to Partners so that Partners may have an opportunity to take measures to protect the Data / Datasets as appropriate.

- +

7. Data User will use appropriate safeguards to prevent use or disclosure of the Data / Datasets other than as provided for by this Agreement, and Data User will report immediately to Partners in writing any use or disclosure not provided for by this Agreement of which it becomes aware. The Data User acknowledges that any use or disclosure of the Data / Datasets that is inconsistent with the terms of this Agreement may cause irreparable injury to Partners and agrees that Partners will be entitled to seek injunctive relief with respect to such use and/or disclosure, in addition to seeking any other remedy available at law or in equity.

- +

8. All Data / Datasets disclosed pursuant to this Agreement, including without limitation all written and tangible forms thereof, shall be and remain the property of Partners and Partners shall at all times retain all rights, title and interest in and to the Data / Datasets. Upon the expiration or earlier termination of this Agreement as provided in Section 12 below, the Data User shall cease using the Data / Datasets and shall destroy (or return if so requested by Partners) all of the Data / Datasets received in tangible form, including notes, reports, and other information to the extent it contains the Data / Datasets, and shall keep no copies, except to the extent specifically Required by Law(s) made known to Partners by the Data User.

- +

9. THE DATA / DATASETS ARE PROVIDED "AS IS." PARTNERS MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, CONCERNING THE DATA OR DATASETS OR THE RIGHTS GRANTED HEREIN, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, AND THE ABSENCE OF LATENT OR OTHER DEFECTS, WHETHER OR NOT DISCOVERABLE, AND HEREBY DISCLAIMS THE SAME.

- +

10. IN NO EVENT SHALL PARTNERS OR ANY OF PARTNERS' RESPECTIVE TRUSTEES, DIRECTORS, OFFICERS, MEDICAL OR PROFESSIONAL STAFF, EMPLOYEES AND AGENTS BE LIABLE TO THE DATA USER FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES OF ANY KIND ARISING IN ANY WAY OUT OF THIS AGREEMENT OR RIGHTS GRANTED HEREIN, HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, INCLUDING WITHOUT LIMITATION, ECONOMIC DAMAGES OR INJURY TO PROPERTY OR LOST PROFITS, REGARDLESS OF WHETHER PARTNERS SHALL BE ADVISED, SHALL HAVE OTHER REASON TO KNOW, OR IN FACT SHALL KNOW OF THE POSSIBILITY OF THE FOREGOING.

- +

11. Data User agrees not to use the name or logo of Partners or any of its affiliates or any of their respective trustees, directors, officers, staff members, employees, students or agents for any purpose without Partners' prior written approval; provided, however, that Data User will acknowledge Partners as the source of the Data / Datasets in any publication or presentation arising from the Specific Purpose.

- +

12. This Agreement shall become effective upon Partners' release of Data / Datasets to Data User and shall expire upon Data User's completion of the Specific Purpose. Partners may terminate the Agreement and Data User's access to Data / Datasets hereunder at any prior time and for any reason upon written notice to the Data User.

- +

13. To the extent Data User is permitted under the terms of this Agreement to retain any portion of the Data / Dataset, or any copies thereof, upon the expiration or termination of the Agreement, the Data User's obligations under the Agreement with respect to such Data / Datasets shall survive such expiration or termination for as long as Data User retains the Data / Datasets.

- +

14. This Agreement may be modified or amended only in a writing signed by duly authorized representatives of both the Data User (where Data User is an organization) and Partners. This Agreement shall be governed by and construed in accordance with the laws of the Commonwealth of Massachusetts. Any claim or action brought under this Agreement shall be brought in the federal or state courts of Massachusetts.

- +

15. All notices required by this Agreement shall be provided to the signatory for each party at the address identified below.

- +

16. Sections 3 through 11 and Sections 13, 14, and 15 of this Agreement shall survive its expiration or termination.

- +
- +

Agreed to by:


- -
PARTNERS
- + +
DATA USER
+
- - + +
- - + +
- - + +
- - + +
- -
- -
DATA USER
-
- - + +
- - + +
-
- - +
+
+
+ + +
+
+
+
+ + +
+
- - + +
- +
@@ -224,186 +255,217 @@
DATA USER

Partners HealthCare

i2b2 National Center for Biomedical Computing

- +
- +
DATA USE AND CONFIDENTIALITY AGREEMENT
for Commercial Organizations
for access to data from the Challenges in Natural Language Processing for Clinical Data Shared Task Competitions
 - +
-

This Data Use and Confidentiality Agreement (the "Agreement") is made as of the day of , by and between Partners HealthCare System, Inc., through its i2b2 National Center for Biomedical Computing, and on behalf of itself and its affiliates (collectively, "Partners") and ("Data User").

+

This Data Use and Confidentiality Agreement (the "Agreement") is made as of the day of , by and between Partners HealthCare System, Inc., through its i2b2 National Center for Biomedical Computing, and on behalf of itself and its affiliates (collectively, "Partners") and ("Data User").


WHEREAS, the Partners i2b2 National Center for Biomedical Computing operates an annual Challenges in Natural Language Processing for Clinical Data Shared Task Competition (the "Competition"); and

- +

WHEREAS, Partners controls certain patient-level data in the form of patient discharge summaries from its clinical information systems, which data have been De-Identified within the meaning of the Health Insurance Portability and Accountability Act of 1996 privacy regulations ("HIPAA") and previously utilized and annotated by participants in the Competition (the "Data"); and

- +

WHEREAS, Partners has an interest in supporting the research and development of Natural Language Processing tools and analytics that may advance the meaningful use of electronic health records; and

WHEREAS, to the extent permitted by its Institutional Review Board and institutional policies, Partners now wishes to make the Data, in the form of one or more "Datasets," available to Data User for the purpose of conducting such independent research and development (the "Purpose"), and Data User wishes to receive the Datasets for this Purpose under the terms and conditions of access set forth herein;

- +

NOW, THEREFORE, in consideration of the mutual promises and covenants set forth below, the parties hereby agree as follows:

- +

1. Data User is either (check one):

- + - +

2. Data User will describe to Partners via the electronic registration process for Data access at https://portal.dbmi.hms.harvard.edu the specific natural language processing research and development use for the Data / Datasets proposed by Data User (the "Specific Purpose"). For avoidance of doubt, permissible uses may include use of the Data / Datasets for evaluation and testing of a natural language processing tool or technology but will not extend to proposals that include or incorporate the Data / Datasets into such product. Partners will provide the Data / Datasets requested by the Data User upon Partners' approval, in its sole discretion, of the Specific Purpose.

- +

3. Any Data/Datasets provided to Data User under this Agreement will be De-Identified within the meaning of HIPAA. Data User agrees that Data User will not attempt to identify or re-identify any individual patient or group of patients from the Data / Datasets.

- +

4. Data User agrees that Data User will use the Data / Datasets solely for the Specific Purpose and for no other purpose.

- +

5. Data User understands and agrees that the Data / Datasets are proprietary and confidential to Partners and agrees that Data User will not disclose, disseminate, or otherwise share the Data / Datasets to or with any other person or entity, including any subcontractor, for any purpose, without the prior written consent of Partners. To the extent Partners agrees in writing to permit such further access, the Data User will ensure that such further recipient of the Data / Datasets agrees in writing to all of the same restrictions, conditions and obligations that apply to Data User with respect to the Data / Datasets, and will make Partners a third-party beneficiary of such agreement.

- +

6. If the Data User determines that it is Required by Law (as that term is defined in the HIPAA privacy regulations) to use or disclose the Data / Datasets other than as provided for in this Agreement, Data User shall provide prompt written notice of such determination to Partners so that Partners may have an opportunity to take measures to protect the Data / Datasets as appropriate.

- +

7. Data User will use appropriate safeguards to prevent use or disclosure of the Data / Datasets other than as provided for by this Agreement, and Data User will report immediately to Partners in writing any use or disclosure not provided for by this Agreement of which it becomes aware. The Data User acknowledges that any use or disclosure of the Data / Datasets that is inconsistent with the terms of this Agreement may cause irreparable injury to Partners and agrees that Partners will be entitled to seek injunctive relief with respect to such use and/or disclosure, in addition to seeking any other remedy available at law or in equity.

- +

8. All Data / Datasets disclosed pursuant to this Agreement, including without limitation all written and tangible forms thereof, shall be and remain the property of Partners and Partners shall at all times retain all rights, title and interest in and to the Data / Datasets. Upon the expiration or earlier termination of this Agreement as provided in Section 14 below, the Data User shall cease using the Data / Datasets and shall destroy (or return if so requested by Partners) all of the Data / Datasets received in tangible form, including notes, reports, and other information to the extent it contains the Data / Datasets, and shall keep no copies, except to the extent specifically Required by Law(s) made known to Partners by the Data User.

- +

9. THE DATA / DATASETS ARE PROVIDED "AS IS." PARTNERS MAKES NO REPRESENTATIONS OR WARRANTIES OF ANY KIND, EXPRESS OR IMPLIED, CONCERNING THE DATA OR DATASETS OR THE RIGHTS GRANTED HEREIN, INCLUDING, WITHOUT LIMITATION, WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NONINFRINGEMENT, AND THE ABSENCE OF LATENT OR OTHER DEFECTS, WHETHER OR NOT DISCOVERABLE, AND HEREBY DISCLAIMS THE SAME.

- +

10. IN NO EVENT SHALL PARTNERS OR ANY OF PARTNERS' RESPECTIVE TRUSTEES, DIRECTORS, OFFICERS, MEDICAL OR PROFESSIONAL STAFF, EMPLOYEES AND AGENTS BE LIABLE TO THE DATA USER FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, PUNITIVE OR CONSEQUENTIAL DAMAGES OF ANY KIND ARISING IN ANY WAY OUT OF THIS AGREEMENT OR RIGHTS GRANTED HEREIN, HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, INCLUDING WITHOUT LIMITATION, ECONOMIC DAMAGES OR INJURY TO PROPERTY OR LOST PROFITS, REGARDLESS OF WHETHER PARTNERS SHALL BE ADVISED, SHALL HAVE OTHER REASON TO KNOW, OR IN FACT SHALL KNOW OF THE POSSIBILITY OF THE FOREGOING.

- +

11. Data User shall indemnify, defend and hold harmless Partners and its affiliates and their respective trustees, directors, officers, medical and professional staff, employees, and agents and their respective successors, heirs and assigns (the Indemnities"), against any liability, damage, loss or expense (including reasonable attorneys' fees and expenses of litigation) incurred by or imposed upon the Indemnitees or any one of them in connection with any claims, suits, actions, demands or judgments arising out of any theory of product liability (including, but not limited to, actions in the form of contract, tort, warranty, or strict liability) concerning any product, tool, technology, process or service made, used, or sold or performed pursuant to any right granted under this Agreement. Data User agrees, at its own expense, to provide attorneys reasonably acceptable to Partners to defend against any actions brought or filed against any party indemnified hereunder with respect to the subject of indemnity contained herein, whether or not such actions are rightfully brought; provided, however, that any Indemnitee shall have the right to retain its own counsel, at the expense of Data User, if representation of such Indemnitee by counsel retained by Data User would be inappropriate because of conflict of interests of such Indemnitee and any other party represented by such counsel. Data User agrees to keep Partners informed of the progress in the defense and disposition of such claim and to consult with Partners prior to any proposed settlement.

- +

12. Data User shall maintain insurance sufficient to meet its obligations under Section 11 of this Agreement.

- +

13. Data User agrees not to use the name or logo of Partners or any of its affiliates or any of their respective trustees, directors, officers, staff members, employees, students or agents for any purpose without Partners' prior written approval; provided, however, that Data User will acknowledge Partners as the source of the Data / Datasets in any publication or presentation arising from the Specific Purpose.

- +

14. This Agreement shall become effective upon Partners' release of Data / Datasets to Data User and shall expire upon Data User's completion of the Specific Purpose. Partners may terminate the Agreement and Data User's access to Data / Datasets hereunder at any prior time and for any reason upon written notice to the Data User.

- +

15. To the extent Data User is permitted under the terms of this Agreement to retain any portion of the Data / Dataset, or any copies thereof, upon the expiration or termination of the Agreement, the Data User's obligations under the Agreement with respect to such Data / Datasets shall survive such expiration or termination for as long as Data User retains the Data / Datasets.

- +

16. This Agreement may be modified or amended only in a writing signed by duly authorized representatives of both the Data User (where Data User is an organization) and Partners. This Agreement shall be governed by and construed in accordance with the laws of the Commonwealth of Massachusetts. Any claim or action brought under this Agreement shall be brought in the federal or state courts of Massachusetts.

- +

17. All notices required by this Agreement shall be provided to the signatory for each party at the address identified below.

- +

18. Sections 3 through 13 and Sections 15, 16, and 17 of this Agreement shall survive its expiration or termination.

- +
- +

Agreed to by:


- -
PARTNERS
- + +
DATA USER
+
- - + +
- - + +
- - + +
- - + +
- -
- -
DATA USER
-
- - + +
- - + +
-
- - +
+
+
+ + +
+
+
+
+ + +
+
- - + +
- +
@@ -418,23 +480,50 @@
DATA USER
// Display either the academic or corporate form depending on the selection $('#form-type').change(function() { var selected_value = $("input[name='form-type']:checked").val(); - + if (selected_value == "academic") { + + // Hide subforms + $('#commercial-individual-questions').hide(); + $('#commercial-corporation-questions').hide(); + + // Hide the other section of the DUA and enable this one $('#academic-form').show(); $('#commerical-form').hide(); - $("#academic-form :input").prop('required',true); + // Show and enable and make required academic inputs + $("#academic-form :input[data-required='required']").prop('required',true); + $("#academic-form :input").prop('disabled',false); + + // Hide and disable and make not-required commerical inputs $("#commerical-form :input").prop('required',false); - $("#commerical-form :input").val(''); + $("#commerical-form :input").prop('disabled',true); + $("#commerical-form :input").not(':input[type=radio]').val(''); + $("#commerical-form :input[type=radio]").prop('checked', false); + + // This can be 'individual' or 'corporation' + $("input[name='commercial-registrant-is']").prop("checked", false); } if (selected_value == "commercial") { + + // Hide subforms + $('#individual-questions').hide(); + $('#corporation-questions').hide(); + + // Hide the other section of the DUA and enable this one $('#academic-form').hide(); $('#commerical-form').show(); + // Show and enable and make required commerical inputs + $("#commerical-form :input[data-required='required']").prop('required',true); + $("#commerical-form :input").prop('disabled',false); + + // Hide and disable and make not-required academic inputs $("#academic-form :input").prop('required',false); - $("#academic-form :input").val(''); - $("#commerical-form :input").prop('required',true); + $("#academic-form :input").prop('disabled',true); + $("#academic-form :input").not(':input[type=radio]').val(''); + $("#academic-form :input[type=radio]").prop('checked', false); } // Display the submit button again @@ -444,23 +533,35 @@
DATA USER
// Academic form: When someone clicks what kind of registrant they are, hide the opposite form section $('#registrant-is').change(function() { var selected_value = $("input[name='registrant-is']:checked").val(); - + if (selected_value == "individual") { $('#individual-questions').show(); $('#corporation-questions').hide(); - $("#individual-questions :input").prop('required',true); + // Show and enable and make required individual inputs + $("#individual-questions :input[data-required='required']").prop('required',true); + $("#individual-questions :input").prop('disabled',false); + + // Hide and disable and make not-required corporation inputs $("#corporation-questions :input").prop('required',false); - $("#corporation-questions :input").val(''); + $("#corporation-questions :input").prop('disabled',true); + $("#corporation-questions :input").not(':input[type=radio]').val(''); + $("#corporation-questions :input[type=radio]").prop('checked', false); } if (selected_value == "corporation") { $('#individual-questions').hide(); $('#corporation-questions').show(); + // Show and enable and make required corporation inputs + $("#corporation-questions :input[data-required='required']").prop('required',true); + $("#corporation-questions :input").prop('disabled',false); + + // Hide and disable and make not-required individual inputs $("#individual-questions :input").prop('required',false); - $("#individual-questions :input").val(''); - $("#corporation-questions :input").prop('required',true); + $("#individual-questions :input").prop('disabled',true); + $("#individual-questions :input").not(':input[type=radio]').val(''); + $("#individual-questions :input[type=radio]").prop('checked', false); } // These don't need to be required. @@ -469,29 +570,44 @@
DATA USER
$('#research-team-person-4').prop('required',false); $('#research-team-person-5').prop('required',false); }); - + // Commercial form: When someone clicks what kind of registrant they are, hide the opposite form section $('#commercial-registrant-is').change(function() { - + + // Toggle fields var selected_value = $("input[name='commercial-registrant-is']:checked").val(); - - if (selected_value == "individual") { - $('#commercial-individual-questions').show(); - $('#commercial-corporation-questions').hide(); - $("#commercial-individual-questions :input").prop('required',true); - $("#commercial-corporation-questions :input").prop('required',false); - $("#commercial-corporation-questions :input").val(''); + if (selected_value == "corporation") { + + $('#commercial-individual-questions').hide(); + $('#commercial-corporation-questions').show(); + + // Make commercial inputs required + $("#commercial-corporation-questions :input[data-required='required']").prop('required',true); + $("#commercial-corporation-questions :input").prop('disabled',false); + + // Hide and disable and make not-required individual inputs + $("#commercial-individual-questions :input").prop('required',false); + $("#commercial-individual-questions :input").prop('disabled',true); + $("#commercial-individual-questions :input").not(':input[type=radio]').val(''); + $("#commercial-individual-questions :input[type=radio]").prop('checked', false); } - if (selected_value == "corporation") { - $('#commercial-individual-questions').hide(); - $('#commercial-corporation-questions').show(); + if (selected_value == "individual") { + + $('#commercial-individual-questions').show(); + $('#commercial-corporation-questions').hide(); + + // Make individual inputs required + $("#commercial-individual-questions :input[data-required='required']").prop('required',true); + $("#commercial-individual-questions :input").prop('disabled',false); - $("#commercial-individual-questions :input").prop('required',false); - $("#commercial-individual-questions :input").val(''); - $("#commercial-corporation-questions :input").prop('required',true); + // Hide and disable and make not-required corporation inputs + $("#commercial-corporation-questions :input").prop('required',false); + $("#commercial-corporation-questions :input").prop('disabled',true); + $("#commercial-corporation-questions :input").not(':input[type=radio]').val(''); + $("#commercial-corporation-questions :input[type=radio]").prop('checked', false); } }); }); - \ No newline at end of file + diff --git a/app/static/agreementforms/nlp_purpose.html b/app/static/agreementforms/nlp_purpose.html index 391af22f..2ba163c1 100644 --- a/app/static/agreementforms/nlp_purpose.html +++ b/app/static/agreementforms/nlp_purpose.html @@ -3,4 +3,4 @@
- \ No newline at end of file + diff --git a/app/static/agreementforms/uw-dua.html b/app/static/agreementforms/uw-dua.html new file mode 100644 index 00000000..7a93a404 --- /dev/null +++ b/app/static/agreementforms/uw-dua.html @@ -0,0 +1,9 @@ +{% load static %} +

Please download the following PDF, complete and sign it, then upload the completed/signed copy. You may either a) use Adobe Acrobat to complete/sign the form electronically or b) print a hard-copy, complete/sign it by hand, and scan it.

+
+
+ Download +   |   + View complete instructions +
+
diff --git a/app/static/hms_dbmi_logo.png b/app/static/hms_dbmi_logo.png index 7cbeb7f1..cd4b8635 100644 Binary files a/app/static/hms_dbmi_logo.png and b/app/static/hms_dbmi_logo.png differ diff --git a/app/static/submissionforms/n2c2-submissions.html b/app/static/submissionforms/n2c2-submissions.html index 52f51e6c..19016443 100644 --- a/app/static/submissionforms/n2c2-submissions.html +++ b/app/static/submissionforms/n2c2-submissions.html @@ -1,13 +1,17 @@
- - + +
- + + +
+
+
- - +
- +
+ + +
+ +
+ + +
- +
\ No newline at end of file diff --git a/app/templates/base.html b/app/templates/base.html index 135d0f77..fb545e7e 100644 --- a/app/templates/base.html +++ b/app/templates/base.html @@ -8,6 +8,7 @@ + @@ -36,7 +37,7 @@ - + @@ -45,11 +46,11 @@ - + {% block tab_name %}DBMI Portal{% endblock %} +
\ No newline at end of file diff --git a/app/templates/manage/notification.html b/app/templates/manage/notification.html new file mode 100644 index 00000000..f7408fad --- /dev/null +++ b/app/templates/manage/notification.html @@ -0,0 +1,22 @@ +{% load bootstrap3 %} + +
+
+
+
+ +

Message Team

+
+
+ {% csrf_token %} + {% bootstrap_form notification_form %} +
+
+ {% buttons %} + + + {% endbuttons %} +
+
+
+
diff --git a/app/templates/manage/project-base.html b/app/templates/manage/project-base.html index 71ec8caf..581548cf 100644 --- a/app/templates/manage/project-base.html +++ b/app/templates/manage/project-base.html @@ -183,14 +183,16 @@
Edit - Edit + ic-target="#modal-hosted-file-logs-body" + ic-beforeSend-action="initDownloadLogs"> + Logs @@ -261,6 +263,18 @@

Access management

+{% if project.teams_source %} +
+
+

Info  This project is configured to import teams from {{ project.teams_source.name }}. + {% if project.teams_source_message %} + {{ project.teams_source_message | safe }} + {% endif %} +

+
+
+{% endif %} +
{% if project.has_teams %} @@ -339,49 +353,7 @@

- {% for participant_info in participants %} - - {{ participant_info.participant.user.email }} - {% if project.has_teams %} - {{ participant_info.participant.team.team_leader.email }} - {% endif %} - - {% if participant_info.view_permissions %} - Access granted - {% else %} - No access - {% endif %} - - - {% for form in participant_info.signed_forms %} - - {{ form.agreement_form.short_name }} - - {% endfor %} - - - {% if participant_info.view_permissions %} - - {% else %} - {% if participant_info.signed_accepted_agreement_forms == num_required_forms %} - {% if not project.has_teams %} - - {% else %} - Grant approval via team management. - {% endif %} - {% else %} - Forms incomplete or pending your review - {% endif %} - {% endif %} - - {{ participant_info.download_count }} - {{ participant_info.upload_count }} - - {% endfor %} + {# Loaded via DataTables.js #}

@@ -425,7 +397,7 @@

Submissions

{{ submission.participant.team.team_leader.email }} {% endif %} {{ submission.challenge_task.title }} - {{ submission.upload_date|timezone:"America/New_York" }} (EST) + {{ submission.upload_date|date:"c" }} {{ submission.uuid }}
+ + + + +
+
+
+ +
+ + + + + + + + + + + {% for export in submissions_exports %} + + + + + {% endfor %} @@ -523,7 +541,7 @@
Edit File
RequesterRequested DateFile IDActions
{{ export.requester.email }}{{ export.request_date|date:"c" }}{{ export.uuid }} +
{% for form in member.signed_agreement_forms %} - + {{ form.agreement_form.short_name }} {% endfor %} + + {% if member.access_granted %} @@ -239,7 +268,7 @@

Challenge Submissions

{% endfor %}
- +
+ + +{# Add a placeholder for the modal contact form #} + {% endblock %} {% block footerscripts %} @@ -319,7 +352,7 @@

Previous comments:

"searching": false, "order": [[5, "desc"]] // Sort by status column (4th column) }); - + $('#team-uploads-table').DataTable({ "paging": false, "info": false, @@ -363,8 +396,8 @@

Previous comments:

}, 1500); }, error: function (jqXHR, textStatus, errorThrown) { - $('#delete-submission-confirm').text('Could not delete submission. Please contact an admin for help'); - + $('#delete-submission-confirm').text('Could not delete submission. Please contact an admin for help'); + // Refresh the page. setTimeout(function(){ window.location = window.location.pathname @@ -381,7 +414,7 @@

Previous comments:

var status = ""; $('.team-status-buttons').click(function(event) { - + if (event.target.id == "change-status-to-pending") { status = "pending"; } else if (event.target.id == "change-status-to-ready") { @@ -391,7 +424,7 @@

Previous comments:

} else if (event.target.id == "change-status-to-deactivated") { status = "deactivated"; } - + var request_data = { team: team, project: project, @@ -433,12 +466,51 @@

Previous comments:

alert('Failed to delete team.'); }); }); + + // AJAX for posting + $("#notification-form-button").on('click', function () { + + var url = "{% url 'manage:team-notification' %}?project={{ project.id }}&team={{ team.id }}"; + + $.ajax({ + type: 'GET', + url: url, + success: function (data, textStatus, jqXHR) { + $('#notification-form-modal').html(data); + $('#notification-form-modal').modal('show'); + }, + error : function(xhr,errmsg,err) { + console.log(xhr.status + ": " + xhr.responseText); + } + }); + return false; +}); + +$('#notification-form-modal').on('submit', '#notification-form', function() { + $.ajax({ + url : "{% url 'manage:team-notification' %}", + type : "POST", + data: $(this).serialize(), + context: this, + success : function(json) { + $('#notification-form-modal').modal('hide'); + notify('success', 'Thanks, your message has been submitted!', 'thumbs-up'); + }, + error : function(xhr,errmsg,err) { + $('#notification-form-modal').modal('hide'); + notify('danger', 'Something happened, please try again', 'exclamation-sign'); + console.log(xhr.status + ": " + xhr.responseText); + } + }); + return false; +}); + -{% endblock %} \ No newline at end of file + +{# Add a placeholder for any modal dialogs #} +
+
+
+
+
+
+{% endblock %} diff --git a/app/templates/manage/upload-signed-agreement-form.html b/app/templates/manage/upload-signed-agreement-form.html new file mode 100644 index 00000000..795486a1 --- /dev/null +++ b/app/templates/manage/upload-signed-agreement-form.html @@ -0,0 +1,20 @@ +{% load bootstrap3 %} + +
+
+ +

Signed Agreement Form Upload

+
+
+ {% csrf_token %} + {% bootstrap_form form inline=True %} +
+
+ {% buttons %} + + + {% endbuttons %} +
+
diff --git a/app/templates/projects/list-data-challenges.html b/app/templates/projects/list-data-challenges.html index f19b4e80..9e5a1994 100644 --- a/app/templates/projects/list-data-challenges.html +++ b/app/templates/projects/list-data-challenges.html @@ -3,12 +3,10 @@ {% block headscripts %} {% endblock %} -{% block title %}Data Challenges{% endblock %} +{% block title %}n2c2 Challenges{% endblock %} {% block subtitle %} -There are currently no challenges in progress. -

You may click on a challenge below to learn more about it, but only previously registered participants of a given challenge may access its corresponding set of files.

-

To access our publicly available data sets, including the n2c2 NLP data sets that originally came out of challenges held during the i2b2 project, visit Data Sets.

-

If you have your own annotations on n2c2 (formerly i2b2) data that you would like to contribute to the research community, you may submit them at any time through n2c2 Data Upload: Community generated annotations. Thank you!

+

Registration for the current challenges has closed. Learn more on the n2c2 website.

+

If you have your own annotations on n2c2 (formerly i2b2) data that you would like to contribute to the research community, you may submit them at any time through n2c2 Data Upload: Community generated annotations. Thank you!

{% endblock %} {% block subcontent %} diff --git a/app/templates/projects/list-data-projects.html b/app/templates/projects/list-data-projects.html index 998243f3..5f5dd000 100644 --- a/app/templates/projects/list-data-projects.html +++ b/app/templates/projects/list-data-projects.html @@ -3,7 +3,7 @@ {% block headscripts %} {% endblock %} -{% block title %}Data Sets{% endblock %} +{% block title %}n2c2 Data Sets{% endblock %} {% block subtitle %}Click on a data set below to learn more{% endblock %} {% block subcontent %} diff --git a/app/templates/projects/participate/complete-tasks.html b/app/templates/projects/participate/complete-tasks.html index 2ae5b809..0b5ff512 100644 --- a/app/templates/projects/participate/complete-tasks.html +++ b/app/templates/projects/participate/complete-tasks.html @@ -7,8 +7,8 @@ {% if task_enabled %}

- Task: {{ task_detail.task.title }} - + Task: {{ task_detail.task.title }} + {% if task_detail.submissions_left is not None %} {{ task_detail.submissions_left }} submissions left {% endif %} @@ -32,24 +32,31 @@

You have used up all of your available submissions. You may delete a previous submission if you wish to submit a new one.

{% endif %} {% else %} -
+ {% if task_detail.task.submission_form_file_path %}
{{ task_detail.task.submission_form_file_path|get_html_form_file_contents | safe }}
{% endif %} - + +
+ +
- - - - - - + +
+
+ +
+
+ +
+
+
{% csrf_token %} @@ -63,7 +70,7 @@

{% for submission in task_detail.submissions %}
  • {{ submission.participant.user.email }} on {{ submission.upload_date|timezone:"America/New_York" }} (EST) - + {# Hide the submission metadata json here #} @@ -94,7 +101,7 @@ - +

    • @@ -120,88 +127,86 @@ var input = $(this); var fileName = input.val().replace(/\\/g, '/').replace(/.*\//, ''); - $(this).parent().parent().find('.file-upload-submit').text('Submit ' + fileName); - $(this).parent().parent().find('.file-upload-browse').hide(); - $(this).parent().parent().find('.file-upload-submit').show(); - - $(this).parent().parent().find('.file-upload-filename').val(fileName); + $('#file-upload-filename').val(fileName); }); // Set the handler for the participant submission form. $(document).on('submit', '.participant-submission-form', function (event) { - // Get the file. - var file = $(this).find(".file-upload-file").prop('files')[0]; - if (file == null) { - error('A file has not been selected, please try again'); - return false; - } - - // Only allow zip files for upload. - if (file.name.split('.').pop() != 'zip') { - error('Only .zip files are accepted.'); - return false; - } - - // Get the form data. - var form = objectifyForm($(this).serializeArray()); - - // Remove the file from the serialized form as it will not be needed yet. - delete form['file']; - - // Add info about the file. - form['content_type'] = file.content; - - // Disable the form and the buttons. - $(this).find(".file-upload-submit").button('loading'); - - $.ajax({ - method: "POST", - data: form, - url: "{% url 'projects:upload_challengetasksubmission_file' %}", - success: function (data, textStatus, jqXHR) { - console.log("submit.success: " + textStatus); - upload(data["post"], data["file"], file, form, $(this)); - }, - error: function (jqXHR, textStatus, errorThrown) { - console.log("submit.error: " + errorThrown); - error('Something happened, please try again'); - } - }); - - // Return false to prevent a form submit effect - return false; + // Get the form + var form = $(this); + + // Get the file input + var fileInput = form.find(".file-upload-file"); + + // Get the file. + var file = fileInput.prop('files')[0]; + if (file == null) { + error('A file has not been selected, please try again', form); + return false; + } + + // Validate content type + if( file.type !== fileInput.data("content-type")) { + console.log(`Blob type ${file.type} !== ${fileInput.data("content-type")}`); + error(`Content type "${file.type}" is not accepted. Only files of type "${fileInput.data("content-type")}" are accepted.`, form); + return true; + } + + // Get the form data. + var formData = objectifyForm($(form).serializeArray()); + + console.log(formData); + + // Remove the file from the serialized form as it will not be needed yet. + delete formData['file']; + + // Add info about the file. + formData['content_type'] = file.content; + + // Disable the form and the buttons. + form.find(".file-upload-submit").button('loading'); + + $.ajax({ + method: "POST", + data: formData, + url: "{% url 'projects:upload_challengetasksubmission_file' %}", + success: function (data, textStatus, jqXHR) { + console.log("submit.success: " + textStatus); + upload(data["post"], data["file"], file, formData, form); + }, + error: function (jqXHR, textStatus, errorThrown) { + console.log("submit.error: " + errorThrown); + error('Something happened, please try again', form); + } + }); + + // Return false to prevent a form submit effect + return false; }); // Uploads the file to AWS S3. - function upload(post, fileRecord, file, submission_form, form_element) { + function upload(post, fileRecord, file, formData, form) { console.log('upload: ' + fileRecord["filename"]); // Construct the needed data using the policy for AWS - var form = new FormData(); + var uploadForm = new FormData(); var fields = post["fields"]; for(var key in fields) { - form.append(key, fields[key]); + uploadForm.append(key, fields[key]); } // Add the file - form.append('file', file); + uploadForm.append('file', file); // Send the file to S3. $.ajax({ url: post["url"], datatype: 'xml', - data: form, + data: uploadForm, type: 'POST', contentType: false, processData: false, - xhr: function () { - - // Add a progress handler - var xhr = $.ajaxSettings.xhr(); - xhr.upload.onprogress = progress; - return xhr; - }, success: function (data, textStatus, jqXHR) { console.log('upload.success: ' + textStatus); @@ -209,7 +214,7 @@ // $("#participant-submission-form :input").prop("disabled", true); // Inform the server that the upload completed. - complete(fileRecord, submission_form, form_element); + complete(fileRecord, formData, form); }, error: function (jqXHR, textStatus, errorThrown) { console.log('upload.error: ' + errorThrown + ', ' + textStatus); @@ -217,20 +222,20 @@ // Check if aborted. if( !jqXHR.getAllResponseHeaders() ) { // Cancelled notification. - aborted(form_element); + aborted(form); } else { - error('Something happened, please try again', form_element); + error('Something happened, please try again', form); } } }); } - function complete(fileRecord, submission_form, form_element) { + function complete(fileRecord, formData, form) { console.log('complete: ' + fileRecord["filename"]); - - // Combine information about the fileservice metadata on the file with the + + // Combine information about the fileservice metadata on the file with the // original form submission. - $.extend(fileRecord, submission_form) + $.extend(fileRecord, formData) // Inform the server that the upload completed. $.ajax({ @@ -241,7 +246,7 @@ console.log("complete.success: " + textStatus); // Notify. - success(form_element); + success(form); // Refresh the page. setTimeout(function(){ @@ -252,35 +257,36 @@ console.log("complete.error: " + errorThrown); // Error with message. - error('Something happened, please try again', form_element); + error('Something happened, please try again', form); } }); } - function progress(event) { - // Ensure it can be used - if (event.lengthComputable) { - var progress = event.loaded / event.total * 100; - console.log('upload.progress: ' + progress); - } - } - - function error(message, form_element) { - form_element.find('.file-upload-submit').text('File failed to upload'); + function error(message, form) { notify('danger', message, 'glyphicon glyphicon-remove'); + + // Disable the form and the buttons. + form.find(".file-upload-submit").button('reset'); } - function aborted(form_element) { - form_element.find('.file-upload-submit').text('File uploaded canceled'); + function aborted(form) { + form.find('.file-upload-submit').text('File uploaded canceled'); notify('warning', 'The upload was cancelled', 'glyphicon glyphicon-warning-sign'); } - function success(form_element) { - // Add a little delay - setTimeout(function(){ - form_element.find('.file-upload-submit').text('File uploaded successfully!'); - notify('success', 'The upload has completed successfully! Refreshing page.', 'glyphicon glyphicon-ok'); - }, 500); + function success(form) { + + // Update the text. + $('#file-upload-progress').text('Completed!'); + + // Add a little delay + setTimeout(function(){ + notify('success', 'The upload has completed successfully! Refreshing page.', 'glyphicon glyphicon-ok'); + }, 500); + + // Disable the form and the buttons. + form.find(".file-upload-submit").prop("disabled", true); + form.find(".file-upload-submit").text('File uploaded successfully'); } }); @@ -292,7 +298,7 @@ // Grab the information from an adjacent div var submission_info = $(this).next(".submission-info-details").text(); - + // Parse the string into an actual json var submission_info_json = JSON.parse(submission_info); var submission_info_json_prettyprint = JSON.stringify(submission_info_json, null, 4); @@ -341,4 +347,4 @@ } }); }); - \ No newline at end of file + diff --git a/app/templates/projects/participate/signed-agreement-forms.html b/app/templates/projects/participate/signed-agreement-forms.html index 10dfcdfd..3743ee47 100644 --- a/app/templates/projects/participate/signed-agreement-forms.html +++ b/app/templates/projects/participate/signed-agreement-forms.html @@ -7,7 +7,7 @@

    {{ signed_form.agreement_form.name }}

    Signed {{ signed_form.date_signed|timezone:"America/New_York" }} (EST)
    - View + View
    -{% endfor %} \ No newline at end of file +{% endfor %} diff --git a/app/templates/projects/participate/sub-project-listing.html b/app/templates/projects/participate/sub-project-listing.html new file mode 100644 index 00000000..61fa4ce0 --- /dev/null +++ b/app/templates/projects/participate/sub-project-listing.html @@ -0,0 +1,13 @@ +{% load projects_extras %} +{% load tz %} + +
    + {% for project in panel.additional_context.sub_projects %} +
    +
    + {% include "projects/project-blurb.html" with project=project project_counter=forloop.counter0 %} +
    +
    +
    + {% endfor %} +
    diff --git a/app/templates/projects/participate/view-signed-agreement-form.html b/app/templates/projects/participate/view-signed-agreement-form.html index e68ae7e0..649f3686 100644 --- a/app/templates/projects/participate/view-signed-agreement-form.html +++ b/app/templates/projects/participate/view-signed-agreement-form.html @@ -2,6 +2,7 @@ {% load tz %} {% block headscripts %} + {% endblock %} {% block tab_name %}Signed Agreement Form{% endblock %} @@ -15,8 +16,26 @@ -
    +
    + {# Check type #} + {% if signed_form.agreement_form.type == "FILE" %} + + {# Handle instances where a DUA was submitted when it was not required to upload a file #} + {% if not signed_form.upload %} +
    + +
    + {% else %} +
    + +
    + {% endif %} + + {% else %} {{ signed_form.agreement_text|linebreaks }} + {% endif %}
    @@ -33,6 +52,21 @@

    Information

    For Project: {{ signed_form.project.name }}

    Form Name: {{ signed_form.agreement_form.name }}

    Signed on: {{ signed_form.date_signed|timezone:"America/New_York" }} (EST)

    + {% if signed_form.fields %} +

    Fields:

    + + + + + + {% for field, value in signed_form.fields.items %} + + + + + {% endfor %} +
    FieldValue
    {{ field }}{{ value }}
    + {% endif %} @@ -44,7 +78,7 @@

    Actions

    - Download + Download
    @@ -52,7 +86,7 @@

    Actions