diff --git a/.github/workflows/backendfrontend.yml b/.github/workflows/backendfrontend.yml index 81bff9ab..728ecca0 100644 --- a/.github/workflows/backendfrontend.yml +++ b/.github/workflows/backendfrontend.yml @@ -41,7 +41,7 @@ jobs: --health-retries 5 steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - uses: conda-incubator/setup-miniconda@v2 with: @@ -54,7 +54,7 @@ jobs: run: conda install -y -c conda-forge postgresql=14.* - name: Use Node.js v14 - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: '14' diff --git a/.github/workflows/build-docker.yml b/.github/workflows/build-docker.yml index 19d9e244..87ce4d24 100644 --- a/.github/workflows/build-docker.yml +++ b/.github/workflows/build-docker.yml @@ -14,7 +14,15 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 + - + name: Buildx cache + uses: actions/cache@v3 + with: + path: buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('buildx-cache/**') }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Docker meta (backend) id: metaBackend @@ -35,6 +43,9 @@ jobs: type=ref,event=branch type=semver,pattern={{version}} type=semver,pattern={{major}}.{{minor}} + - + name: Set up qemu + uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v2 @@ -47,23 +58,27 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push backend - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: context: . target: backend push: true tags: ${{ steps.metaBackend.outputs.tags }} labels: ${{ steps.metaBackend.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local,src=buildx-cache + cache-to: type=local,dest=buildx-cache-out,mode=max + platforms: linux/amd64,linux/arm64 - name: Build and push static files - uses: docker/build-push-action@v3 + uses: docker/build-push-action@v4 with: context: . target: frontend push: true tags: ${{ steps.metaStatic.outputs.tags }} labels: ${{ steps.metaStatic.outputs.labels }} - cache-from: type=gha - cache-to: type=gha,mode=max + cache-from: type=local,src=buildx-cache-out + platforms: linux/amd64,linux/arm64 + - + name: Clean up cache + run: rm -rf buildx-cache && mv buildx-cache-out buildx-cache \ No newline at end of file diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml new file mode 100644 index 00000000..0fda7354 --- /dev/null +++ b/.github/workflows/create-release.yml @@ -0,0 +1,47 @@ +name: create-release + +on: + push: + tags: + - "v*" + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + + - name: Extract changelog + run: | + cat CHANGELOG.md | perl -e ' + BEGIN { + $myversion = $ARGV[0]; + $myversion =~ s/^v//; + } + while() { + if(/^## *\[\Q$myversion\E\]/) { + $flag = 1; + } elsif(/^## *\[[0-9.]*\]/) { + $flag = 0; + } elsif($flag) { + print; + } + }' -- {{ github.ref_name }} > release-changelog.md + + # Fail if the changelog is empty, i.e. there is no changelog for this release + [ -s release-changelog.md ] + + - name: Create release artifacts + run: | + IMAGE_TAG="{{ github.ref_name }}" + sed "s/DEFAULT_IMAGE_TAG=latest/DEFAULT_IMAGE_TAG=${IMAGE_TAG#v}/" install/get-teamware.sh > ./get-teamware.sh + tar cvzf install.tar.gz README.md docker-compose*.yml generate-docker-env.sh create-django-db.sh nginx custom-policies Caddyfile + + - name: Create release + uses: softprops/action-gh-release@v1 + with: + body_path: release-changelog.md + files: | + get-teamware.sh + install.tar.gz \ No newline at end of file diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index e473e776..72077051 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -2,14 +2,13 @@ name: Build and deploy documentation on: push: branches: - - master - dev jobs: build-and-deploy: runs-on: ubuntu-latest steps: - name: Checkout 🛎️ - uses: actions/checkout@v2.3.1 + uses: actions/checkout@v3 - name: Install conda uses: conda-incubator/setup-miniconda@v2 with: @@ -19,15 +18,16 @@ jobs: - name: Install python dependencies 🐍 run: pip install -r requirements.txt -r requirements-dev.txt - name: Use Node.js v14 - uses: actions/setup-node@v2 + uses: actions/setup-node@v3 with: node-version: '14' - name: Install and Build 🔧 run: | npm install + npm run install:docs npm run build:docs - name: Deploy 🚀 - uses: JamesIves/github-pages-deploy-action@4.0.0 + uses: JamesIves/github-pages-deploy-action@v4 with: branch: docs-gh-pages # The branch the action should deploy to. folder: docs/site/gate-teamware # The folder the action should deploy. diff --git a/.github/workflows/image-build-integration-tests.yml b/.github/workflows/image-build-integration-tests.yml index a4c9fbe1..34562048 100644 --- a/.github/workflows/image-build-integration-tests.yml +++ b/.github/workflows/image-build-integration-tests.yml @@ -19,7 +19,18 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Buildx cache + uses: actions/cache@v3 + with: + path: buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('buildx-cache/**') }} + restore-keys: | + ${{ runner.os }}-buildx- - name: Generate env shell: bash @@ -31,20 +42,19 @@ jobs: # Display the environment variable file for the logs cat .env - - - name: Set up Docker cache - uses: satackey/action-docker-layer-caching@v0.0.11 - continue-on-error: true - with: - key: integration-test-docker-cache-{hash} - restore-keys: | - integration-test-docker-cache- - - uses: docker-practice/actions-setup-docker@master - - run: | + - name: Build images + run: | + BUILDX_ARGS_BACKEND="--cache-from type=local,src=buildx-cache --cache-to type=local,dest=buildx-cache-out,mode=max" + export BUILDX_ARGS_BACKEND + BUILDX_ARGS_FRONTEND="--cache-from type=local,src=buildx-cache-out" + export BUILDX_ARGS_FRONTEND + ./build-images.sh - - uses: actions/setup-node@v1 + rm -rf buildx-cache && mv buildx-cache-out buildx-cache + + - uses: actions/setup-node@v3 with: node-version: 14 diff --git a/.github/workflows/image-tests.yml b/.github/workflows/image-tests.yml index d975daa0..1239d5ab 100644 --- a/.github/workflows/image-tests.yml +++ b/.github/workflows/image-tests.yml @@ -12,15 +12,18 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - - name: Set up Docker cache - uses: satackey/action-docker-layer-caching@v0.0.11 - continue-on-error: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v2 + + - name: Buildx cache + uses: actions/cache@v3 with: - key: unit-test-docker-cache-{hash} + path: buildx-cache + key: ${{ runner.os }}-buildx-${{ hashFiles('buildx-cache/**') }} restore-keys: | - unit-test-docker-cache- + ${{ runner.os }}-buildx- - name: Run the backend and frontend tests in containers run: | @@ -38,7 +41,10 @@ jobs: cat .env # Build the test image with npm and pytest installed - docker build -t teamware-main:latest --target test . + docker buildx build --cache-from type=local,src=buildx-cache --cache-to type=local,dest=buildx-cache-out,mode=max --load -t teamware-main:latest --target test . + + # Clear up the cache + rm -rf buildx-cache && mv buildx-cache-out buildx-cache # Export the environment variables source .env diff --git a/.github/workflows/integration-tests-cypress-record.yml b/.github/workflows/integration-tests-cypress-record.yml index a004776f..06b121b7 100644 --- a/.github/workflows/integration-tests-cypress-record.yml +++ b/.github/workflows/integration-tests-cypress-record.yml @@ -13,7 +13,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Generate env shell: bash @@ -38,7 +38,7 @@ jobs: - run: | ./build-images.sh - - uses: actions/setup-node@v1 + - uses: actions/setup-node@v3 with: node-version: 14 diff --git a/.github/workflows/validate-cff.yml b/.github/workflows/validate-cff.yml index b042b3f6..4ae675e8 100644 --- a/.github/workflows/validate-cff.yml +++ b/.github/workflows/validate-cff.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out a copy of the repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Check whether the citation metadata from CITATION.cff is valid uses: citation-file-format/cffconvert-github-action@2.0.0 diff --git a/CHANGELOG.md b/CHANGELOG.md index 5a53b8c5..529fb0d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,15 +1,46 @@ # Changelog -## [unreleased] +## [development] ### Added + +### Fixed + +## [2.0.0] 2023-04-13 +### Added +- Isolate documentation build chain ([#326](https://github.com/GateNLP/gate-teamware/pull/326)) +- Add doi to citation file and doi badge ([#332](https://github.com/GateNLP/gate-teamware/pull/332)) +- Add logging and alter data type for telemetry ([#333](https://github.com/GateNLP/gate-teamware/pull/333)) +- Add more logging when telemetry is switched off and send_telemetry is called ([#337](https://github.com/GateNLP/gate-teamware/pull/337)) + ### Changed +- Update node 12 actions to newer node 16 versions ([#334](https://github.com/GateNLP/gate-teamware/pull/334)) + +### Fixed +- Fix for documentation build breaking ([#336](https://github.com/GateNLP/gate-teamware/pull/336)) + +## [0.4.0] 2023-04-03 +### Added +- Privacy policy & Terms & Conditions ([#298](https://github.com/GateNLP/gate-teamware/pull/298)) +- Helm chart moved to its own repo ([#299](https://github.com/GateNLP/gate-teamware/pull/299)) +- Added a cookies policy page ([#301](https://github.com/GateNLP/gate-teamware/pull/301)) +- Dynamic options for checkbox/radio/selector ([#303](https://github.com/GateNLP/gate-teamware/pull/303)) +- Simpler install process for new users ([#305](https://github.com/GateNLP/gate-teamware/pull/305)) +- Multi-arch build support ([#306](https://github.com/GateNLP/gate-teamware/pull/306)) +- Added label to the "Other" issue report, converted bold headings to section headers (h2) instead ([#310](https://github.com/GateNLP/gate-teamware/pull/310)) +- Add footer link to repository and add a little more info to about page ([#312](https://github.com/GateNLP/gate-teamware/pull/312)) +- Allowing users to be deleted from the system ([#318](https://github.com/GateNLP/gate-teamware/pull/318)) +- Update the "making a release" documentation to match latest changes ([#319](https://github.com/GateNLP/gate-teamware/pull/319)) + ### Fixed +- Fixed t.currentAnnotationTask is null error ([#302](https://github.com/GateNLP/gate-teamware/pull/302)) +- Don't redeploy docs on push to master ([#316](https://github.com/GateNLP/gate-teamware/pull/316)) +- Admin role should imply manager ([#321](https://github.com/GateNLP/gate-teamware/pull/321)) -## [0.3.1] +## [0.3.1] - 2023-02-17 ### Fixed - Missed underscore in documentation link ([#292](https://github.com/GateNLP/gate-teamware/pull/292)) -## [0.3.0] +## [0.3.0] - 2023-02-16 ### Added - Telemetry data sending ([#270](https://github.com/GateNLP/gate-teamware/pull/270)) - Upgrade Docker actions ([#272](https://github.com/GateNLP/gate-teamware/pull/272)) diff --git a/CITATION.cff b/CITATION.cff index 4fa19eae..0dc88d2a 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -22,6 +22,11 @@ authors: given-names: Kalina orcid: https://orcid.org/0000-0001-6152-9600 cff-version: 1.2.0 +identifiers: +- description: The collection of archived snapshots of all versions of GATE Teamware + 2 + type: doi + value: 10.5281/zenodo.7821718 keywords: - NLP - machine learning @@ -32,4 +37,4 @@ repository-code: https://github.com/GateNLP/gate-teamware title: GATE Teamware type: software url: https://gatenlp.github.io/gate-teamware/ -version: 0.3.1 +version: 2.0.0 diff --git a/Caddyfile b/Caddyfile new file mode 100644 index 00000000..de2aa135 --- /dev/null +++ b/Caddyfile @@ -0,0 +1,12 @@ +# +# This is a configuration file for https://caddyserver.com to implement +# a very simple HTTPS reverse proxy in the docker-compose stack. +# +{$DJANGO_ALLOWED_HOSTS} + +handle /static/* { + reverse_proxy nginx:80 +} +handle * { + reverse_proxy backend:8000 +} diff --git a/Dockerfile b/Dockerfile index 29f1130a..bd9e801b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,5 @@ -FROM node:14-buster-slim as nodebuilder +# Node build currently only works on amd64 +FROM --platform=linux/amd64 node:14-buster-slim as nodebuilder RUN mkdir /app/ WORKDIR /app/ COPY package.json package-lock.json ./ @@ -9,11 +10,12 @@ RUN npm run build FROM python:3.9-slim-buster AS backend +ARG TARGETARCH ENV PYTHONUNBUFFERED 1 RUN apt-get --allow-releaseinfo-change update && \ apt-get -y install gcc libpq-dev libmagic1 postgresql-client && \ rm -rf /var/lib/apt/lists/* -ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /sbin/tini +ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TARGETARCH /sbin/tini RUN addgroup --gid 1001 "gate" && \ adduser --disabled-password --gecos "GATE User,,," \ --home /app --ingroup gate --uid 1001 gate && \ diff --git a/README.md b/README.md index eeb3b278..b7d9927f 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ ![](/frontend/public/static/img/gate-teamware-logo.svg "GATE Teamware") +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7821718.svg)](https://doi.org/10.5281/zenodo.7821718) + A web application for collaborative document annotation. Full documentation can be [found here][docs]. @@ -17,18 +19,29 @@ GATE teamware provides a flexible web app platform for managing classification o * Deploy with [kubernetes](https://kubernetes.io/) or [docker compose](https://docs.docker.com/compose/). # Running the app -## Requirements -We recommend the following software as a minimum requirement for running GATE Teamware: -* recommended OS: linux or macOS. -* [git](http://git-scm.com/) -* [docker](https://www.docker.com/) & [docker-compose](https://docs.docker.com/compose/) -* [bash](https://www.gnu.org/software/bash/) - -## Instructions + +## Latest release + +The simplest way to deploy your own copy of GATE Teamware is to use Docker Compose on Linux or Mac. Installation on Windows is possible but not officially supported - you need to be able to run `bash` shell scripts for the quick-start installer. + +1. Install Docker - [Docker Engine](https://docs.docker.com/engine/) for Linux servers or [Docker Desktop](https://docs.docker.com/desktop/) for Mac. +2. Install [Docker Compose](https://github.com/docker/compose), if your Docker does not already include it (Compose is included by default with Docker Desktop) +3. Download the [installation script](https://gate.ac.uk/get-teamware.sh) into an empty directory, run it and follow the instructions. + +``` +mkdir gate-teamware +cd gate-teamware +curl -LO https://gate.ac.uk/get-teamware.sh +bash ./get-teamware.sh +``` + +[A Helm chart](https://github.com/GateNLP/charts/tree/main/gate-teamware) is also available to allow deployment on Kubernetes. + +## Building locally Follow these steps to run the app on your local machine using `docker-compose`: 1. Clone this repository by running `git clone https://github.com/GateNLP/gate-teamware.git` and move into the `gate-teamware` directory. 1. From inside the `gate-teamware` directory run `./generate-docker-env.sh` to create a set of passwords and keys in a `.env` file. -1. Run `./build-images.sh` to build the backend and frontend images, this may take a while the first time it is run. +1. Run `./build-images.sh` to build the backend and frontend images, this may take a while the first time it is run. Images are built using `docker buildx`, which requires Docker Engine 19.03 or later. 1. Run `./deploy.sh production` or `./deploy.sh staging`. Note: You may want to change the value of `DJANGO_ALLOWED_HOSTS` in `deploy.sh` to match the URL(s) that you are deploying to. Open `127.0.0.1:8076` (the default IP & port) in your browser. The initial admin login has the username `admin` and password `password`, this should be changed immediately. Note: the port is set in `docker-compose.yml`. @@ -36,7 +49,7 @@ Open `127.0.0.1:8076` (the default IP & port) in your browser. The initial admin *Notes on deployment*: * A development server can be run without docker, see the [developer documentation][dev-docs] for more info. -* The app can also be deployed on a kubernetes cluster, helm charts are available in the `charts/` directory. +* [A Helm chart](https://github.com/GateNLP/charts/tree/main/gate-teamware) is available to deploy GATE Teamware on a kubernetes cluster. # Development Developer documentation is [provided here][dev-docs]. diff --git a/VERSION b/VERSION index 9e11b32f..227cea21 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.3.1 +2.0.0 diff --git a/backend/management/commands/check_create_superuser.py b/backend/management/commands/check_create_superuser.py index 81dbc565..0b1dfea5 100644 --- a/backend/management/commands/check_create_superuser.py +++ b/backend/management/commands/check_create_superuser.py @@ -2,7 +2,6 @@ from django.contrib.auth import get_user_model from django.core.management.base import BaseCommand, CommandError -from backend.rpc import _generate_user_activation class Command(BaseCommand): @@ -17,8 +16,8 @@ def handle(self, *args, **options): email = os.environ.get("SUPERUSER_EMAIL") if not User.objects.filter(username=username).exists(): - user = User.objects.create_superuser(username=username, password=password, email=email) - _generate_user_activation(user) + User.objects.create_superuser(username=username, password=password, email=email, + is_account_activated=True) self.stdout.write(f'No superusers found in database.\nSuperuser created with username {username}') else: diff --git a/backend/management/commands/load_test_fixture.py b/backend/management/commands/load_test_fixture.py index 70c3e708..7795f923 100644 --- a/backend/management/commands/load_test_fixture.py +++ b/backend/management/commands/load_test_fixture.py @@ -35,6 +35,7 @@ def create_db_users(): annotator.is_account_activated = True annotator.save() + @test_fixture def create_db_users_with_project(): """ @@ -81,6 +82,15 @@ def create_db_users_with_project(): } document = Document.objects.create(project=project, data=doc_data) +@test_fixture +def create_db_users_with_project_annotator_personal_info_deleted(): + """ + Same as create_db_users_with_project but with annoator's personal info deleted + """ + create_db_users_with_project() + annotator_user = get_user_model().objects.get(username="annotator") + annotator_user.delete_user_personal_information() + @test_fixture def create_db_users_with_project_admin_is_annotator(): create_db_users_with_project() diff --git a/backend/migrations/0029_serviceuser_agreed_privacy_policy.py b/backend/migrations/0029_serviceuser_agreed_privacy_policy.py new file mode 100644 index 00000000..d09c0570 --- /dev/null +++ b/backend/migrations/0029_serviceuser_agreed_privacy_policy.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-02-22 10:56 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0028_project_uuid'), + ] + + operations = [ + migrations.AddField( + model_name='serviceuser', + name='agreed_privacy_policy', + field=models.BooleanField(default=False), + ), + ] diff --git a/backend/migrations/0030_serviceuser_is_deleted.py b/backend/migrations/0030_serviceuser_is_deleted.py new file mode 100644 index 00000000..1d961db2 --- /dev/null +++ b/backend/migrations/0030_serviceuser_is_deleted.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2023-03-16 16:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0029_serviceuser_agreed_privacy_policy'), + ] + + operations = [ + migrations.AddField( + model_name='serviceuser', + name='is_deleted', + field=models.BooleanField(default=False), + ), + ] diff --git a/backend/migrations/0031_auto_20230316_1713.py b/backend/migrations/0031_auto_20230316_1713.py new file mode 100644 index 00000000..e3bf2b87 --- /dev/null +++ b/backend/migrations/0031_auto_20230316_1713.py @@ -0,0 +1,25 @@ +# Generated by Django 3.2.15 on 2023-03-16 17:13 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('backend', '0030_serviceuser_is_deleted'), + ] + + operations = [ + migrations.AlterField( + model_name='annotation', + name='user', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='annotations', to=settings.AUTH_USER_MODEL), + ), + migrations.AlterField( + model_name='project', + name='owner', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='owns', to=settings.AUTH_USER_MODEL), + ), + ] diff --git a/backend/models.py b/backend/models.py index ee452c42..7f714177 100644 --- a/backend/models.py +++ b/backend/models.py @@ -12,11 +12,12 @@ from django.utils import timezone from django.db.models import Q, F, Count -from backend.utils.misc import get_value_from_key_path, insert_value_to_key_path +from backend.utils.misc import get_value_from_key_path, insert_value_to_key_path, generate_random_string from backend.utils.telemetry import TelemetrySender log = logging.getLogger(__name__) + class UserDocumentFormatPreference: JSON = 0 CSV = 1 @@ -26,6 +27,13 @@ class UserDocumentFormatPreference: (CSV, 'CSV') ) +def document_preference_str(pref: UserDocumentFormatPreference.USER_DOC_FORMAT_PREF) -> str: + if pref == UserDocumentFormatPreference.JSON: + return "JSON" + else: + return "CSV" + + class DocumentType: ANNOTATION = 0 TRAINING = 1 @@ -53,6 +61,8 @@ class ServiceUser(AbstractUser): receive_mail_notifications = models.BooleanField(default=True) doc_format_pref = models.IntegerField(choices=UserDocumentFormatPreference.USER_DOC_FORMAT_PREF, default=UserDocumentFormatPreference.JSON) + agreed_privacy_policy = models.BooleanField(default=False) + is_deleted = models.BooleanField(default=False) @property def has_active_project(self): @@ -117,6 +127,41 @@ def is_manager_or_above(self): else: return False + def clear_pending_annotations(self) -> None: + """ + Clear all of the user's pending annotation in the system to allow other annotators + to take up the task slot. + """ + pending_annotations = self.annotations.filter(status=Annotation.PENDING) + pending_annotations.delete() + + def delete_user_personal_information(self) -> None: + """ + Replace user's personal data with placeholder + """ + self.is_deleted = True + retry_limit = 1000 + retry_counter = 0 + while retry_counter < retry_limit: + random_suffix = generate_random_string(settings.DELETED_USER_USERNAME_HASH_LENGTH) + deleted_username = f"{settings.DELETED_USER_USERNAME_PREFIX}_{random_suffix}" + if not get_user_model().objects.filter(username=deleted_username).exists(): + break + retry_counter += 1 + + if retry_counter >= retry_limit: + raise Exception("Could not delete user, reached hash generation retries limit") + + self.username = deleted_username + self.first_name = settings.DELETED_USER_FIRSTNAME + self.last_name = settings.DELETED_USER_LASTNAME + self.email = f"{self.username}@{settings.DELETED_USER_EMAIL_DOMAIN}" + self.save() + + # Also clear all pending annotations + self.clear_pending_annotations() + + def default_document_input_preview(): return {"text": "

Some html text in bold.

Paragraph 2.

"} @@ -127,12 +172,12 @@ class Project(models.Model): Model to store annotation projects. """ name = models.TextField(default="New project") - uuid = models.UUIDField(primary_key = False, default = uuid.uuid4, editable = False) + uuid = models.UUIDField(primary_key=False, default=uuid.uuid4, editable=False) description = models.TextField(default="") annotator_guideline = models.TextField(default="") created = models.DateTimeField(default=timezone.now) configuration = models.JSONField(default=list) - owner = models.ForeignKey(get_user_model(), on_delete=models.SET_NULL, null=True, related_name="owns") + owner = models.ForeignKey(get_user_model(), on_delete=models.CASCADE, null=True, related_name="owns") annotations_per_doc = models.IntegerField(default=3) annotator_max_annotation = models.FloatField(default=0.6) # Allow annotators to reject document @@ -175,12 +220,11 @@ def get_project_export_field_names(cls): fields = Project.get_project_config_fields({"owner", "id", "created", "uuid"}) return [field.name for field in fields] - - def clone(self, new_name = None, clone_name_prefix="Copy of ", owner = None): + def clone(self, new_name=None, clone_name_prefix="Copy of ", owner=None): """ Clones the Project object, does not retain documents and annotator membership """ - exclude_fields = { "name", "owner", "id", "created", "uuid" } + exclude_fields = {"name", "owner", "id", "created", "uuid"} # Setting project name new_project_name = new_name if new_name is not None else "" @@ -197,7 +241,6 @@ def clone(self, new_name = None, clone_name_prefix="Copy of ", owner = None): new_project.save() return new_project - @property def num_documents(self): return self.documents.filter(doc_type=DocumentType.ANNOTATION).count() @@ -701,13 +744,15 @@ def check_project_complete(self): if settings.TELEMETRY_ON: self.send_telemetry(status="complete") - def send_telemetry(self, status:str): + def send_telemetry(self, status: str): """ Sends telemetry data for the project depending on the status. """ if settings.TELEMETRY_ON: ts = TelemetrySender(status=status, data=self.get_telemetry_stats()) ts.send() + else: + log.info(f"Telemetry is switched off. Not sending telemetry data for project {self.pk}.") def get_annotators_dict(self): return { @@ -985,7 +1030,7 @@ class Annotation(models.Model): (ABORTED, 'Aborted') ) - user = models.ForeignKey(get_user_model(), on_delete=models.SET_NULL, related_name="annotations", null=True) + user = models.ForeignKey(get_user_model(), on_delete=models.CASCADE, related_name="annotations", null=True) document = models.ForeignKey(Document, on_delete=models.CASCADE, related_name="annotations") _data = models.JSONField(default=dict) diff --git a/backend/rpc.py b/backend/rpc.py index d4981312..161fb364 100644 --- a/backend/rpc.py +++ b/backend/rpc.py @@ -3,6 +3,7 @@ import datetime import json +import os from urllib.parse import urljoin from django.conf import settings from django.contrib.auth import authenticate, get_user_model, login as djlogin, logout as djlogout @@ -25,8 +26,8 @@ from backend.errors import AuthError from backend.rpcserver import rpc_method, rpc_method_auth, rpc_method_manager, rpc_method_admin from backend.models import Project, Document, DocumentType, Annotation, AnnotatorProject, AnnotationChangeHistory, \ - UserDocumentFormatPreference -from backend.utils.misc import get_value_from_key_path, insert_value_to_key_path + UserDocumentFormatPreference, document_preference_str +from backend.utils.misc import get_value_from_key_path, insert_value_to_key_path, read_custom_document from backend.utils.serialize import ModelSerializer log = logging.getLogger(__name__) @@ -34,6 +35,40 @@ serializer = ModelSerializer() User = get_user_model() +##################################### +### Initilisation ### +##################################### +@rpc_method +def initialise(request): + """ + Provide the initial context information to initialise the Teamware app + + context_object: + user: + isAuthenticated: bool + isManager: bool + isAdmin: bool + configs: + docFormatPref: bool + global_configs: + allowUserDelete: bool + """ + context_object = { + "user": is_authenticated(request), + "configs": { + "docFormatPref": get_user_document_pref_from_request(request) + }, + "global_configs": { + "allowUserDelete": settings.ALLOW_USER_DELETE + } + } + return context_object + +def get_user_document_pref_from_request(request): + if request.user.is_authenticated: + return document_preference_str(request.user.doc_format_pref) + else: + return document_preference_str(UserDocumentFormatPreference.JSON) ##################################### ### Login/Logout/Register Methods ### @@ -75,10 +110,14 @@ def login(request, payload): user = authenticate(username=payload["username"], password=payload["password"]) if user is not None: + + if user.is_deleted: + raise AuthError("Cannot login with a deleted account") + djlogin(request, user) context["username"] = user.username context["isAuthenticated"] = user.is_authenticated - context["isManager"] = user.is_manager + context["isManager"] = user.is_manager or user.is_staff context["isAdmin"] = user.is_staff context["isActivated"] = user.is_activated return context @@ -98,9 +137,10 @@ def register(request, payload): username = payload.get("username") password = payload.get("password") email = payload.get("email") + agreed_privacy_policy = True if not get_user_model().objects.filter(username=username).exists(): - user = get_user_model().objects.create_user(username=username, password=password, email=email) + user = get_user_model().objects.create_user(username=username, password=password, email=email, agreed_privacy_policy=agreed_privacy_policy) _generate_user_activation(user) djlogin(request, user) context["username"] = payload["username"] @@ -307,10 +347,7 @@ def get_user_details(request): data["user_role"] = user_role # Convert doc preference to string - if user.doc_format_pref == UserDocumentFormatPreference.JSON: - data["doc_format_pref"] = "JSON" - else: - data["doc_format_pref"] = "CSV" + data["doc_format_pref"] = document_preference_str(user.doc_format_pref) return data @rpc_method_auth @@ -367,16 +404,27 @@ def get_user_annotations_in_project(request, project_id, current_page=1, page_si else: paginated_docs = user_annotated_docs - documents_out = [] for document in paginated_docs: annotations_list = [annotation.get_listing() for annotation in document.annotations.filter(user=user)] documents_out.append(document.get_listing(annotations_list)) - return {"items": documents_out, "total_count": total_count} +@rpc_method_auth +def user_delete_personal_information(request): + request.user.delete_user_personal_information() + + +@rpc_method_auth +def user_delete_account(request): + if settings.ALLOW_USER_DELETE: + request.user.delete() + else: + raise Exception("Teamware's current configuration does not allow user accounts to be deleted.") + + ################################## ### Project Management Methods ### ################################## @@ -417,7 +465,6 @@ def get_project(request, project_id): return out_proj - @rpc_method_manager def clone_project(request, project_id): with transaction.atomic(): @@ -669,7 +716,7 @@ def get_possible_annotators(request, proj_id): active_annotators = User.objects.filter(annotatorproject__status=AnnotatorProject.ACTIVE).values_list('id', flat=True) project_annotators = project.annotators.all().values_list('id', flat=True) # Do an exclude filter to remove annotator with the those ids - valid_annotators = User.objects.exclude(id__in=active_annotators).exclude(id__in=project_annotators) + valid_annotators = User.objects.filter(is_deleted=False).exclude(id__in=active_annotators).exclude(id__in=project_annotators) output = [serializer.serialize(annotator, {"id", "username", "email"}) for annotator in valid_annotators] return output @@ -941,6 +988,47 @@ def admin_update_user_password(request, username, password): user.save() +@rpc_method_admin +def admin_delete_user_personal_information(request, username): + user = User.objects.get(username=username) + user.delete_user_personal_information() + + +@rpc_method_admin +def admin_delete_user(request, username): + if settings.ALLOW_USER_DELETE: + user = User.objects.get(username=username) + user.delete() + else: + raise Exception("Teamware's current configuration does not allow the deleting of users") + + +################################## +### Privacy Policy/T&C Methods ### +################################## + +@rpc_method +def get_privacy_policy_details(request): + details = settings.PRIVACY_POLICY + + custom_docs = { + 'CUSTOM_PP_DOCUMENT': read_custom_document(settings.CUSTOM_PP_DOCUMENT_PATH) if os.path.isfile( + settings.CUSTOM_PP_DOCUMENT_PATH) else None, + 'CUSTOM_TC_DOCUMENT': read_custom_document(settings.CUSTOM_TC_DOCUMENT_PATH) if os.path.isfile( + settings.CUSTOM_TC_DOCUMENT_PATH) else None + } + + details.update(custom_docs) + + url = { + 'URL': request.headers['Host'] + } + + details.update(url) + + return details + + ############################### ### Utility Methods ### ############################### diff --git a/backend/signals.py b/backend/signals.py index 05e01ee4..5b24a401 100644 --- a/backend/signals.py +++ b/backend/signals.py @@ -2,13 +2,3 @@ from django.dispatch import receiver from backend.models import ServiceUser, Annotation -@receiver(pre_delete, sender=ServiceUser) -def user_delete_cleanup(sender, **kwargs): - """ Perform cleanup when a user is deleted""" - - delete_user = kwargs["instance"] - - # Remove all pending annotations - pending_annotations = delete_user.annotations.filter(status=Annotation.PENDING) - for pending in pending_annotations: - pending.delete() diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index f895a463..33a3fba1 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -1,6 +1,7 @@ import math from datetime import timedelta from django.db import models +from django.conf import settings from django.contrib.auth import get_user_model from django.test import TestCase, Client from django.utils import timezone @@ -23,6 +24,10 @@ def check_model_fields(self, model_class, field_name_types_dict): class TestUserModel(TestCase): + def test_agree_privacy_policy(self): + user = get_user_model().objects.create(username="test1", agreed_privacy_policy=True) + self.assertTrue(user.agreed_privacy_policy) + def test_document_association_check(self): user = get_user_model().objects.create(username="test1") user2 = get_user_model().objects.create(username="test2") @@ -88,41 +93,91 @@ def test_check_user_active_project(self): self.assertFalse(user.has_active_project) self.assertEqual(user.active_project, None) - def test_delete_user_with_annotations(self): - user = get_user_model().objects.create(username="test1") - project = Project.objects.create(owner=user) + +def create_each_annotation_status_for_user(user: get_user_model(), project: Project): + annotation_statuses = [Annotation.PENDING, + Annotation.COMPLETED, + Annotation.REJECTED, + Annotation.TIMED_OUT, + Annotation.ABORTED] + for annotation_status in annotation_statuses: Annotation.objects.create(user=user, document=Document.objects.create(project=project), + status=annotation_status) + + +class TestUserModelDeleteUser(TestCase): + + def setUp(self) -> None: + self.user = get_user_model().objects.create(username="test1", + first_name="TestFirstname", + last_name="TestLastname", + email="test@email.com") + self.project = Project.objects.create(owner=self.user) + create_each_annotation_status_for_user(self.user, self.project) + + self.user2 = get_user_model().objects.create(username="test2", + first_name="TestFirstname", + last_name="TestLastname", + email="test2@email.com") + project2 = Project.objects.create(owner=self.user2) + Annotation.objects.create(user=self.user, + document=Document.objects.create(project=project2), status=Annotation.PENDING) - Annotation.objects.create(user=user, - document=Document.objects.create(project=project), - status=Annotation.COMPLETED) - Annotation.objects.create(user=user, - document=Document.objects.create(project=project), - status=Annotation.REJECTED) - Annotation.objects.create(user=user, - document=Document.objects.create(project=project), - status=Annotation.TIMED_OUT) - Annotation.objects.create(user=user, - document=Document.objects.create(project=project), - status=Annotation.ABORTED) + create_each_annotation_status_for_user(user=self.user2, project=project2) + def test_clear_pending_annotations(self): + # Check that there's at least one pending annotation for the user + self.assertEqual(2, Annotation.objects.filter(status=Annotation.PENDING, user=self.user).count()) + + self.user.clear_pending_annotations() + + # No pending annotation for the user + self.assertEqual(0, Annotation.objects.filter(status=Annotation.PENDING, user=self.user).count()) + + def test_remove_user_personal_data(self): + """ + Tests deleting all personal information of the user from the system and replacing + all personally identifiable information with placeholders + """ + self.user.delete_user_personal_information() + self.user.refresh_from_db() + TestUserModelDeleteUser.check_user_personal_data_deleted(self, self.user) + + + + @staticmethod + def check_user_personal_data_deleted(test_obj, user): + # Make sure db is refereshed first user.refresh_from_db() - # 5 annotations for the user - self.assertEqual(5, user.annotations.all().count(), "User should have 5 annotations") - # 5 annotations in the entire system - self.assertEqual(5, Annotation.objects.all().count(), "Entire system should have 5 annotations") + # User should be marked as deleted + test_obj.assertTrue(user.is_deleted) - # Delete user, the pending annotation should be removed - user.delete() + # Deleted username is a combination of [DELETED_USER_USERNAME_PREFIX]_hashedvalues + test_obj.assertTrue(user.username.startswith(settings.DELETED_USER_USERNAME_PREFIX)) + # Deleted mail is a combination of [DELETED_USER_USERNAME_PREFIX]_hashedvalues@[DELETED_USER_EMAIL_DOMAIN] + test_obj.assertTrue(user.email.startswith(settings.DELETED_USER_USERNAME_PREFIX)) + test_obj.assertTrue(user.email.endswith(settings.DELETED_USER_EMAIL_DOMAIN)) + # First name and last name should be DELETED_USER_FIRSTNAME and DELETED_USER_LASTNAME + test_obj.assertEqual(user.first_name, settings.DELETED_USER_FIRSTNAME) + test_obj.assertEqual(user.last_name, settings.DELETED_USER_LASTNAME) + + # Removed user should not have pending annotations + test_obj.assertEqual(0, Annotation.objects.filter(status=Annotation.PENDING, user=user).count()) + + def test_delete_user(self): + user_id = self.user.pk + self.user.delete() + TestUserModelDeleteUser.check_user_is_deleted(self, user_id) + + @staticmethod + def check_user_is_deleted(test_obj, user_id): + test_obj.assertEqual(0, Annotation.objects.filter(user_id=user_id).count(), + "Deleted user should not have any annotations") + test_obj.assertEqual(0, Project.objects.filter(owner_id=user_id).count(), + "Deleted user should not have any projects") - self.assertEqual(4, Annotation.objects.all().count(), "Entire system should have 4 annotations") - self.assertEqual(0, Annotation.objects.filter(status=Annotation.PENDING).count(), "S") # No pending annotation - self.assertEqual(1, Annotation.objects.filter(status=Annotation.COMPLETED).count()) - self.assertEqual(1, Annotation.objects.filter(status=Annotation.REJECTED).count()) - self.assertEqual(1, Annotation.objects.filter(status=Annotation.TIMED_OUT).count()) - self.assertEqual(1, Annotation.objects.filter(status=Annotation.ABORTED).count()) class TestDocumentModel(ModelTestCase): @@ -1138,7 +1193,7 @@ def test_export_raw_anonymized(self): for document in self.project.documents.all(): doc_dict = document.get_doc_annotation_dict("raw", anonymize=True) - + for aset_key, aset_data in doc_dict["annotation_sets"].items(): self.assertTrue(isinstance(aset_data.get("name", None), int)) @@ -1146,7 +1201,7 @@ def test_export_raw_deanonymized(self): for document in self.project.documents.all(): doc_dict = document.get_doc_annotation_dict("raw", anonymize=False) - + for aset_key, aset_data in doc_dict["annotation_sets"].items(): self.assertTrue(isinstance(aset_data.get("name", None), str)) diff --git a/backend/tests/test_rpc_endpoints.py b/backend/tests/test_rpc_endpoints.py index df741d17..1933acbf 100644 --- a/backend/tests/test_rpc_endpoints.py +++ b/backend/tests/test_rpc_endpoints.py @@ -19,10 +19,11 @@ get_user_annotations_in_project, add_project_test_document, add_project_training_document, \ get_project_training_documents, get_project_test_documents, project_annotator_allow_annotation, \ annotator_leave_project, login, change_annotation, delete_annotation_change_history, get_annotation_task_with_id, \ - set_user_document_format_preference + set_user_document_format_preference, initialise, is_authenticated, user_delete_personal_information, \ + user_delete_account, admin_delete_user_personal_information, admin_delete_user from backend.rpcserver import rpc_method from backend.errors import AuthError - +from backend.tests.test_models import create_each_annotation_status_for_user, TestUserModelDeleteUser from backend.tests.test_rpc_server import TestEndpoint @@ -77,6 +78,54 @@ def test_user_auth(self): content_type="application/json") self.assertEqual(response.status_code, 200) +class TestInitialise(TestEndpoint): + + def test_initialise(self): + context_object = initialise(self.get_request()) + self.assertTrue("user" in context_object) + self.assertTrue("configs" in context_object) + self.assertTrue("docFormatPref" in context_object["configs"]) + self.assertTrue("global_configs" in context_object) + self.assertTrue("allowUserDelete" in context_object["global_configs"]) + +class TestIsAuthenticated(TestEndpoint): + + def test_is_authenticated_anonymous(self): + context = is_authenticated(self.get_request()) + self.assertFalse(context["isAuthenticated"]) + self.assertFalse(context["isManager"]) + self.assertFalse(context["isAdmin"]) + + def test_is_authenticated_annotator(self): + user = self.get_default_user() + user.is_staff = False + user.is_manager = False + user.save() + context = is_authenticated(self.get_loggedin_request()) + self.assertTrue(context["isAuthenticated"]) + self.assertFalse(context["isManager"]) + self.assertFalse(context["isAdmin"]) + + def test_is_authenticated_manager(self): + user = self.get_default_user() + user.is_staff = False + user.is_manager = True + user.save() + context = is_authenticated(self.get_loggedin_request()) + self.assertTrue(context["isAuthenticated"]) + self.assertTrue(context["isManager"]) + self.assertFalse(context["isAdmin"]) + + def test_is_authenticated_admin(self): + user = self.get_default_user() + user.is_staff = True + user.is_manager = False + user.save() + context = is_authenticated(self.get_loggedin_request()) + self.assertTrue(context["isAuthenticated"]) + self.assertTrue(context["isManager"]) + self.assertTrue(context["isAdmin"]) + class TestAppCreatedUserAccountsCannotLogin(TestEndpoint): def test_app_created_user_accounts_cannot_login(self): @@ -93,6 +142,23 @@ def test_app_created_user_accounts_cannot_login(self): with self.assertRaises(AuthError, msg="Should raise an error if logging in with blank as password"): login(self.get_request(), {"username": "doesnotexist", "password": ""}) +class TestDeletedUserAccountCannotLogin(TestEndpoint): + + def test_deleted_user_account_cannot_login(self): + """ + Ensures that user accounts that opts to have their personal information removed from the system + but not wiped cannot login again. + """ + + username = "deleted" + password = "test1password" + + get_user_model().objects.create_user(username=username, + password=password, + is_deleted=True) + with self.assertRaises(AuthError, msg="Should raise an error if trying to login with a deleted account"): + login(self.get_request(), {"username": username, "password": password}) + class TestUserRegistration(TestEndpoint): @@ -181,8 +247,6 @@ def test_user_password_reset(self): # Should now generate a password reset token self.call_rpc(self.get_client(), "generate_password_reset", test_user.username) - - # Check that token generaet is valid test_user.refresh_from_db() self.assertTrue(len(test_user.reset_password_token) > settings.ACTIVATION_TOKEN_LENGTH) @@ -211,6 +275,50 @@ def test_user_password_reset(self): self.assertTrue(test_user.reset_password_token is None) self.assertTrue(test_user.reset_password_token_expire is None) + +class TestRPCDeleteUser(TestEndpoint): + + def setUp(self) -> None: + self.user = get_user_model().objects.create(username="test1", + first_name="TestFirstname", + last_name="TestLastname", + email="test@email.com") + self.project = Project.objects.create(owner=self.user) + create_each_annotation_status_for_user(self.user, self.project) + + self.user2 = get_user_model().objects.create(username="test2", + first_name="TestFirstname", + last_name="TestLastname", + email="test2@email.com") + project2 = Project.objects.create(owner=self.user2) + Annotation.objects.create(user=self.user, + document=Document.objects.create(project=project2), + status=Annotation.PENDING) + create_each_annotation_status_for_user(user=self.user2, project=project2) + + def test_user_delete_personal_information(self): + request = self.get_request() + request.user = self.user + user_delete_personal_information(request) + TestUserModelDeleteUser.check_user_personal_data_deleted(self, self.user) + + def test_user_delete_account(self): + request = self.get_request() + request.user = self.user + user_id = self.user.pk + user_delete_account(request) + TestUserModelDeleteUser.check_user_is_deleted(self, user_id) + + def test_admin_delete_user_personal_information(self): + admin_delete_user_personal_information(self.get_loggedin_request(), self.user.username) + TestUserModelDeleteUser.check_user_personal_data_deleted(self, self.user) + + def test_admin_delete_user(self): + user_id = self.user.pk + admin_delete_user(self.get_loggedin_request(), self.user.username) + TestUserModelDeleteUser.check_user_is_deleted(self, user_id) + + class TestUserConfig(TestEndpoint): def test_change_password(self): @@ -725,6 +833,7 @@ def test_list_possible_annotators(self): ann1 = get_user_model().objects.create(username="ann1") ann2 = get_user_model().objects.create(username="ann2") ann3 = get_user_model().objects.create(username="ann3") + ann4 = get_user_model().objects.create(username="ann4", is_deleted=True) proj = Project.objects.create(owner=user) proj2 = Project.objects.create(owner=user) @@ -732,9 +841,9 @@ def test_list_possible_annotators(self): # Listing all annotators for project 1 and 2 without anyone added to project possible_annotators = get_possible_annotators(self.get_loggedin_request(), proj_id=proj.pk) - self.assertEqual(4, len(possible_annotators), "Should list all users") + self.assertEqual(4, len(possible_annotators), "Should list all users without deleted user") possible_annotators = get_possible_annotators(self.get_loggedin_request(), proj_id=proj2.pk) - self.assertEqual(4, len(possible_annotators), "Should list all users") + self.assertEqual(4, len(possible_annotators), "Should list all users without deleted user") project_annotators = get_project_annotators(self.get_loggedin_request(), proj_id=proj.pk) self.assertEqual(0, len(project_annotators)) diff --git a/backend/tests/test_telemetry.py b/backend/tests/test_telemetry.py index 38dfec37..a565a6f5 100644 --- a/backend/tests/test_telemetry.py +++ b/backend/tests/test_telemetry.py @@ -21,7 +21,7 @@ def test_telemetry_sender(self, mocker): proj = Project.objects.first() - ts = TelemetrySender("completed", {}) + ts = TelemetrySender("completed", { 'uuid': 'mock-uuid'}) mocker.post(ts.url, status_code=201) # set up mocker for http post request ts.send() diff --git a/backend/utils/misc.py b/backend/utils/misc.py index 7cd91153..8e916764 100644 --- a/backend/utils/misc.py +++ b/backend/utils/misc.py @@ -1,3 +1,6 @@ +import string +import random + def get_value_from_key_path(obj_dict, key_path, delimiter="."): """ Gets value from a dictionary following a delimited key_path. Does not work for path with array elements. @@ -39,3 +42,24 @@ def insert_value_to_key_path(obj_dict, key_path, value, delimiter="."): return True return False + + +def read_custom_document(path): + """ + Reads in a text file and returns as a string. + Primarily used for reading in custom privacy policy and/or terms & conditions documents. + """ + with open(path) as file: + doc_str = file.read() + return doc_str + + +def generate_random_string(length) -> string: + """ + Generates random ascii string of lowercase, uppercase and digits of length + + @param length Length of the generated random string + @return Generated random string + """ + use_characters = string.ascii_letters + string.digits + return ''.join([random.choice(use_characters) for i in range(length)]) diff --git a/backend/utils/telemetry.py b/backend/utils/telemetry.py index 7632dac7..59246f73 100644 --- a/backend/utils/telemetry.py +++ b/backend/utils/telemetry.py @@ -1,9 +1,11 @@ import json +import logging from threading import Thread import requests from urllib.parse import urljoin from django.conf import settings +log = logging.getLogger(__name__) class TelemetrySender: def __init__(self, status: str, data: dict) -> None: @@ -20,7 +22,11 @@ def send(self): if settings.TELEMETRY_ON: self.thread = Thread(target=self._post_request) self.thread.run() + else: + log.info(f"Telemetry is switched off. Not sending telemetry data for project {self.data['uuid']}.") def _post_request(self): - r = requests.post(self.url, data=json.dumps(self.data)) + log.info(f"Sending telemetry data for project {self.data['uuid']} to {self.url}.") + r = requests.post(self.url, json=self.data) self.http_status_code = r.status_code + log.info(f"{self.http_status_code}: {r.text}") diff --git a/build-images.sh b/build-images.sh index 771f4f68..093b7f4c 100755 --- a/build-images.sh +++ b/build-images.sh @@ -7,6 +7,6 @@ set -o allexport source .env set +o allexport -docker build -t $IMAGE_REGISTRY$MAIN_IMAGE:$IMAGE_TAG --target backend . +docker buildx build $BUILDX_ARGS_BACKEND --load -t $IMAGE_REGISTRY$MAIN_IMAGE:$IMAGE_TAG --target backend . -docker build -t $IMAGE_REGISTRY$STATIC_IMAGE:$IMAGE_TAG --target frontend . +docker buildx build $BUILDX_ARGS_FRONTEND --load -t $IMAGE_REGISTRY$STATIC_IMAGE:$IMAGE_TAG --target frontend . diff --git a/charts/README.md b/charts/README.md deleted file mode 100644 index f208ec69..00000000 --- a/charts/README.md +++ /dev/null @@ -1,84 +0,0 @@ -Helm Chart for Teamware -======================= - -This directory contains a Helm chart to deploy GATE Teamware to a Kubernetes cluster. The chart has been developed against Kubernetes 1.23 and may not be compatible with earlier versions, and requires Helm version 3.7 or later. - -## Prerequisites - -In order to run under Kubernetes there are a few prerequisites that must be satisfied first. Most important, Kubernetes clusters cannot generally work with locally built images, so the `backend` and `static` images must be pushed to a remote Docker registry, such as Docker Hub or `ghcr.io`, and the registry name passed to helm when installing or upgrading the chart. Secondly, the chart relies on a pair of pre-existing "secrets" in the Kubernetes namespace where the chart will be installed, holding the (ideally randomly generated) passwords for the postgresql superuser and app user, and the django `SECRET_KEY` value. The [kubernetes-secret-generator](https://github.com/mittwald/kubernetes-secret-generator) tool is useful for this. - -To set up a new installation of teamware: - -- Create a suitable namespace in the cluster, and install any necessary `imagePullSecrets` - this may require additional admin privileges -- Create the random secrets described above - if your cluster has `kubernetes-secret-generator` installed then thic can be done using `kubectl create -f initial-secrets.yaml -n {namespace}` -- Create a suitable YAML file to override any defaults from `gate-teamware/values.yaml`. - -Things you will commonly need to override include: - -- `hostName` - set this to the fully-qualified public hostname of the teamware installation, e.g. `annotate.gate.ac.uk` -- `publicUrl` - set this to the fully qualified _public_ base URL of the site, including the protocol and port (if not 80/443) but no trailing slash. The default is `https://{hostName}` so you should only need to override if your app is not served over HTTPS, or if it uses a non-standard port number. -- `ingress.className` - ingress class to use, if the cluster does not have a default or if you want to use a different class from the default one. -- `ingress.tls.secret` - name of the secret holding the TLS certificate for the configured `hostName`. Whether this is required or optional depends on the cluster and its configured ingress controller, e.g. the GATE cluster is set up to use a `*.gate.ac.uk` wildcard certificate for ingresses that do not specify their own, so on that cluster if the `hostName` matches that wildcard then a separate secret is not required. -- `email` settings to be able to send registration and password reminder emails - - `adminAddress` - email address of the administrator, used as the "from" address on generated emails - - `backend` - "smtp" to send mail via an SMTP server, "gmail" to use the GMail API. - - for the "smtp" backend: - - `host` and `port` (default 587) - - `security` if the server requires an encrypted connection - either "tls" for STARTTLS on a regular port, or "ssl" for immediate TLS-on-connect as often used on port 465. - - `user` and `passwordSecret` if the server requires authentication - `user` is the actual login username, `passwordSecret` is the name of a pre-existing Kubernetes secret containing a "password" key - - `clientCertSecret` if the server requires TLS client certificate authentication. This is the name of a standard Kubernetes "tls" type secret which contains `tls.key` and `tls.crt` entries. - - for the "gmail" backend - - `clientId` - the OAuth client ID for the GMail API - - `secretName` - Kubernetes secret containing entries for "client-secret" (the OAuth client secret) and "refresh-token" (the authenticated refresh token) -- `postgresql.primary.persistence.size` (default "8Gi") - requested size for the persistent volume holding PostgreSQL data -- `postgresql.primary.persistence.storageClass` - storage class for the PostgreSQL data volume. This is required if your cluster does not have a default StorageClass configured - - alternatively, set `primary.persistence.existingClaim` to use an existing PVC rather than letting the StatefulSet create its own. -- `migrations.run` - set this to `true` in order to run the Django database migrations after the chart is installed. The backend pods _do not_ run migrations at startup, as this is unsafe if there are multiple replicas or if autoscaling is in use, what this setting does is to run a one-off `Job` that just does the migrations and then exits. -- `backend.replicaCount` (default 1) - the number of replicas of the Django container to run. Alternatively you can set `backend.autoscaling.enabled` to `true` for auto-scaling based on CPU usage -- `staticFiles.replicaCount` (default 1) - the same for the static files nginx, though this is highly unlikely to need more than one replica as it's a simple static file server -- you can also set `resources`, `nodeSelector` and/or `tolerations` if required, under both the `backend` and `staticFiles` sections - -The images to be run are specified in three parts, `imageRegistry` is the registry prefix (e.g. `registry.example.com/teamware/`) which _must_ end with a slash, then `backend` and `staticFiles` have `image.repository` for the image name (default "teamware-backend" and "teamware-static" respectively) and `image.tag` for the tag, which defaults to match the chart version number, plus `pullPolicy` (default "IfNotPresent") and `pullSecrets` (if you are using a private registry whose credentials are not already configured on the default ServiceAccount for this namespace). - -The chart also supports running regular backups of the database to S3 (or a compatible storage system), these can be configured using the settings under the `backup` section, see `gate-teamware/values.yaml` for more details. - -## Install/upgrade - -With the configured values file in place, installing or upgrading the chart uses the standard Helm command: - -``` -helm upgrade --install gate-teamware ./gate-teamware/ \ - --namespace {ns} --values {override-values-file} -``` - -e.g. - -``` -helm upgrade --install gate-teamware ./gate-teamware/ \ - --namespace teamware-prod --values prod-values.yaml -``` - -## Changelog - -### Version 0.2.0 - -**Breaking changes** - -- default postgresql database name changed from `annotations_db` to `teamware_db` - if you are upgrading an existing installation rather than installing fresh you must either: - - explicitly override `postgresql.auth.database=annotations_db` in order to remain compatible with your existing database, or - - ensure you have a recent backup of the database, uninstall the chart completely, delete the old postgresql PV and PVC, do a fresh install of the chart to create the database under its new name, then restore the most recent backup to the new `teamware_db` database. - - The [postgres-restore-s3 tool](https://github.com/schickling/dockerfiles/tree/master/postgres-restore-s3) may be useful for this, but the chart cannot configure this automatically as it requires credentials that are able to _read_ from your backup bucket, and ideally the credentials provisioned for the backup CronJob should only provide _write_ access. - -### Version 0.1.1 - -No breaking changes. - -Minor changes: - -- Reduced log verbosity for the static files pod by not logging k8s health check probes. - -### Version 0.1.0 - -Initial Helm chart. - - diff --git a/charts/gate-teamware/.gitignore b/charts/gate-teamware/.gitignore deleted file mode 100644 index ebf1d3dc..00000000 --- a/charts/gate-teamware/.gitignore +++ /dev/null @@ -1 +0,0 @@ -charts diff --git a/charts/gate-teamware/.helmignore b/charts/gate-teamware/.helmignore deleted file mode 100644 index 0e8a0eb3..00000000 --- a/charts/gate-teamware/.helmignore +++ /dev/null @@ -1,23 +0,0 @@ -# Patterns to ignore when building packages. -# This supports shell glob matching, relative path matching, and -# negation (prefixed with !). Only one pattern per line. -.DS_Store -# Common VCS dirs -.git/ -.gitignore -.bzr/ -.bzrignore -.hg/ -.hgignore -.svn/ -# Common backup files -*.swp -*.bak -*.tmp -*.orig -*~ -# Various IDEs -.project -.idea/ -*.tmproj -.vscode/ diff --git a/charts/gate-teamware/Chart.lock b/charts/gate-teamware/Chart.lock deleted file mode 100644 index d7888c89..00000000 --- a/charts/gate-teamware/Chart.lock +++ /dev/null @@ -1,6 +0,0 @@ -dependencies: -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 11.1.28 -digest: sha256:12b37d57de36c36343ad00f2be4c51d5ad0272c6384578a2ae09e55bc57f50bc -generated: "2022-05-10T19:51:48.776542+01:00" diff --git a/charts/gate-teamware/Chart.yaml b/charts/gate-teamware/Chart.yaml deleted file mode 100644 index ded389b5..00000000 --- a/charts/gate-teamware/Chart.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: v2 -name: gate-teamware -description: A Helm chart for Kubernetes - -# A chart can be either an 'application' or a 'library' chart. -# -# Application charts are a collection of templates that can be packaged into versioned archives -# to be deployed. -# -# Library charts provide useful utilities or functions for the chart developer. They're included as -# a dependency of application charts to inject those utilities and functions into the rendering -# pipeline. Library charts do not define any templates and therefore cannot be deployed. -type: application - -# This is the chart version. This version number should be incremented each time you make changes -# to the chart and its templates, including the app version. -# Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.2.1 - -# This is the version number of the application being deployed. This version number should be -# incremented each time you make changes to the application. Versions are not expected to -# follow Semantic Versioning. They should reflect the version the application is using. -# It is recommended to use it with quotes. -appVersion: "1.0.0" - -dependencies: -- name: postgresql - repository: https://charts.bitnami.com/bitnami - version: 11.1.28 # chart version 11.1.28 wraps Postgresql version 14 diff --git a/charts/gate-teamware/templates/_helpers.tpl b/charts/gate-teamware/templates/_helpers.tpl deleted file mode 100644 index 92a4076e..00000000 --- a/charts/gate-teamware/templates/_helpers.tpl +++ /dev/null @@ -1,73 +0,0 @@ -{{/* -Expand the name of the chart. -*/}} -{{- define "gate-teamware.name" -}} -{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Create a default fully qualified app name. -We truncate at 55 chars because some Kubernetes name fields are limited to 63 (by the DNS naming spec), so 55 gives us space to add "-backend". -If release name contains chart name it will be used as a full name. -*/}} -{{- define "gate-teamware.fullname" -}} -{{- if .Values.fullnameOverride }} -{{- .Values.fullnameOverride | trunc 55 | trimSuffix "-" }} -{{- else }} -{{- $name := default .Chart.Name .Values.nameOverride }} -{{- if contains $name .Release.Name }} -{{- .Release.Name | trunc 55 | trimSuffix "-" }} -{{- else }} -{{- printf "%s-%s" .Release.Name $name | trunc 55 | trimSuffix "-" }} -{{- end }} -{{- end }} -{{- end }} - -{{/* -Create chart name and version as used by the chart label. -*/}} -{{- define "gate-teamware.chart" -}} -{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} -{{- end }} - -{{/* -Common labels -*/}} -{{- define "gate-teamware.labels" -}} -helm.sh/chart: {{ include "gate-teamware.chart" . }} -{{- if .Chart.AppVersion }} -app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} -{{- end }} -app.kubernetes.io/managed-by: {{ .Release.Service }} -{{- end }} - -{{/* -Selector labels -*/}} -{{- define "gate-teamware.selectorLabels" -}} -app.kubernetes.io/name: {{ include "gate-teamware.name" . }} -app.kubernetes.io/instance: {{ .Release.Name }} -{{- end }} - -{{/* -Create the name of the service account to use -*/}} -{{- define "gate-teamware.serviceAccountName" -}} -{{- if .Values.serviceAccount.create }} -{{- default (include "gate-teamware.fullname" .) .Values.serviceAccount.name }} -{{- else }} -{{- default "default" .Values.serviceAccount.name }} -{{- end }} -{{- end }} - -{{/* -Get the name of the secret containing the superuser initial password, either a -specified existing secret or one generated by this chart. -*/}} -{{- define "gate-teamware.superuserSecret" -}} - {{- if .Values.superuser.existingSecret -}} - {{- .Values.superuser.existingSecret -}} - {{- else -}} - {{- printf "%s-superuser" (include "gate-teamware.fullname" .) -}} - {{- end -}} -{{- end }} diff --git a/charts/gate-teamware/templates/cronjob-backup.yaml b/charts/gate-teamware/templates/cronjob-backup.yaml deleted file mode 100644 index e7639431..00000000 --- a/charts/gate-teamware/templates/cronjob-backup.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{{- range $name, $cron := .Values.backup.schedule }} -apiVersion: batch/v1 -kind: CronJob -metadata: - name: {{ printf "backup-%s-%s" $name (include "gate-teamware.fullname" $) | trunc 52 | trimSuffix "-" }} - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: backup-{{ $name }} -spec: - schedule: {{ $cron | quote }} - {{- if $.Values.backup.concurrencyPolicy }} - concurrencyPolicy: {{ $.Values.backup.concurrencyPolicy }} - {{- end }} - {{- if ne $.Values.backup.failedJobsHistoryLimit "" }} - failedJobsHistoryLimit: {{ $.Values.backup.failedJobsHistoryLimit }} - {{- end }} - {{- if ne $.Values.backup.successfulJobsHistoryLimit "" }} - successfulJobsHistoryLimit: {{ $.Values.backup.successfulJobsHistoryLimit }} - {{- end }} - jobTemplate: - spec: - template: - metadata: - labels: - {{- include "gate-teamware.labels" $ | nindent 12 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 12 }} - app.kubernetes.io/component: backup-{{ $name }} - spec: - serviceAccountName: {{ include "gate-teamware.serviceAccountName" $ }} - restartPolicy: Never - containers: - - name: backup - image: {{ $.Values.backup.image | quote }} - imagePullPolicy: {{ $.Values.backup.pullPolicy }} - env: - - name: S3_ACCESS_KEY_ID - valueFrom: - secretKeyRef: - name: {{ $.Values.backup.credentialsSecret | required "backup.credentialsSecret is required" | quote }} - key: access-key-id - - name: S3_SECRET_ACCESS_KEY - valueFrom: - secretKeyRef: - name: {{ $.Values.backup.credentialsSecret | required "backup.credentialsSecret is required" | quote }} - key: secret-key - - name: S3_BUCKET - value: {{ $.Values.backup.bucketName | required "backup.bucketName is required" | quote }} - - name: S3_PREFIX - value: {{ with $.Values.backup.keyPrefix }}{{ . }}/{{ end }}{{ $name }} - {{- if $.Values.backup.endpoint }} - - name: S3_ENDPOINT - value: {{ $.Values.backup.endpoint | quote }} - {{- end }} - {{- if $.Values.backup.region }} - - name: S3_REGION - value: {{ $.Values.backup.region | quote }} - {{- end }} - - name: POSTGRES_HOST - value: {{ include "postgresql.primary.fullname" $.Subcharts.postgresql | quote }} - - name: POSTGRES_PORT - value: {{ include "postgresql.service.port" $.Subcharts.postgresql | quote }} - - name: POSTGRES_DATABASE - value: {{ $.Values.postgresql.auth.database | quote }} - - name: POSTGRES_USER - value: {{ $.Values.postgresql.auth.username | quote }} - - name: POSTGRES_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "postgresql.secretName" $.Subcharts.postgresql }} - key: password - ---- -{{- end }}{{/* range over schedule */}} diff --git a/charts/gate-teamware/templates/deployment-backend.yaml b/charts/gate-teamware/templates/deployment-backend.yaml deleted file mode 100644 index 4a0fc43e..00000000 --- a/charts/gate-teamware/templates/deployment-backend.yaml +++ /dev/null @@ -1,179 +0,0 @@ -{{- with .Values.backend }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "gate-teamware.fullname" $ }}-backend - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: backend -spec: - {{- if not .autoscaling.enabled }} - replicas: {{ .replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "gate-teamware.selectorLabels" $ | nindent 6 }} - app.kubernetes.io/component: backend - template: - metadata: - {{- with .podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "gate-teamware.selectorLabels" $ | nindent 8 }} - app.kubernetes.io/component: backend - spec: - {{- with .imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gate-teamware.serviceAccountName" $ }} - securityContext: - {{- toYaml .podSecurityContext | nindent 8 }} - containers: - - name: {{ $.Chart.Name }} - securityContext: - {{- toYaml .securityContext | nindent 12 }} - image: "{{ $.Values.imageRegistry }}{{ .image.repository }}:{{ .image.tag | default $.Chart.AppVersion }}" - imagePullPolicy: {{ .image.pullPolicy }} - ports: - - name: http - containerPort: 8000 - protocol: TCP - {{- with .extraArgs }} - args: {{- toYaml . | nindent 10 }} - {{- end }} - env: - - name: DJANGO_SETTINGS_MODULE - value: teamware.settings.deployment - - name: DJANGO_ALLOWED_HOSTS - value: {{ $.Values.hostName | quote }} - - name: DJANGO_APP_URL - {{- if $.Values.publicUrl }} - value: {{ $.Values.publicUrl | quote }} - {{- else }} - value: {{ printf "https://%s" $.Values.hostName | quote }} - {{- end }} - # Don't run migrations at startup - - name: TEAMWARE_SKIP_SETUP - value: "true" - - name: DJANGO_DB_NAME - value: {{ $.Values.postgresql.auth.database | quote }} - - name: DB_HOST - value: {{ include "postgresql.primary.fullname" $.Subcharts.postgresql | quote }} - - name: DB_PORT - value: {{ include "postgresql.service.port" $.Subcharts.postgresql | quote }} - - name: DB_USERNAME - value: {{ $.Values.postgresql.auth.username | quote }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "postgresql.secretName" $.Subcharts.postgresql }} - key: password - - name: DJANGO_SECRET_KEY - valueFrom: - secretKeyRef: - name: {{ .djangoSecret | required "backend.djangoSecret must be set to the name of an existing secret with an entry 'secret-key' and a value consisting of 40+ random printable characters" }} - key: secret-key - {{- with $.Values.email }} - {{- if .backend }} - - name: DJANGO_ADMIN_EMAIL - value: {{ .adminAddress | required "email.adminAddress is required if email sending is enabled" | quote }} - {{- end }} - {{- if .activationEnabled }} - - name: DJANGO_ACTIVATION_WITH_EMAIL - value: "true" - {{- end }} - {{- if eq .backend "smtp" }} - {{- /* SMTP email settings */}} - - name: DJANGO_EMAIL_BACKEND - value: "django.core.mail.backends.smtp.EmailBackend" - - name: DJANGO_EMAIL_HOST - value: {{ .smtp.host | required "email.smtp.host must be specified if email.backend is smtp" | quote }} - - name: DJANGO_EMAIL_PORT - value: {{ .smtp.port | quote }} - {{- if .smtp.user }} - - name: DJANGO_EMAIL_HOST_USER - value: {{ .smtp.user | quote }} - - name: DJANGO_EMAIL_HOST_PASSWORD - valueFrom: - secretKeyRef: - name: {{ .smtp.passwordSecret | required "smtp.passwordSecret is required if smtp.user is set" | quote }} - key: password - {{- end }} - {{- if .smtp.security }} - - name: DJANGO_EMAIL_SECURITY - value: {{ .smtp.security | quote }} - {{- if .smtp.clientCertSecret }} - {{- /* These paths are fixed - the actual key and cert are mounted from the secret as a volume */}} - - name: DJANGO_EMAIL_CLIENT_KEY - value: /email-client-cert/tls.key - - name: DJANGO_EMAIL_CLIENT_CERTIFICATE - value: /email-client-cert/tls.crt - {{- end }}{{/* if clientCertSecret */}} - {{- end }}{{/* if security */}} - {{- else if eq .backend "gmail" }} - {{- /* GMail API email settings */}} - - name: DJANGO_EMAIL_BACKEND - value: "gmailapi_backend.mail.GmailBackend" - - name: DJANGO_GMAIL_API_CLIENT_ID - value: {{ .gmail.clientId | required "gmail.clientId required when email.backend is gmail" | quote }} - - name: DJANGO_GMAIL_API_CLIENT_SECRET - valueFrom: - secretKeyRef: - name: {{ .gmail.secretName | required "gmail.secretName required when email.backend is gmail" | quote }} - key: client-secret - - name: DJANGO_GMAIL_API_REFRESH_TOKEN - valueFrom: - secretKeyRef: - name: {{ .gmail.secretName | quote }} - key: refresh-token - {{- end }}{{/* if backend gmail */}} - {{- end }}{{/* with .Values.email */}} - {{- with .extraEnv }} - {{- toYaml . | nindent 10 }} - {{- end }} - {{- if and $.Values.email.smtp.security $.Values.email.smtp.clientCertSecret }} - volumeMounts: - - name: email-client-cert - mountPath: /email-client-cert - readOnly: true - {{- end }}{{/* if smtp client cert */}} - livenessProbe: - httpGet: - path: / - port: http - httpHeaders: - - name: Host - value: {{ $.Values.hostName | quote }} - readinessProbe: - httpGet: - path: / - port: http - httpHeaders: - - name: Host - value: {{ $.Values.hostName | quote }} - resources: - {{- toYaml .resources | nindent 12 }} - {{- if and $.Values.email.smtp.security $.Values.email.smtp.clientCertSecret }} - volumes: - - name: email-client-cert - secret: - secretName: {{ $.Values.email.smtp.clientCertSecret | quote }} - {{- end }}{{/* if smtp client cert */}} - {{- with .nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -{{ end }} diff --git a/charts/gate-teamware/templates/deployment-static.yaml b/charts/gate-teamware/templates/deployment-static.yaml deleted file mode 100644 index 95ae9284..00000000 --- a/charts/gate-teamware/templates/deployment-static.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{{- with .Values.staticFiles }} -apiVersion: apps/v1 -kind: Deployment -metadata: - name: {{ include "gate-teamware.fullname" $ }}-static - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: static-files -spec: - {{- if not .autoscaling.enabled }} - replicas: {{ .replicaCount }} - {{- end }} - selector: - matchLabels: - {{- include "gate-teamware.selectorLabels" $ | nindent 6 }} - app.kubernetes.io/component: static-files - template: - metadata: - {{- with .podAnnotations }} - annotations: - {{- toYaml . | nindent 8 }} - {{- end }} - labels: - {{- include "gate-teamware.selectorLabels" $ | nindent 8 }} - app.kubernetes.io/component: static-files - spec: - {{- with .imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gate-teamware.serviceAccountName" $ }} - securityContext: - {{- toYaml .podSecurityContext | nindent 8 }} - containers: - - name: {{ $.Chart.Name }} - securityContext: - {{- toYaml .securityContext | nindent 12 }} - image: "{{ $.Values.imageRegistry }}{{ .image.repository }}:{{ .image.tag | default $.Chart.AppVersion }}" - imagePullPolicy: {{ .image.pullPolicy }} - env: - - name: HEALTH_PORT - value: {{ .healthPort | quote }} - ports: - - name: http - containerPort: 80 - protocol: TCP - - name: health - containerPort: {{ .healthPort }} - protocol: TCP - livenessProbe: - httpGet: - path: /healthz - port: health - readinessProbe: - httpGet: - path: /healthz - port: health - resources: - {{- toYaml .resources | nindent 12 }} - {{- with .nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} ---- -{{ end }} diff --git a/charts/gate-teamware/templates/hpa-backend.yaml b/charts/gate-teamware/templates/hpa-backend.yaml deleted file mode 100644 index 47681212..00000000 --- a/charts/gate-teamware/templates/hpa-backend.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- with .Values.staticFiles }} -{{- if .autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "gate-teamware.fullname" $ }}-backend - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: backend -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "gate-teamware.fullname" $ }}-backend - minReplicas: {{ .autoscaling.minReplicas }} - maxReplicas: {{ .autoscaling.maxReplicas }} - metrics: - {{- if .autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/gate-teamware/templates/hpa-static.yaml b/charts/gate-teamware/templates/hpa-static.yaml deleted file mode 100644 index 18fe2f39..00000000 --- a/charts/gate-teamware/templates/hpa-static.yaml +++ /dev/null @@ -1,32 +0,0 @@ -{{- with .Values.staticFiles }} -{{- if .autoscaling.enabled }} -apiVersion: autoscaling/v2beta1 -kind: HorizontalPodAutoscaler -metadata: - name: {{ include "gate-teamware.fullname" $ }}-static - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/name: static-files -spec: - scaleTargetRef: - apiVersion: apps/v1 - kind: Deployment - name: {{ include "gate-teamware.fullname" $ }}-static - minReplicas: {{ .autoscaling.minReplicas }} - maxReplicas: {{ .autoscaling.maxReplicas }} - metrics: - {{- if .autoscaling.targetCPUUtilizationPercentage }} - - type: Resource - resource: - name: cpu - targetAverageUtilization: {{ .autoscaling.targetCPUUtilizationPercentage }} - {{- end }} - {{- if .autoscaling.targetMemoryUtilizationPercentage }} - - type: Resource - resource: - name: memory - targetAverageUtilization: {{ .autoscaling.targetMemoryUtilizationPercentage }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/gate-teamware/templates/ingress.yaml b/charts/gate-teamware/templates/ingress.yaml deleted file mode 100644 index ba551c36..00000000 --- a/charts/gate-teamware/templates/ingress.yaml +++ /dev/null @@ -1,42 +0,0 @@ -{{- $fullName := include "gate-teamware.fullname" . -}} -apiVersion: networking.k8s.io/v1 -kind: Ingress -metadata: - name: {{ $fullName }} - labels: - {{- include "gate-teamware.labels" . | nindent 4 }} - {{- include "gate-teamware.selectorLabels" . | nindent 4 }} - {{- with .Values.ingress.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -spec: - {{- with .Values.ingress.className }} - ingressClassName: {{ . | quote }} - {{- end }} - {{- if .Values.ingress.tls.enabled }} - tls: - - hosts: - - {{ .Values.hostName | quote }} - {{- with .Values.ingress.tls.secret }} - secretName: {{ . | quote }} - {{- end }} - {{- end }} - rules: - - host: {{ .Values.hostName | quote }} - http: - paths: - - pathType: "Prefix" - path: "/static" - backend: - service: - name: {{ $fullName }}-static - port: - number: {{ .Values.staticFiles.service.port }} - - pathType: "Prefix" - path: "/" - backend: - service: - name: {{ $fullName }}-backend - port: - number: {{ .Values.backend.service.port }} diff --git a/charts/gate-teamware/templates/job-migrate.yaml b/charts/gate-teamware/templates/job-migrate.yaml deleted file mode 100644 index 14de28f0..00000000 --- a/charts/gate-teamware/templates/job-migrate.yaml +++ /dev/null @@ -1,96 +0,0 @@ -{{- if .Values.migrations.run }} -{{- with .Values.backend }} -apiVersion: batch/v1 -kind: Job -metadata: - name: {{ include "gate-teamware.fullname" $ }}-migrate-{{ $.Release.Revision }} - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: migrations -spec: - {{- with $.Values.migrations.ttl }} - ttlSecondsAfterFinished: {{ . }} - {{- end }} - template: - spec: - {{- with .imagePullSecrets }} - imagePullSecrets: - {{- toYaml . | nindent 8 }} - {{- end }} - serviceAccountName: {{ include "gate-teamware.serviceAccountName" $ }} - securityContext: - {{- toYaml .podSecurityContext | nindent 8 }} - restartPolicy: Never - {{- with $.Values.migrations.waitFor }} - initContainers: - - name: wait-for - image: {{ print .registry .image ":" .tag | quote }} - env: - - name: POSTGRES_HOST - value: {{ printf "%s:%s" (include "postgresql.primary.fullname" $.Subcharts.postgresql) (include "postgresql.service.port" $.Subcharts.postgresql) | quote }} - command: - - /wait-for - args: - - --host="$(POSTGRES_HOST)" - - --timeout={{ .timeout }} - {{- end }} - containers: - - name: {{ $.Chart.Name }} - securityContext: - {{- toYaml .securityContext | nindent 12 }} - image: "{{ $.Values.imageRegistry }}{{ .image.repository }}:{{ .image.tag | default $.Chart.AppVersion }}" - imagePullPolicy: {{ .image.pullPolicy }} - env: - - name: DJANGO_SETTINGS_MODULE - value: teamware.settings.deployment - - name: DJANGO_ALLOWED_HOSTS - value: {{ $.Values.hostName | quote }} - # Only run migrations, nothing else - - name: TEAMWARE_ONLY_SETUP - value: "true" - - name: DJANGO_DB_NAME - value: {{ $.Values.postgresql.auth.database | quote }} - - name: DB_HOST - value: {{ include "postgresql.primary.fullname" $.Subcharts.postgresql | quote }} - - name: DB_PORT - value: {{ include "postgresql.service.port" $.Subcharts.postgresql | quote }} - - name: DB_USERNAME - value: {{ $.Values.postgresql.auth.username | quote }} - - name: DB_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "postgresql.secretName" $.Subcharts.postgresql }} - key: password - - name: DJANGO_SECRET_KEY - valueFrom: - secretKeyRef: - name: {{ .djangoSecret | required "backend.djangoSecret must be set to the name of an existing secret with an entry 'secret-key' and a value consisting of 40+ random printable characters" }} - key: secret-key - {{- with $.Values.superuser }} - - name: SUPERUSER_EMAIL - value: {{ .email | quote }} - - name: SUPERUSER_USERNAME - value: {{ .username | quote }} - - name: SUPERUSER_PASSWORD - valueFrom: - secretKeyRef: - name: {{ include "gate-teamware.superuserSecret" $ | quote }} - key: password - {{- end }} - resources: - {{- toYaml .resources | nindent 12 }} - {{- with .nodeSelector }} - nodeSelector: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .affinity }} - affinity: - {{- toYaml . | nindent 8 }} - {{- end }} - {{- with .tolerations }} - tolerations: - {{- toYaml . | nindent 8 }} - {{- end }} -{{- end }} -{{- end }} diff --git a/charts/gate-teamware/templates/secret-superuser.yaml b/charts/gate-teamware/templates/secret-superuser.yaml deleted file mode 100644 index 516549fb..00000000 --- a/charts/gate-teamware/templates/secret-superuser.yaml +++ /dev/null @@ -1,11 +0,0 @@ -{{- if not .Values.superuser.existingSecret -}} -apiVersion: v1 -kind: Secret -metadata: - name: {{ include "gate-teamware.superuserSecret" . | quote }} - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} -data: - password: {{ .Values.superuser.password | b64enc | quote }} -{{- end }} diff --git a/charts/gate-teamware/templates/service-backend.yaml b/charts/gate-teamware/templates/service-backend.yaml deleted file mode 100644 index 92d463cf..00000000 --- a/charts/gate-teamware/templates/service-backend.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- with .Values.backend }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "gate-teamware.fullname" $ }}-backend - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: backend -spec: - type: {{ .service.type }} - ports: - - port: {{ .service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: backend -{{- end }} diff --git a/charts/gate-teamware/templates/service-static.yaml b/charts/gate-teamware/templates/service-static.yaml deleted file mode 100644 index 404a2d3e..00000000 --- a/charts/gate-teamware/templates/service-static.yaml +++ /dev/null @@ -1,20 +0,0 @@ -{{- with .Values.staticFiles }} -apiVersion: v1 -kind: Service -metadata: - name: {{ include "gate-teamware.fullname" $ }}-static - labels: - {{- include "gate-teamware.labels" $ | nindent 4 }} - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: static-files -spec: - type: {{ .service.type }} - ports: - - port: {{ .service.port }} - targetPort: http - protocol: TCP - name: http - selector: - {{- include "gate-teamware.selectorLabels" $ | nindent 4 }} - app.kubernetes.io/component: static-files -{{- end }} diff --git a/charts/gate-teamware/templates/serviceaccount.yaml b/charts/gate-teamware/templates/serviceaccount.yaml deleted file mode 100644 index 702e6424..00000000 --- a/charts/gate-teamware/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "gate-teamware.serviceAccountName" . }} - labels: - {{- include "gate-teamware.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/charts/gate-teamware/values.yaml b/charts/gate-teamware/values.yaml deleted file mode 100644 index ecd78ec4..00000000 --- a/charts/gate-teamware/values.yaml +++ /dev/null @@ -1,239 +0,0 @@ -# Default values for gate-teamware. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -# Host name under which the app will be exposed by the ingress -hostName: "example.gate.ac.uk" -publicUrl: "" - -superuser: - # Email address for the default superuser account. This account is created - # by the migration job only in the case where there are no superuser accounts - # already in the database - email: teamware-admin@example.com - # Username for the default superuser - username: admin - # Initial password for the default superuser, ignored if existingSecret is set - password: "admin" - # Name of an existing secret containing the initial superuser password as the - # "password" key. - existingSecret: "" - -# Docker registry prefix from which the backend and staticFiles images will be -# pulled - if specified, this value _must_ end with a forward slash, e.g. -# imageRegistry: "ghcr.io/gatenlp/" -imageRegistry: "" - -migrations: - # Whether to run a job at chart install time to perform the Django migrations - - # this needs to be set to true on the first install, but could be changed to - # false for subsequent ones if there are no DB changes. We deliberately do not - # run migrations as part of the regular Django startup as this is dangerous - # when there may be more than one replica of the backend pod. - run: false - ttl: 300 - - waitFor: - registry: "ghcr.io/patrickdappollonio/" - image: "wait-for" - tag: "latest" - timeout: "30s" - -backend: - image: - repository: teamware-backend - pullPolicy: IfNotPresent - pullSecrets: [] - # Overrides the image tag whose default is the chart appVersion. - tag: "" - replicaCount: 1 - djangoSecret: "" - service: - type: ClusterIP - port: 80 - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - - nodeSelector: {} - - tolerations: [] - - affinity: {} - - podAnnotations: {} - - podSecurityContext: {} - # fsGroup: 2000 - - # extra environment variable settings and command arguments to be passed to the django container - extraEnv: [] - extraArgs: [] - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - - -staticFiles: - image: - repository: teamware-static - pullPolicy: IfNotPresent - pullSecrets: [] - # Overrides the image tag whose default is the chart appVersion. - tag: "" - replicaCount: 1 - service: - type: ClusterIP - port: 80 - resources: {} - # We usually recommend not to specify default resources and to leave this as a conscious - # choice for the user. This also increases chances charts run on environments with little - # resources, such as Minikube. If you do want to specify resources, uncomment the following - # lines, adjust them as necessary, and remove the curly braces after 'resources:'. - # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi - - healthPort: 8888 - - autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - - nodeSelector: {} - - tolerations: [] - - affinity: {} - - podAnnotations: {} - - podSecurityContext: {} - # fsGroup: 2000 - - securityContext: {} - # capabilities: - # drop: - # - ALL - # readOnlyRootFilesystem: true - # runAsNonRoot: true - # runAsUser: 1000 - -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - - -ingress: - className: "" - annotations: {} - # kubernetes.io/tls-acme: "true" - tls: - enabled: true - secret: "" - -email: - # Admin email address, used as the "from" address on automated emails - adminAddress: "" - # "smtp" for SMTP(S) sender, "gmail" for GMail API - backend: "" - # Should we require activation of new accounts with an emailed token, to - # verify the email address? - activationEnabled: false - smtp: - host: "" - port: "587" - # username, if SMTP server requires authentication - user: "" - # kubernetes secret containing the SMTP password as the "password" key - passwordSecret: "" - # SMTP security, if required - "TLS" for STARTTLS (typically on port 25 or - # 587), "SSL" for TLS-on-connect (typically port 465) - security: "" - # Name of a TLS secret containing the client key and certificate, if - # required - clientCertSecret: "" - gmail: - # OAuth client ID for the GMail API - clientId: "" - # Name of a Kubernetes secret with keys "client-secret" for the OAuth - # client secret and "refresh-token" for the GMail API refresh token. - secretName: "" - -# Database settings -postgresql: - architecture: standalone - auth: - database: teamware_db - username: gate - existingSecret: postgres-credentials - - # Additional useful settings include persistence.storageClass and persistence.size - -# Configure regular database backups to S3 (or compatible) storage -backup: - image: "schickling/postgres-backup-s3:latest" - pullPolicy: Always - # API endpoint, if not the public Amazon S3 - endpoint: "" - # S3 region e.g. us-east-1 - region: "" - # Bucket to which the backups should be stored - bucketName: "" - # common key prefix for all the backups - keyPrefix: "" - # Kubernetes secret with keys "access-key-id" and "secret-key" that have - # write access to the specified bucket - credentialsSecret: "" - - concurrencyPolicy: "" - failedJobsHistoryLimit: "" - successfulJobsHistoryLimit: "" - - # Backup schedule. Keys in the map are the name of the backup schedule - # (e.g. daily, weekly), values are the cron expression defining the schedule. - # The actual backup files will be placed at - # s3://///_.sql.gz - # - # Note that the cron job creates new backups but does not expire old ones - - # if you want old backups to expire (or be moved to cheaper storage like - # infrequent access or Glacier) then you should configure that at the bucket - # level using S3 lifecycle rules. - schedule: {} - # daily: "13 01 * * *" - # weekly: "13 02 * * 0" - # monthly: "13 00 1 * *" diff --git a/charts/initial-secrets.yaml b/charts/initial-secrets.yaml deleted file mode 100644 index a784843e..00000000 --- a/charts/initial-secrets.yaml +++ /dev/null @@ -1,24 +0,0 @@ -# Generate a random 42-byte secret key for Django and encode it as URL-safe base64 -apiVersion: v1 -kind: Secret -metadata: - name: django-secret - annotations: - secret-generator.v1.mittwald.de/autogenerate: secret-key - secret-generator.v1.mittwald.de/type: string - secret-generator.v1.mittwald.de/encoding: base64url - secret-generator.v1.mittwald.de/length: "42b" -data: {} ---- -# Generate random 16 byte passwords for the postgresql superuser and app user -apiVersion: v1 -kind: Secret -metadata: - name: postgres-credentials - annotations: - secret-generator.v1.mittwald.de/autogenerate: "postgres-password,password" - secret-generator.v1.mittwald.de/type: string - secret-generator.v1.mittwald.de/encoding: base64url - secret-generator.v1.mittwald.de/length: "16b" -data: {} ---- diff --git a/custom-policies/.gitignore b/custom-policies/.gitignore new file mode 100644 index 00000000..59a1e2b8 --- /dev/null +++ b/custom-policies/.gitignore @@ -0,0 +1,2 @@ +privacy-policy.md +terms-and-conditions.md diff --git a/custom-policies/privacy-policy.md.template b/custom-policies/privacy-policy.md.template new file mode 100644 index 00000000..b36a3c0a --- /dev/null +++ b/custom-policies/privacy-policy.md.template @@ -0,0 +1,21 @@ +# GATE Teamware Privacy Policy + +_(last updated YYYY-MM-DD)_ + +## Definitions of Roles and Terminology + +For the purposes of this privacy policy the following roles are defined... + +--- + +> If the standard GATE Teamware privacy policy does not meet +> your needs, you will need to create your own policy as a +> Markdown document. A simple way to start would be to +> run Teamware initially with the default policy, visit the +> policy page in your browser, and copy and paste the content +> into [StackEdit](https://stackedit.io/app) to convert it to +> Markdown, then edit it to meet your requirements. +> +> Save your completed policy document in this folder as +> `privacy-policy.md`, and Teamware will serve it in place +> of the standard policy. \ No newline at end of file diff --git a/custom-policies/terms-and-conditions.md.template b/custom-policies/terms-and-conditions.md.template new file mode 100644 index 00000000..8fa1db45 --- /dev/null +++ b/custom-policies/terms-and-conditions.md.template @@ -0,0 +1,19 @@ +# Terms & Conditions + +_(last updated YYYY-MM-DD)_ + +These terms and conditions (the “Terms and Conditions”) govern the use of GATE Teamware... + +--- + +> If the standard GATE Teamware terms document does not meet +> your needs, you will need to create your own version as a +> Markdown document. A simple way to start would be to +> run Teamware initially with the default terms, visit the +> T&C page in your browser, and copy and paste the content +> into [StackEdit](https://stackedit.io/app) to convert it to +> Markdown, then edit it to meet your requirements. +> +> Save your completed document in this folder as +> `terms-and-conditions.md`, and Teamware will serve it in +> place of the standard terms. \ No newline at end of file diff --git a/cypress/integration/admin.spec.js b/cypress/integration/admin.spec.js index e0da4e78..528554ed 100644 --- a/cypress/integration/admin.spec.js +++ b/cypress/integration/admin.spec.js @@ -1,4 +1,4 @@ -import { adminUsername, password, manageUsersPageStr } from '../support/params.js'; +import {adminUsername, password, manageUsersPageStr, annotatorUsername} from '../support/params.js'; describe("Test admin user management", () => { @@ -94,4 +94,27 @@ describe("Test admin user management", () => { }) -}) \ No newline at end of file + it('Delete annotator personal info', () => { + + // Select the annotator user + cy.contains("annotator").click() + cy.contains("Delete user").click({force: true}) + cy.contains("Unlock").click() + cy.get(".modal-dialog").contains("Delete").click() + cy.get("#users").contains(annotatorUsername).should("not.exist") + + }) + + it('Delete annotator', () => { + + // Select the annotator user + cy.contains("annotator").click() + cy.contains("Delete user").click({force: true}) + cy.get("input[name='delete-all-user-data']").check({force: true}) + cy.contains("Unlock").click() + cy.get(".modal-dialog").contains("Delete").click() + cy.get("#users").contains(annotatorUsername).should("not.exist") + + }) + +}) diff --git a/cypress/integration/annotator-mgmt.spec.js b/cypress/integration/annotator-mgmt.spec.js index 2ce71e88..3f881cbb 100644 --- a/cypress/integration/annotator-mgmt.spec.js +++ b/cypress/integration/annotator-mgmt.spec.js @@ -1,4 +1,4 @@ -import { projectsPageStr, adminUsername, password } from '../support/params.js'; +import {projectsPageStr, adminUsername, password, annotatorUsername} from '../support/params.js'; describe('Annotator Leaving Test', () => { @@ -148,3 +148,28 @@ describe('Annotator Management Test', () => { }) }) + +describe("Adding annotator to project should not show deleted accounts", ()=>{ + beforeEach(()=>{ + const fixtureName = 'create_db_users_with_project_annotator_personal_info_deleted' + cy.migrate_integration_db(fixtureName) + cy.login(adminUsername, password) + }) + + it.only("Deleted user should not show up in project users admin search", ()=>{ + cy.visit("/") + + // Goes to project page + cy.contains(projectsPageStr).click() + cy.get("h1").should("contain", projectsPageStr) + + cy.contains('Test project').click() + + // Go to annotator management tab + cy.contains('Annotators').click() + + cy.contains('Add annotators').click() + cy.get("#possibleAnnotators").contains(annotatorUsername).should("not.exist") + + }) +}) diff --git a/cypress/integration/users.spec.js b/cypress/integration/users.spec.js index 4b6b28f9..2e2e2621 100644 --- a/cypress/integration/users.spec.js +++ b/cypress/integration/users.spec.js @@ -180,4 +180,35 @@ describe('User options', () => { }) }) -}) \ No newline at end of file +}) + +describe('User delete account', ()=>{ + beforeEach(() => { + const fixtureName = 'create_db_users' + cy.migrate_integration_db(fixtureName) + }) + + it("Test user delete personal information", ()=>{ + cy.login(annotatorUsername, password) + cy.visit("/") + cy.contains(annotatorUsername).click() + cy.contains("Account").click() + cy.contains("Delete my account").click() + cy.contains("Unlock").click() + cy.get(".modal-dialog").contains("Delete").click() + cy.contains(annotatorUsername).should("not.exist") + }) + + it("Test user delete account", ()=>{ + cy.login(annotatorUsername, password) + cy.visit("/") + cy.contains(annotatorUsername).click() + cy.contains("Account").click() + cy.contains("Delete my account").click() + cy.get("input[name='delete-all-user-data']").check({force: true}) + cy.contains("Unlock").click() + cy.get(".modal-dialog").contains("Delete").click() + cy.contains(annotatorUsername).should("not.exist") + }) + +}) diff --git a/deploy.sh b/deploy.sh index fb3b8af1..c72704df 100755 --- a/deploy.sh +++ b/deploy.sh @@ -6,14 +6,9 @@ source .env DEPLOY_ENV=$1 -DJANGO_SETTINGS_MODULE=teamware.settings.deployment -export DJANGO_SETTINGS_MODULE - case $DEPLOY_ENV in production|prod) - DJANGO_ALLOWED_HOSTS=$TEAMWARE_HOST_URL_PRODUCTION - export DJANGO_ALLOWED_HOSTS echo "Deploying with DJANGO_ALLOWED_HOSTS: $DJANGO_ALLOWED_HOSTS" ;; diff --git a/docker-compose-https.yml b/docker-compose-https.yml new file mode 100644 index 00000000..c4a067ac --- /dev/null +++ b/docker-compose-https.yml @@ -0,0 +1,43 @@ +# +# This is an optional additional file that adds a Caddy reverse proxy +# (https://caddyserver.com) in front of your Teamware installation as a +# simple way to make it available over HTTPS. +# +# If your host is accessible from the internet on port 80 & 443, and the +# host name(s) you have configured under DJANGO_ALLOWED_HOSTS in your +# .env file are mapped correctly in the public DNS, you can add this file +# to your docker compose configuration to serve GATE Teamware as +# https://{HOST} (for each host name in DJANGO_ALLOWED_HOSTS), with +# publicly-trusted certificates provisioned automatically from ZeroSSL +# using the ACME protocol. +# +# docker compose -f docker-compose.yml -f docker-compose-https.yml up -d +# +# If your host is not accessible from the internet you can test this by +# setting DJANGO_ALLOWED_HOSTS=localhost, in which case Caddy will generate +# and use its own private certificate authority to issue a certificate +# instead of ACME. +# +# Note you should set DJANGO_APP_URL=https://{HOST} in your .env so Teamware +# uses the correct URL in emails that it sends to your users. +# + +version: "3.3" + +services: + caddy: + image: caddy:latest + restart: always + environment: + - DJANGO_ALLOWED_HOSTS + ports: + - "80:80" + - "443:443" + volumes: + - "caddy-config:/config" + - "caddy-data:/data" + - ./Caddyfile:/etc/caddy/Caddyfile + +volumes: + caddy-data: + caddy-config: \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 746a030e..2d3ef61e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,11 +10,32 @@ services: - DJANGO_SETTINGS_MODULE - DJANGO_SECRET_KEY - DJANGO_DB_NAME + - DJANGO_APP_URL + - DJANGO_EMAIL_BACKEND + - DJANGO_GMAIL_API_CLIENT_ID + - DJANGO_GMAIL_API_CLIENT_SECRET + - DJANGO_GMAIL_API_REFRESH_TOKEN + - DJANGO_EMAIL_HOST + - DJANGO_EMAIL_PORT + - DJANGO_EMAIL_HOST_USER + - DJANGO_EMAIL_HOST_PASSWORD + - DJANGO_EMAIL_SECURITY + - DJANGO_EMAIL_CLIENT_KEY + - DJANGO_EMAIL_CLIENT_CERTIFICATE + - DJANGO_ACTIVATION_WITH_EMAIL - DB_USERNAME - DB_PASSWORD - SUPERUSER_USERNAME - SUPERUSER_PASSWORD - SUPERUSER_EMAIL + - PP_HOST_NAME + - PP_HOST_ADDRESS + - PP_HOST_CONTACT + - PP_ADMIN_NAME + - PP_ADMIN_ADDRESS + - PP_ADMIN_CONTACT + volumes: + - ./custom-policies:/app/custom-policies/ depends_on: - db diff --git a/docs/docs.config.js b/docs/docs.config.js index d4f15568..9e6da9cd 100644 --- a/docs/docs.config.js +++ b/docs/docs.config.js @@ -1,9 +1,9 @@ module.exports = { - documentationDir: "docs/docs", - documentationVersionsDir: "docs/versioned", - buildTargetDir:"docs/site/gate-teamware/", + documentationDir: "docs", + documentationVersionsDir: "versioned", + buildTargetDir:"site/gate-teamware/", base:"/gate-teamware/", - frontendSourceDir:"frontend/src", + frontendSourceDir:"../frontend/src", latestVersionName: "development", defaultVersion:"development" } diff --git a/docs/docs/.vuepress/versions.json b/docs/docs/.vuepress/versions.json index e53ebbab..fb9581d8 100644 --- a/docs/docs/.vuepress/versions.json +++ b/docs/docs/.vuepress/versions.json @@ -6,6 +6,14 @@ "text": "0.3.0", "value": "/gate-teamware/0.3.0/" }, + { + "text": "0.4.0", + "value": "/gate-teamware/0.4.0/" + }, + { + "text": "2.0.0", + "value": "/gate-teamware/2.0.0/" + }, { "text": "development", "value": "/gate-teamware/development/" diff --git a/docs/docs/README.md b/docs/docs/README.md index b6fd3544..3586c291 100644 --- a/docs/docs/README.md +++ b/docs/docs/README.md @@ -21,6 +21,107 @@ To use an existing instance of GATE Teamware as a project manager or admin, find Documentation on deploying your own instance can be found in the [Developer Guide](developerguide). +## Installation Guide + +### Quick Start + +The simplest way to deploy your own copy of GATE Teamware is to use Docker Compose on Linux or Mac. Installation on Windows is possible but not officially supported - you need to be able to run `bash` shell scripts for the quick-start installer. + +1. Install Docker - [Docker Engine](https://docs.docker.com/engine/) for Linux servers or [Docker Desktop](https://docs.docker.com/desktop/) for Mac. +2. Install [Docker Compose](https://github.com/docker/compose), if your Docker does not already include it (Compose is included by default with Docker Desktop) +3. Download the [installation script](https://gate.ac.uk/get-teamware.sh) into an empty directory, run it and follow the instructions. + +``` +mkdir gate-teamware +cd gate-teamware +curl -LO https://gate.ac.uk/get-teamware.sh +bash ./get-teamware.sh +``` + +This will make the Teamware application available as `http://localhost:8076`, with the option to expose it as a public `https://` URL if your server is directly internet-accessible - for production use we recommend deploying Teamware with a suitable internet-facing reverse proxy, or use Kubernetes as described below. + +### Deployment using Kubernetes + +A Helm chart to deploy Teamware on Kubernetes is published to the GATE team public charts repository. The chart requires [Helm](https://helm.sh) version 3.7 or later, and is compatible with Kubernetes version 1.23 or later. Earlier Kubernetes versions back to 1.19 _may_ work provided autoscaling is not enabled, but these have not been tested. + +The following quick start instructions assume you have a compatible Kubernetes cluster and a working installation of `kubectl` and `helm` (3.7 or later) with permission to create all the necessary resource types in your target namespace. + +First generate a random "secret key" for the Django application. This must be at least 50 random characters, a quick way to do this is + +``` +# 42 random bytes base64 encoded becomes 56 random characters +kubectl create secret generic -n {namespace} django-secret \ + --from-literal="secret-key=$( openssl rand -base64 42 )" +``` + +Add the GATE charts repository to your Helm configuration: + +``` +helm repo add gate https://repo.gate.ac.uk/repository/charts +helm repo update +``` + +Create a `values.yaml` file with the key settings required for teamware. The following is a minimal set of values for a typical installation: + +```yaml +# Public-facing web hostname of the teamware application, the public +# URL will be https://{hostName} +hostName: teamware.example.com + +email: + # "From" address on emails sent by Teamware + adminAddress: admin@teamware.example.com + # Send email via an SMTP server - alternatively "gmail" to use GMail API + backend: "smtp" + smtp: + host: mail.example.com + # You will also need to set user and passwordSecret if your + # mail server requires authentication + +privacyPolicy: + # Contact details of the host and administrator of the teamware + # instance, if no admin defined, defaults to the host values. + host: + # Name of the host + name: "Service Host" + # Host's physical address + address: "123 Example Street, City. Country." + # A method of contacting the host, field supports HTML for e.g. linking to a form + contact: "Email" + admin: + name: "Dr. Service Admin" + address: "Department of Example Studies, University of Example, City. Country." + contact: "Email" + +backend: + # Name of the random secret you created above + djangoSecret: django-secret + +# Initial "super user" created on the first install. These are just +# the *initial* settings, you can (and should!) change the password +# once Teamware is up and running +superuser: + email: me@example.com + username: admin + password: changeme +``` + +Some of these may be omitted or others may be required depending on the setup of your specific cluster - see the [chart README](https://github.com/GateNLP/charts/blob/main/gate-teamware/README.md) and the chart's own values file (which you can retrieve with `helm show values gate/gate-teamware`) for full details. In particular these values assume: + +- your cluster has an ingress controller, with a default ingress class configured, and that controller has a default TLS certificate that is compatible with your chosen hostname (e.g. a `*.example.com` wildcard) +- your cluster has a default storageClass configured to provision PVCs, and at least 8 GB of available PV capacity +- you can send email via an SMTP server with no authentication +- the default GATE Teamware terms and privacy documents are suitable for your deployment and compliant with the laws of your location. If this is not the case you can supply your own custom policy documents in a ConfigMap +- you do not need to back up your PostgreSQL database - the chart does include the option to store backups in Amazon S3 or another compatible object store, see the full README for details + +Once you have created your values file, you can install the chart or upgrade an existing installation using + +``` +helm upgrade --install gate-teamware gate/gate-teamware \ + --namespace {namespace} --values teamware-values.yaml +``` + + ## Bug reports and feature requests Please make bug reports and feature requests as Issues on the [GATE Teamware GitHub repo](https://github.com/GATENLP/gate-teamware). diff --git a/docs/docs/annotatorguide/README.md b/docs/docs/annotatorguide/README.md index 15cf8ac3..4a3ddae2 100644 --- a/docs/docs/annotatorguide/README.md +++ b/docs/docs/annotatorguide/README.md @@ -14,5 +14,14 @@ Annotating a project: * Once you've finished annotating a certain number of documents in a project (specified by the project manager) your task will be deemed complete, and you will be able to be recruited into another annotation project. + +## Deleting your account +At any time you can choose to stop participating and delete your account. You can do this by: + +* Click on your username in the top right corner and then `Account`. +* Click on `Delete my account`. +* When deleting your account, by default your personal information will be removed but your annotations will remain on the system. To completely remove all of your annotations, click on the checkbox next to `Also remove any annotations, projects and documents that I own:`. +* Click the `Unlock` button. +* Then click `Delete` to remove your account. diff --git a/docs/docs/developerguide/README.md b/docs/docs/developerguide/README.md index 8923f932..0696512c 100644 --- a/docs/docs/developerguide/README.md +++ b/docs/docs/developerguide/README.md @@ -119,8 +119,8 @@ To run separately: npm run serve:frontend ``` -## Deployment using Docker -Deployment is via [docker-compose](https://docs.docker.com/compose/), using [NGINX](https://www.nginx.com/) to serve static content, a separate [postgreSQL](https://hub.docker.com/_/postgres) service containing the database and a database backup service (see `docker-compose.yml` for details). +## Deploying a development version using Docker +Deployment is via [docker-compose](https://docs.docker.com/compose/), using [NGINX](https://www.nginx.com/) to serve static content, a separate [postgreSQL](https://hub.docker.com/_/postgres) service containing the database and a database backup service (see `docker-compose.yml` for details). Pre-built images can be run using most versions of Docker but _building_ images requires `docker buildx`, which means either Docker Desktop or version 19.03 or later of Docker Engine. 1. Run `./generate-docker-env.sh` to create a `.env` file containing randomly generated secrets which are mounted as environment variables into the container. See [below](#env-config) for details. @@ -179,13 +179,6 @@ This will first launch the database container, then via Django's `dbshell` comma 4. Redeploy the stack, via `./deploy.sh staging` or `./deploy.sh production`, whichever is the case. 5. The database *should* be restored. - -## Deployment using Kubernetes -Helm charts and instructions for deploying teamware via Kubernetes are available in the `charts/` folder. - -*More documentation to follow* - - ## Configuration ### Django settings files @@ -198,6 +191,8 @@ A SQLite3 database is used during development and during integration testing. For staging and production, postgreSQL is used, running from a `postgres-12` docker container. Settings are found in `teamware/settings/base.py` and `deployment.py` as well as being set as environment variables by `./generate-docker-env.sh` and passed to the container as configured in `docker-compose.yml`. +In Kubernetes deployments the PostgreSQL database is installed using the Bitnami `postresql` public chart. + ### Sending E-mail It's recommended to specify e-mail configurations through environment variables (`.env`). As these settings will include username and passwords that should not be tracked by version control. @@ -243,3 +238,46 @@ This package includes the script linked in the documentation above, which simpli DJANGO_GMAIL_API_CLIENT_SECRET='google_assigned_secret' DJANGO_GMAIL_API_REFRESH_TOKEN='google_assigned_token' ``` + + +#### Teamware Privacy Policy and Terms & Conditions + +Teamware includes a default privacy policy and terms & conditions, which are required for running the application. + +The default privacy policy is intended to be compliant with UK GDPR regulations, which may comply with the rights of users of your deployment, however it is your responsibility to ensure that this is the case. + +If the default privacy policy covers your use case, then you will need to include configuration for a few contact details. + +Contact details are required for the **host** and the **administrator**: the **host** is the organisation or individual responsible for managing the deployment of the teamware instance and the **administrator** is the organisation or individual responsible for managing users, projects and data on the instance. In many cases these roles will be filled by the same organisation or individual, so in this case specifying just the **host** details is sufficient. + +For deployment from source, set the following environment variables: + +* `PP_HOST_NAME` +* `PP_HOST_ADDRESS` +* `PP_HOST_CONTACT` +* `PP_ADMIN_NAME` +* `PP_ADMIN_ADDRESS` +* `PP_ADMIN_CONTACT` + +For deployment using docker-compose, set these values in `.env`. + +If the host and administrator are the same, you can just set the `PP_HOST_*` variables above which will be used for both. + +##### Including a custom Privacy Policy and/or Terms & Conditions + +If the default privacy policy or terms & conditions do not cover your use case, you can easily replace these with your own documents. + +If deploying from source, include markdown (`.md`) files in a `custom-policies` directory in the project root with the exact names `custom-policies/privacy-policy.md` and/or `custom-policies/terms-and-conditions.md` which will be rendered at the corresponding pages on the running web app. If you are not familiar with the Markdown language there are a number of free WYSIWYG-style editor tools available including [StackEdit](https://stackedit.io/app) (browser based) and [Zettlr](https://www.zettlr.com) (desktop app). + +If deploying with docker compose, place the `custom-policies` directory at the same location as the `docker-compose.yml` file before running `./deploy.sh` as above. + +An example custom privacy policy file contents might look like: + +```md +# Organisation X Teamware Privacy Policy +... +... +## Definitions of Roles and Terminology +... +... +``` \ No newline at end of file diff --git a/docs/docs/developerguide/api_docs.md b/docs/docs/developerguide/api_docs.md index 711a8366..03964b37 100644 --- a/docs/docs/developerguide/api_docs.md +++ b/docs/docs/developerguide/api_docs.md @@ -71,6 +71,30 @@ UNAUTHORIZED_ERROR = -32001 +### initialise() + + +::: tip Description +Provide the initial context information to initialise the Teamware app + + context_object: + user: + isAuthenticated: bool + isManager: bool + isAdmin: bool + configs: + docFormatPref: bool + global_configs: + allowUserDelete: bool +::: + + + + + + + + ### is_authenticated() @@ -302,6 +326,26 @@ Gets a list of documents in a project where the user has performed annotations i +### user_delete_personal_information() + + + + + + + + + +### user_delete_account() + + + + + + + + + ### create_project() @@ -988,6 +1032,46 @@ Allow annotator to leave their currently associated project. +### admin_delete_user_personal_information(username) + + + + +#### Parameters + +* username + + + + + + + +### admin_delete_user(username) + + + + +#### Parameters + +* username + + + + + + + +### get_privacy_policy_details() + + + + + + + + + ### get_endpoint_listing() diff --git a/docs/docs/developerguide/documentation.md b/docs/docs/developerguide/documentation.md index 8f84e1bb..3de09904 100644 --- a/docs/docs/developerguide/documentation.md +++ b/docs/docs/developerguide/documentation.md @@ -4,6 +4,14 @@ Documentation versioning is managed by the custom node script located at `docs/m Various configuration parameters used for management of documentation versioning can be found in `docs/docs.config.js`. +## Installing dependencies required to serve the documentation site + +The documentation uses vuepress and other libraries which has to be installed separately running the following command from the root of the project: + +```bash +npm run install:docs +``` + ## Editing the documentation The latest version of the documentation is located at `/docs/docs`. The archived (versioned) documentation are located in `/docs/versioned/version_number`. diff --git a/docs/docs/developerguide/releases.md b/docs/docs/developerguide/releases.md index a0a39b75..ec14d1f2 100644 --- a/docs/docs/developerguide/releases.md +++ b/docs/docs/developerguide/releases.md @@ -8,9 +8,11 @@ Note: Releases are always made from the `master` branch of the repository. 1. **Update the changelog** - This has to be done manually, go through any pull requests to `dev` since the last release. - In github pull requests page, use the search term `is:pr merged:>=yyyy-mm-dd` to find all merged PR from the date since the last version change. - - Include the changes in the `CHANGELOG.md` file, each main item should have a link to the originating PR e.g. \[#123\](https://github.com/GateNLP/gate-teamware/pull/123). - - Also add to release notes later. + - Include the changes in the `CHANGELOG.md` file; the changelog section _MUST_ begin with a level-two heading that starts with the relevant version number in square brackets (`## [N.M.P] Optional descriptive suffix`) as the GitHub workflow that creates a release from the eventual tag depends on this pattern to find the right release notes. Each main item within the changelog should have a link to the originating PR e.g. \[#123\](https://github.com/GateNLP/gate-teamware/pull/123). 1. **Update and check the version numbers** - from the teamware directory run `python version.py check` to check whether all version numbers are up to date. If not, update the master `VERSION` file and run `python version.py update` to update all other version numbers and commit the result. Note that `version.py` requires `pyyaml` for reading `CITATION.cff`, `pyyaml` is included in Teamware's dependencies. 1. **Create a version of the documentation** - Run `npm run docs:create_version`, this will archive the current version of the documentation using the version number in `package.json`. 1. **Create a pull request from `dev` to `master`** including any changes to `CHANGELOG.md`, `VERSION`. -1. **Creating a release** - Create a release via the GitHub interface, the relevant commit can be tagged at this stage or prior to creating the release. +1. **Create a tag** - Once the dev-to-master pull request has been merged, create a tag from the resulting `master` branch named `vN.M.P` (i.e. the new version number prefixed with the letter `v`). This will trigger two GitHub workflows: + - one that builds versioned Docker images for this release and pushes them to `ghcr.io`, updating the `latest` image tag to point to the new release + - one that creates a "release" on GitHub with the necessary artifacts to make the `https://gate.ac.uk/get-teamware.sh` installation mechanism work correctly. The release notes for this release will be generated by extracting the matching section from `CHANGELOG.md`. +1. **Update the Helm chart** - Create a new branch on [https://github.com/GateNLP/charts](https://github.com/GateNLP/charts) to update the `appVersion` of the `gate-teamware` Helm chart to match the version that was just created by the tag workflow. You must also update the chart `version`, bumping the major version number if the new chart is not backwards-compatible with the old. Submit a pull request to the `main` branch, which will publish the new chart when it is merged. diff --git a/docs/docs/manageradminguide/config_examples.js b/docs/docs/manageradminguide/config_examples.js index b046853c..e167b81e 100644 --- a/docs/docs/manageradminguide/config_examples.js +++ b/docs/docs/manageradminguide/config_examples.js @@ -10,12 +10,11 @@ export default { "type": "radio", "title": "Sentiment", "description": "Please select a sentiment of the text above.", - "options": { - "negative": "Negative", - "neutral": "Neutral", - "positive": "Positive" - - } + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] } ], config2: [ @@ -29,12 +28,11 @@ export default { "type": "radio", "title": "Sentiment", "description": "Please select a sentiment of the text above.", - "options": { - "negative": "Negative", - "neutral": "Neutral", - "positive": "Positive" - - } + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] }, { "name": "opinion", @@ -64,6 +62,13 @@ export default { "text": "Custom field: {{customField}}
Another custom field: {{{anotherCustomField}}}
Subfield: {{{subfield.subfieldContent}}}" } ], + configDisplayPreserveNewlines: [ + { + "name": "htmldisplay", + "type": "html", + "text": "
{{text}}
" + } + ], configTextInput: [ { "name": "mylabel", @@ -156,6 +161,32 @@ export default { } ], + configDbpediaExample: [ + { + "name": "uri", + "type": "radio", + "title": "Select the most appropriate URI", + "options":[ + {"fromDocument": "candidates"}, + {"value": "none", "label": "None of the above"}, + {"value": "unknown", "label": "Cannot be determined without more context"} + ] + } + ], + docDbpediaExample: { + "text": "President Bush visited the air base yesterday...", + "candidates": [ + { + "value": "http://dbpedia.org/resource/George_W._Bush", + "label": "George W. Bush (Jnr)" + }, + { + "value": "http://dbpedia.org/resource/George_H._W._Bush", + "label": "George H. W. Bush (Snr)" + } + ] + }, + doc1: {text: "Sometext with html"}, doc2: { @@ -165,6 +196,9 @@ export default { subfieldContent: "Content of a subfield." } }, + docPlainText: { + "text": "This is some text\n\nIt has line breaks that we want to preserve." + }, configPreAnnotation: [ { "name": "htmldisplay", @@ -175,24 +209,24 @@ export default { "name": "radio", "type": "radio", "title": "Test radio input", - "options": { - "val1": "Value 1", - "val2": "Value 2", - "val3": "Value 4", - "val4": "Value 5" - }, + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], "description": "Test radio description" }, { "name": "checkbox", "type": "checkbox", "title": "Test checkbox input", - "options": { - "val1": "Value 1", - "val2": "Value 2", - "val3": "Value 4", - "val4": "Value 5" - }, + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], "description": "Test checkbox description" }, { diff --git a/docs/docs/manageradminguide/documents_annotations_management.md b/docs/docs/manageradminguide/documents_annotations_management.md index bb3c54a8..ac010c9f 100644 --- a/docs/docs/manageradminguide/documents_annotations_management.md +++ b/docs/docs/manageradminguide/documents_annotations_management.md @@ -58,24 +58,24 @@ For an example project configuration shown below, there are three captured label "name": "radio", "type": "radio", "title": "Test radio input", - "options": { - "val1": "Value 1", - "val2": "Value 2", - "val3": "Value 4", - "val4": "Value 5" - }, + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], "description": "Test radio description" }, { "name": "checkbox", "type": "checkbox", "title": "Test checkbox input", - "options": { - "val1": "Value 1", - "val2": "Value 2", - "val3": "Value 4", - "val4": "Value 5" - }, + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], "description": "Test checkbox description" }, { diff --git a/docs/docs/manageradminguide/project_config.md b/docs/docs/manageradminguide/project_config.md index 3509035c..99b26e34 100644 --- a/docs/docs/manageradminguide/project_config.md +++ b/docs/docs/manageradminguide/project_config.md @@ -68,11 +68,11 @@ of annotation will be collected. Here's an example configuration and a preview o "type": "radio", "title": "Sentiment", "description": "Please select a sentiment of the text above.", - "options": { - "negative": "Negative", - "neutral": "Neutral", - "positive": "Positive" - } + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] } ] ``` @@ -116,12 +116,11 @@ Another field can be added to collect more information, e.g. a text field for op "type": "radio", "title": "Sentiment", "description": "Please select a sentiment of the text above.", - "options": { - "negative": "Negative", - "neutral": "Neutral", - "positive": "Positive" - - } + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] }, { "name": "opinion", @@ -226,6 +225,34 @@ Configuration, showing the same field/column in document as-is or as HTML: +If your documents are plain text and include line breaks that need to be preserved when rendering, this can be achieved by using a special HTML wrapper which sets the [`white-space` CSS property](https://developer.mozilla.org/en-US/docs/Web/CSS/white-space). + + + +**Document** + +```json +{ + "text": "This is some text\n\nIt has line breaks that we want to preserve." +} +``` + +**Project configuration** + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "
{{text}}
" + } +] +``` + +
+ +`white-space: pre-line` preserves line breaks but collapses other whitespace down to a single space, `white-space: pre-wrap` would preserve all whitespace including indentation at the start of a line, but would still wrap lines that are too long for the available space. + ### Text input @@ -346,7 +373,7 @@ Configuration, showing the same field/column in document as-is or as HTML: -### Alternative way to provide options for radio, checkbox and selector +### Alternative way to provide options for radio, checkbox and selector A dictionary (key value pairs) and also be provided to the `options` field of the radio, checkbox and selector widgets but note that the ordering of the options are **not guaranteed** as javascript does not sort dictionaries by @@ -375,6 +402,87 @@ the order in which keys are added. +### Dynamic options for radio, checkbox and selector + +All the examples above have a "static" list of available options for the radio, checkbox and selector widgets, where the complete options list is enumerated in the project configuration and every document offers the same set of options. However it is also possible to take some or all of the options from the _document_ data rather than the _configuration_ data. For example: + + + +**Project configuration** + +```json +[ + { + "name": "uri", + "type": "radio", + "title": "Select the most appropriate URI", + "options":[ + {"fromDocument": "candidates"}, + {"value": "none", "label": "None of the above"}, + {"value": "unknown", "label": "Cannot be determined without more context"} + ] + } +] +``` + +**Document** + +```json +{ + "text": "President Bush visited the air base yesterday...", + "candidates": [ + { + "value": "http://dbpedia.org/resource/George_W._Bush", + "label": "George W. Bush (Jnr)" + }, + { + "value": "http://dbpedia.org/resource/George_H._W._Bush", + "label": "George H. W. Bush (Snr)" + } + ] +} +``` + + + +`"fromDocument"` is a dot-separated property path leading to the location within each document where the additional options can be found, for example `"fromDocument":"candidates"` looks for a top-level property named `candidates` in each document, `"fromDocument": "options.custom"` would look for a property named `options` which is itself an object with a property named `custom`. The target property in the document may be in any of the following forms: + +- an array _of objects_, each with `value` and `label` properties, exactly as in the static configuration format - this is the format used in the example above +- an array _of strings_, where the same string will be used as both the value and the label for that option +- an arbitrary ["dictionary"](#options-as-dict) object mapping values to labels +- a _single string_, which is parsed into a list of options + +The "single string" alternative is designed to be easier to use when [importing documents](documents_annotations_management.md#importing-documents) from CSV files. It allows you to provide any number of options in a _single_ CSV column value. Within the column the options are separated by semicolons, and each option is of the form `value=label`. Whitespace around the delimiters is ignored, both between options and between the value and label of a single option. For example given CSV document data of + +| text | options | +|-----------------|---------------------------------------------------| +| Favourite fruit | `apple=Apples; orange = Oranges; kiwi=Kiwi fruit` | + +a `{"fromDocument": "options"}` configuration would produce the equivalent of + +```json +[ + {"value": "apple", "label": "Apples"}, + {"value": "orange", "label": "Oranges"}, + {"value": "kiwi", "label": "Kiwi fruit"} +] +``` + +If your values or labels may need to contain the default separator characters `;` or `=` you can select different separators by adding extra properties to the configuration: + +```json +{"fromDocument": "options", "separator": "~~", "valueLabelSeparator": "::"} +``` + +| text | options | +|-----------------|------------------------------------------------------| +| Favourite fruit | `apple::Apples ~~ orange::Oranges ~~ kiwi::Kiwi fruit` | + +The separators can be more than one character, and you can set `"valueLabelSeparator":""` to disable label splitting altogether and just use the value as its own label. + +### Mixing static and dynamic options + +Static and `fromDocument` options may be freely interspersed in any order, so you can have a fully-dynamic set of options by specifying _only_ a `fromDocument` entry with no static options, or you can have static options that are listed first followed by dynamic options, or dynamic options first followed by static, etc. + + diff --git a/docs/versioned/0.4.0/.vuepress/components/DisplayVersion.vue b/docs/versioned/0.4.0/.vuepress/components/DisplayVersion.vue new file mode 100644 index 00000000..03ec07ed --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/components/DisplayVersion.vue @@ -0,0 +1,21 @@ + + + + + diff --git a/docs/versioned/0.4.0/.vuepress/config.js b/docs/versioned/0.4.0/.vuepress/config.js new file mode 100644 index 00000000..36b12c03 --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/config.js @@ -0,0 +1,41 @@ +const versionData = require("./versions.json") +const path = require("path"); +module.exports = context => ({ + title: 'GATE Teamware Documentation', + description: 'Documentation for GATE Teamware', + base: versionData.base, + themeConfig: { + nav: [ + {text: 'Home', link: '/'}, + {text: 'Annotators', link: '/annotatorguide/'}, + {text: 'Managers & Admins', link: '/manageradminguide/'}, + {text: 'Developer', link: '/developerguide/'} + ], + sidebar: { + '/manageradminguide/': [ + "", + "project_management", + "project_config", + "documents_annotations_management", + "annotators_management" + ], + '/developerguide/': [ + '', + 'testing', + 'releases', + 'documentation', + "api_docs", + + ], + }, + }, + configureWebpack: { + resolve: { + alias: { + '@': path.resolve(__dirname, versionData.frontendSource) + } + } + }, + + +}) diff --git a/docs/versioned/0.4.0/.vuepress/enhanceApp.js b/docs/versioned/0.4.0/.vuepress/enhanceApp.js new file mode 100644 index 00000000..e7aadadf --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/enhanceApp.js @@ -0,0 +1,17 @@ +import Vue from 'vue' +import {BootstrapVue, BootstrapVueIcons, IconsPlugin} from 'bootstrap-vue' + +import 'bootstrap/dist/css/bootstrap.css' +import 'bootstrap-vue/dist/bootstrap-vue.css' + +Vue.use(BootstrapVue) +Vue.use(BootstrapVueIcons) + +export default ({ + Vue, // the version of Vue being used in the VuePress app + options, // the options for the root Vue instance + router, // the router instance for the app + siteData // site metadata +}) => { + +} diff --git a/docs/versioned/0.4.0/.vuepress/theme/components/Navbar.vue b/docs/versioned/0.4.0/.vuepress/theme/components/Navbar.vue new file mode 100644 index 00000000..c3b966db --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/theme/components/Navbar.vue @@ -0,0 +1,143 @@ + + + + + diff --git a/docs/versioned/0.4.0/.vuepress/theme/components/VersionSelector.vue b/docs/versioned/0.4.0/.vuepress/theme/components/VersionSelector.vue new file mode 100644 index 00000000..4cfb5eb9 --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/theme/components/VersionSelector.vue @@ -0,0 +1,33 @@ + + + + + diff --git a/docs/versioned/0.4.0/.vuepress/theme/index.js b/docs/versioned/0.4.0/.vuepress/theme/index.js new file mode 100644 index 00000000..b91b8a57 --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/theme/index.js @@ -0,0 +1,3 @@ +module.exports = { + extend: '@vuepress/theme-default' +} diff --git a/docs/versioned/0.4.0/.vuepress/versions.json b/docs/versioned/0.4.0/.vuepress/versions.json new file mode 100644 index 00000000..a5f3fa77 --- /dev/null +++ b/docs/versioned/0.4.0/.vuepress/versions.json @@ -0,0 +1,23 @@ +{ + "current": "0.4.0", + "base": "/gate-teamware/0.4.0/", + "versions": [ + { + "text": "0.3.0", + "value": "/gate-teamware/0.3.0/" + }, + { + "text": "0.4.0", + "value": "/gate-teamware/0.4.0/" + }, + { + "text": "2.0.0", + "value": "/gate-teamware/2.0.0/" + }, + { + "text": "development", + "value": "/gate-teamware/development/" + } + ], + "frontendSource": "../../../../frontend/src" +} \ No newline at end of file diff --git a/docs/versioned/0.4.0/README.md b/docs/versioned/0.4.0/README.md new file mode 100644 index 00000000..3586c291 --- /dev/null +++ b/docs/versioned/0.4.0/README.md @@ -0,0 +1,138 @@ +# GATE Teamware + +![GATE Teamware logo](./img/gate-teamware-logo.svg "GATE Teamware logo") + +A web application for collaborative document annotation. + +This is a documentation for Teamware version: + +## Key Features +* Free and open source software. +* Configure annotation options using a highly flexible JSON config. +* Set limits on proportions of a task that annotators can annotate. +* Import existing annotations as CSV or JSON. +* Export annotations as CSV or JSON. +* Annotation instructions and document rendering supports markdown and HTML. + +## Getting started +A quickstart guide for annotators is [available here](annotatorguide). + +To use an existing instance of GATE Teamware as a project manager or admin, find instructions in the [Managers and Admins guide](manageradminguide). + +Documentation on deploying your own instance can be found in the [Developer Guide](developerguide). + +## Installation Guide + +### Quick Start + +The simplest way to deploy your own copy of GATE Teamware is to use Docker Compose on Linux or Mac. Installation on Windows is possible but not officially supported - you need to be able to run `bash` shell scripts for the quick-start installer. + +1. Install Docker - [Docker Engine](https://docs.docker.com/engine/) for Linux servers or [Docker Desktop](https://docs.docker.com/desktop/) for Mac. +2. Install [Docker Compose](https://github.com/docker/compose), if your Docker does not already include it (Compose is included by default with Docker Desktop) +3. Download the [installation script](https://gate.ac.uk/get-teamware.sh) into an empty directory, run it and follow the instructions. + +``` +mkdir gate-teamware +cd gate-teamware +curl -LO https://gate.ac.uk/get-teamware.sh +bash ./get-teamware.sh +``` + +This will make the Teamware application available as `http://localhost:8076`, with the option to expose it as a public `https://` URL if your server is directly internet-accessible - for production use we recommend deploying Teamware with a suitable internet-facing reverse proxy, or use Kubernetes as described below. + +### Deployment using Kubernetes + +A Helm chart to deploy Teamware on Kubernetes is published to the GATE team public charts repository. The chart requires [Helm](https://helm.sh) version 3.7 or later, and is compatible with Kubernetes version 1.23 or later. Earlier Kubernetes versions back to 1.19 _may_ work provided autoscaling is not enabled, but these have not been tested. + +The following quick start instructions assume you have a compatible Kubernetes cluster and a working installation of `kubectl` and `helm` (3.7 or later) with permission to create all the necessary resource types in your target namespace. + +First generate a random "secret key" for the Django application. This must be at least 50 random characters, a quick way to do this is + +``` +# 42 random bytes base64 encoded becomes 56 random characters +kubectl create secret generic -n {namespace} django-secret \ + --from-literal="secret-key=$( openssl rand -base64 42 )" +``` + +Add the GATE charts repository to your Helm configuration: + +``` +helm repo add gate https://repo.gate.ac.uk/repository/charts +helm repo update +``` + +Create a `values.yaml` file with the key settings required for teamware. The following is a minimal set of values for a typical installation: + +```yaml +# Public-facing web hostname of the teamware application, the public +# URL will be https://{hostName} +hostName: teamware.example.com + +email: + # "From" address on emails sent by Teamware + adminAddress: admin@teamware.example.com + # Send email via an SMTP server - alternatively "gmail" to use GMail API + backend: "smtp" + smtp: + host: mail.example.com + # You will also need to set user and passwordSecret if your + # mail server requires authentication + +privacyPolicy: + # Contact details of the host and administrator of the teamware + # instance, if no admin defined, defaults to the host values. + host: + # Name of the host + name: "Service Host" + # Host's physical address + address: "123 Example Street, City. Country." + # A method of contacting the host, field supports HTML for e.g. linking to a form + contact: "Email" + admin: + name: "Dr. Service Admin" + address: "Department of Example Studies, University of Example, City. Country." + contact: "Email" + +backend: + # Name of the random secret you created above + djangoSecret: django-secret + +# Initial "super user" created on the first install. These are just +# the *initial* settings, you can (and should!) change the password +# once Teamware is up and running +superuser: + email: me@example.com + username: admin + password: changeme +``` + +Some of these may be omitted or others may be required depending on the setup of your specific cluster - see the [chart README](https://github.com/GateNLP/charts/blob/main/gate-teamware/README.md) and the chart's own values file (which you can retrieve with `helm show values gate/gate-teamware`) for full details. In particular these values assume: + +- your cluster has an ingress controller, with a default ingress class configured, and that controller has a default TLS certificate that is compatible with your chosen hostname (e.g. a `*.example.com` wildcard) +- your cluster has a default storageClass configured to provision PVCs, and at least 8 GB of available PV capacity +- you can send email via an SMTP server with no authentication +- the default GATE Teamware terms and privacy documents are suitable for your deployment and compliant with the laws of your location. If this is not the case you can supply your own custom policy documents in a ConfigMap +- you do not need to back up your PostgreSQL database - the chart does include the option to store backups in Amazon S3 or another compatible object store, see the full README for details + +Once you have created your values file, you can install the chart or upgrade an existing installation using + +``` +helm upgrade --install gate-teamware gate/gate-teamware \ + --namespace {namespace} --values teamware-values.yaml +``` + + +## Bug reports and feature requests +Please make bug reports and feature requests as Issues on the [GATE Teamware GitHub repo](https://github.com/GATENLP/gate-teamware). + +# Using Teamware +Teamware is developed by the [GATE](https://gate.ac.uk) team, an academic research group at The University of Sheffield. As a result, future funding relies on evidence of the impact that the software provides. If you use Teamware, please let us know using the contact form at [gate.ac.uk](https://gate.ac.uk/g8/contact). Please include details on grants, publications, commercial products etc. Any information that can help us to secure future funding for our work is greatly appreciated. + +## Citation +For published work that has used Teamware, please cite this repository. One way is to include a citation such as: + +> Karmakharm, T., Wilby, D., Roberts, I., & Bontcheva, K. (2022). GATE Teamware (Version 0.1.4) [Computer software]. https://github.com/GateNLP/gate-teamware + +Please use the `Cite this repository` button at the top of the [project's GitHub repository](https://github.com/GATENLP/gate-teamware) to get an up to date citation. + +The Teamware version can be found on the 'About' page of your Teamware instance. diff --git a/docs/versioned/0.4.0/annotatorguide/README.md b/docs/versioned/0.4.0/annotatorguide/README.md new file mode 100644 index 00000000..4a3ddae2 --- /dev/null +++ b/docs/versioned/0.4.0/annotatorguide/README.md @@ -0,0 +1,27 @@ +# Annotators Quickstart + +Annotating a project: + +* After signing up to the site, notify the owner of the annotation project you've been recruited of + your username. This will allow them to add you as an annotator to a project. +* After you've been recruited to a project, click on the `Annotate` link on the navigation bar at the + top of the page to start annotating. +* You will be shown the details about the project you're annotating along with a set of form(s) to capture + your annotation. Ensure you've read the Annotator guideline fully before starting the annotation process. +* You can then start annotating documents one at a time. Click on `Submit` to confirm the completion of + annotation, `Clear` to start again or `Reject` to skip the particular document. Be aware some projects + do not allow you to skip documents. +* Once you've finished annotating a certain number of documents in a project (specified by the project + manager) your task will be deemed complete, and you will be able to be recruited into another annotation + project. + +## Deleting your account + +At any time you can choose to stop participating and delete your account. You can do this by: + +* Click on your username in the top right corner and then `Account`. +* Click on `Delete my account`. +* When deleting your account, by default your personal information will be removed but your annotations will remain on the system. To completely remove all of your annotations, click on the checkbox next to `Also remove any annotations, projects and documents that I own:`. +* Click the `Unlock` button. +* Then click `Delete` to remove your account. + diff --git a/docs/versioned/0.4.0/developerguide/README.md b/docs/versioned/0.4.0/developerguide/README.md new file mode 100644 index 00000000..0696512c --- /dev/null +++ b/docs/versioned/0.4.0/developerguide/README.md @@ -0,0 +1,283 @@ +# Developer guide + +## Architecture +``` +├── .github/workflows/ # github actions workflow files +├── teamware/ # Django project +│   └── settings/ +├── backend/ # Django app +├── charts/ # Helm charts for Kubernetes +├── cypress/ # integration test configurations +├── docs/ # documentation +├── examples/ # example data files +├── frontend/ # all frontend, in VueJS framework +├── nginx/ # Nginx configurations +| +# Top level directory contains scripts for management and deployment, +# main project package.json, python requirements, docker configs +├── build-images.sh +├── deploy.sh +├── create-django-db.sh +├── docker-compose.yml +├── Dockerfile +├── generate-docker-env.sh +├── manage.py +├── migrate-integration.sh +├── package.json +├── package-lock.json +├── pytest.ini +├── README.md +├── requirements-dev.txt +├── requirements.txt +└── run-server.sh + +``` + +## Installation for development + +The service depends on a combination of python and javascript libraries. We recommend developing inside a `conda` conda environment as it is able to install +python libraries and nodejs which is used to install javascript libraries. + +* Install anaconda/miniconda +* Create a blank virtual conda env + ```bash + $ conda create -n teamware python=3.9 + ``` +* Activate conda environment + ```bash + $ source activate teamware + # or + $ conda activate teamware + ``` +* Install python dependencies in conda environment using pip + ```bash + (teamware)$ pip install -r requirements.txt -r requirements-dev.txt + ``` +* Install nodejs, postgresql and openssl in the conda environment + ```bash + (teamware)$ conda install -y -c conda-forge postgresql=14.* + (teamware)$ conda install -y -c conda-forge nodejs=14.* + ``` +* Install nodejs dependencies + ```bash + (teamware)$ npm install + ``` + +Set up a new postgreSQL database and user for development: +``` +# Create a new directory for the db data and initialise +mkdir -p pgsql/data +initdb -D pgsql/data + +# Launch postgres in the background +postgres -p 5432 -D pgsql/data & + +# Create a DB user, you'll be prompted to input password, "password" is the default in teamware/settings/base.py for development +createuser -p 5432 -P user --createdb + +# Create a rumours_db with rumours as user +createdb -p 5432 -O user teamware_db + +# Migrate & create database tables +python manage.py migrate + +# create a new superuser - when prompted enter a username and password for the db superuser +python manage.py createsuperuser +``` + +## Updating packages +To update packages after a merge, run the following commands: + +```bash +# Activate the conda environment +source activate teamware +# Update any packages changed in the python requirements.txt and requirements-dev.txt files +pip install -r requirements.txt -r requirements-dev.txt +# Update any packages changed in package.json +npm install +``` + +## Development server +The application uses django's dev server to serve page contents and run the RPC API, it also uses Vue CLI's +development server to serve dynamic assets such as javascript or stylesheets allowing for hot-reloading +during development. + +To run both servers together: + + ```bash + npm run serve + ``` + +To run separately: + +* Django server + ```bash + npm run serve:backend + ``` +* Vue CLI dev server + ```bash + npm run serve:frontend + ``` + +## Deploying a development version using Docker +Deployment is via [docker-compose](https://docs.docker.com/compose/), using [NGINX](https://www.nginx.com/) to serve static content, a separate [postgreSQL](https://hub.docker.com/_/postgres) service containing the database and a database backup service (see `docker-compose.yml` for details). Pre-built images can be run using most versions of Docker but _building_ images requires `docker buildx`, which means either Docker Desktop or version 19.03 or later of Docker Engine. + +1. Run `./generate-docker-env.sh` to create a `.env` file containing randomly generated secrets which are mounted as environment variables into the container. See [below](#env-config) for details. + +2. Then build the images via: + ```bash + ./build-images.sh + ``` + +3. then deploy the stack with + + ```bash + ./deploy.sh production # (or prod) to deploy with production settings + ./deploy.sh staging # (or stag) to deploy with staging settings + ``` + +To bring the stack down, run `docker-compose down`, using the `-v` flag to destroy the database volume (be careful with this). + +### Configuration using environment variables (.env file) + +To allow the app to be easily configured between instances especially inside containers, many of the app's configuration can be done through environment variables. + +Run `./generate-docker-env.sh` to generate a `.env` file with all configurable environment parameters. + +To set values for your own deployment, add values to the variables in `.env`, most existing values will be kept after running `generate-docker-env.sh`, see comments in `.env` for specific details. Anything that is left blank will be filled with a default value. Passwords and keys are filled with auto-generated random values. + +Existing `.env` files are copied into a new file named `saved-env.` by `generate-docker-env.sh`. + +### Backups + +In a docker-compose based deployment, backups of the database are managed by the service `pgbackups` which uses the [`prodrigestivill/postgres-backup-local:12`](https://hub.docker.com/r/prodrigestivill/postgres-backup-local) image. +By default, backups are taken of the database daily, and the `docker-compose.yml` contains settings for the number of backups kept under the options for the `pgbackups` service. +Backups are stored as a gzipped SQL dump from the database. + +#### Taking a manual backup + +A shell script is provided for manually triggering a backup snapshot. +From the main project directory run + +```sh +$ ./backup_manual.sh +``` + +This uses the `pgbackups` service and all settings and envrionment variables it is configured with in `docker-compose.yml`, so backups will be taken to the same location as configured for the main backup schedule. + +#### Restoring from a backup +1. Locate the backup file (`*.sql.gz`) on your system that you would like to restore from. +2. Make sure that the stack is down, from the main project directory run `docker-commpose down`. +3. Run the backup restore shell script, passing in the path to your backup file as the only argument: + +```sh +$ ./backup_restore.sh path/to/my/backup.sql.gz +``` + +This will first launch the database container, then via Django's `dbshell` command, running in the `backend` service, execute a number of SQL commands before and after running all the SQL from the backup file. + +4. Redeploy the stack, via `./deploy.sh staging` or `./deploy.sh production`, whichever is the case. +5. The database *should* be restored. + +## Configuration + +### Django settings files + +Django settings are located in `teamware/settings` folder. The app will use `base.py` setting by default +and this must be overridden depending on use. + +### Database +A SQLite3 database is used during development and during integration testing. + +For staging and production, postgreSQL is used, running from a `postgres-12` docker container. Settings are found in `teamware/settings/base.py` and `deployment.py` as well as being set as environment variables by `./generate-docker-env.sh` and passed to the container as configured in `docker-compose.yml`. + +In Kubernetes deployments the PostgreSQL database is installed using the Bitnami `postresql` public chart. + + +### Sending E-mail +It's recommended to specify e-mail configurations through environment variables (`.env`). As these settings will include username and passwords that should not be tracked by version control. + +#### E-mail using SMTP +SMTP is supported as standard in Django, add the following configurations with your own details +to the list of environment variables: + +```bash +DJANGO_EMAIL_BACKEND='django.core.mail.backends.smtp.EmailBackend' +DJANGO_EMAIL_HOST='myserver.com' +DJANGO_EMAIL_PORT=22 +DJANGO_EMAIL_HOST_USER='username' +DJANGO_EMAIL_HOST_PASSWORD='password' +``` + +#### E-mail using Google API +The [django-gmailapi-backend](https://github.com/dolfim/django-gmailapi-backend) library +has been added to allow sending of mail through Google's API as sending through SMTP is disabled as standard. + +Unlike with SMTP, Google's API requires OAuth authentication which means a project and a credential has to be +created through Google's cloud console. + +* More information on the Gmail API: [https://developers.google.com/gmail/api/guides/sending](https://developers.google.com/gmail/api/guides/sending) +* OAuth credentials for sending emails: [https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough](https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough) + +This package includes the script linked in the documentation above, which simplifies the setup of the API credentials. The following outlines the key steps: + +1. Create a project in the Google developer console, [https://console.cloud.google.com/](https://console.cloud.google.com/) +2. Enable the Gmail API +3. Create OAuth 2.0 credentials, you'll likely want to create a `Desktop` +4. Create a valid refresh_token using the helper script included in the package: + ```bash + gmail_oauth2 --generate_oauth2_token \ + --client_id="" \ + --client_secret="" \ + --scope="https://www.googleapis.com/auth/gmail.send" + ``` +5. Add the created credentials and tokens to the environment variable as shown below: + ```bash + DJANGO_EMAIL_BACKEND='gmailapi_backend.mail.GmailBackend' + DJANGO_GMAIL_API_CLIENT_ID='google_assigned_id' + DJANGO_GMAIL_API_CLIENT_SECRET='google_assigned_secret' + DJANGO_GMAIL_API_REFRESH_TOKEN='google_assigned_token' + ``` + + +#### Teamware Privacy Policy and Terms & Conditions + +Teamware includes a default privacy policy and terms & conditions, which are required for running the application. + +The default privacy policy is intended to be compliant with UK GDPR regulations, which may comply with the rights of users of your deployment, however it is your responsibility to ensure that this is the case. + +If the default privacy policy covers your use case, then you will need to include configuration for a few contact details. + +Contact details are required for the **host** and the **administrator**: the **host** is the organisation or individual responsible for managing the deployment of the teamware instance and the **administrator** is the organisation or individual responsible for managing users, projects and data on the instance. In many cases these roles will be filled by the same organisation or individual, so in this case specifying just the **host** details is sufficient. + +For deployment from source, set the following environment variables: + +* `PP_HOST_NAME` +* `PP_HOST_ADDRESS` +* `PP_HOST_CONTACT` +* `PP_ADMIN_NAME` +* `PP_ADMIN_ADDRESS` +* `PP_ADMIN_CONTACT` + +For deployment using docker-compose, set these values in `.env`. + +If the host and administrator are the same, you can just set the `PP_HOST_*` variables above which will be used for both. + +##### Including a custom Privacy Policy and/or Terms & Conditions + +If the default privacy policy or terms & conditions do not cover your use case, you can easily replace these with your own documents. + +If deploying from source, include markdown (`.md`) files in a `custom-policies` directory in the project root with the exact names `custom-policies/privacy-policy.md` and/or `custom-policies/terms-and-conditions.md` which will be rendered at the corresponding pages on the running web app. If you are not familiar with the Markdown language there are a number of free WYSIWYG-style editor tools available including [StackEdit](https://stackedit.io/app) (browser based) and [Zettlr](https://www.zettlr.com) (desktop app). + +If deploying with docker compose, place the `custom-policies` directory at the same location as the `docker-compose.yml` file before running `./deploy.sh` as above. + +An example custom privacy policy file contents might look like: + +```md +# Organisation X Teamware Privacy Policy +... +... +## Definitions of Roles and Terminology +... +... +``` \ No newline at end of file diff --git a/docs/versioned/0.4.0/developerguide/api_docs.md b/docs/versioned/0.4.0/developerguide/api_docs.md new file mode 100644 index 00000000..711a8366 --- /dev/null +++ b/docs/versioned/0.4.0/developerguide/api_docs.md @@ -0,0 +1,1002 @@ +--- +sidebarDepth: 3 +--- + +# API Documentation + +## Using the JSONRPC endpoints + +::: tip +A single endpoint is used for all API requests, located at `/rpc` +::: + +The API used in the app complies to JSON-RPC 2.0 spec. Requests should always be sent with `POST` and +contain a JSON request object in the body. The response will also be in the form of a JSON object. + +For example, to call the method `subtract(a, b)`. Send `POST` a post request to `/rpc` with the following JSON +in the body: + +```json +{ + "jsonrpc":"2.0", + "method":"subtract", + "params":[ + 42, + 23 + ], + "id":1 +} +``` + +Variables are passed as a list to the `params` field, in this case `a=42` and `b=23`. The `id` field in the top +level of the request object refers to the message ID, this ID value will be matched in the response, +it does not affect the method that is being called. + +The response will be as follows: + +```json +{ + "jsonrpc":"2.0", + "result":19, + "id":1 +} +``` + +In the case of errors, the response will contain an `error` field with error `code` and error `message`: + +```json +{ + "jsonrpc":"2.0", + "error":{ + "code":-32601, + "message":"Method not found" + }, + "id":"1" +} +``` + +The following are error codes used in the app: + +```python +PARSE_ERROR = -32700 +INVALID_REQUEST = -32600 +METHOD_NOT_FOUND = -32601 +INVALID_PARAMS = -32602 +INTERNAL_ERROR = -32603 +AUTHENTICATION_ERROR = -32000 +UNAUTHORIZED_ERROR = -32001 +``` + +## API Listing + + + +### is_authenticated() + + +::: tip Description +Checks that the current user has logged in. +::: + + + + + + + + +### login(payload) + + + + +#### Parameters + +* payload + + + + + + + +### logout() + + + + + + + + + +### register(payload) + + + + +#### Parameters + +* payload + + + + + + + +### generate_user_activation(username) + + + + +#### Parameters + +* username + + + + + + + +### activate_account(username,token) + + + + +#### Parameters + +* username + +* token + + + + + + + +### generate_password_reset(username) + + + + +#### Parameters + +* username + + + + + + + +### reset_password(username,token,new_password) + + + + +#### Parameters + +* username + +* token + +* new_password + + + + + + + +### change_password(payload) + + + + +#### Parameters + +* payload + + + + + + + +### change_email(payload) + + + + +#### Parameters + +* payload + + + + + + + +### set_user_receive_mail_notifications(do_receive_notifications) + + + + +#### Parameters + +* do_receive_notifications + + + + + + + +### set_user_document_format_preference(doc_preference) + + + + +#### Parameters + +* doc_preference + + + + + + + +### get_user_details() + + + + + + + + + +### get_user_annotated_projects() + + +::: tip Description +Gets a list of projects that the user has annotated +::: + + + + + + + + +### get_user_annotations_in_project(project_id,current_page,page_size) + + +::: tip Description +Gets a list of documents in a project where the user has performed annotations in. + :param project_id: The id of the project to query + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + + + + + + + +### create_project() + + + + + + + + + +### delete_project(project_id) + + + + +#### Parameters + +* project_id + + + + + + + +### update_project(project_dict) + + + + +#### Parameters + +* project_dict + + + + + + + +### get_project(project_id) + + + + +#### Parameters + +* project_id + + + + + + + +### clone_project(project_id) + + + + +#### Parameters + +* project_id + + + + + + + +### import_project_config(pk,project_dict) + + + + +#### Parameters + +* pk + +* project_dict + + + + + + + +### export_project_config(pk) + + + + +#### Parameters + +* pk + + + + + + + +### get_projects(current_page,page_size,filters) + + +::: tip Description +Gets the list of projects. Query result can be limited by using current_page and page_size and sorted + by using filters. + + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter option used to search project, currently only string is used to search + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* current_page + +* page_size + +* filters + + + + + + + +### get_project_documents(project_id,current_page,page_size,filters) + + +::: tip Description +Gets the list of documents and its annotations. Query result can be limited by using current_page and page_size + and sorted by using filters + + :param project_id: The id of the project that the documents belong to, is a required variable + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter currently only searches for ID of documents + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + +* filters + + + + + + + +### get_project_test_documents(project_id,current_page,page_size,filters) + + +::: tip Description +Gets the list of documents and its annotations. Query result can be limited by using current_page and page_size + and sorted by using filters + + :param project_id: The id of the project that the documents belong to, is a required variable + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter currently only searches for ID of documents + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + +* filters + + + + + + + +### get_project_training_documents(project_id,current_page,page_size,filters) + + +::: tip Description +Gets the list of documents and its annotations. Query result can be limited by using current_page and page_size + and sorted by using filters + + :param project_id: The id of the project that the documents belong to, is a required variable + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter currently only searches for ID of documents + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + +* filters + + + + + + + +### add_project_document(project_id,document_data) + + + + +#### Parameters + +* project_id + +* document_data + + + + + + + +### add_project_test_document(project_id,document_data) + + + + +#### Parameters + +* project_id + +* document_data + + + + + + + +### add_project_training_document(project_id,document_data) + + + + +#### Parameters + +* project_id + +* document_data + + + + + + + +### add_document_annotation(doc_id,annotation_data) + + + + +#### Parameters + +* doc_id + +* annotation_data + + + + + + + +### get_annotations(project_id) + + +::: tip Description +Serialize project annotations as GATENLP format JSON using the python-gatenlp interface. +::: + + + +#### Parameters + +* project_id + + + + + + + +### delete_documents_and_annotations(doc_id_ary,anno_id_ary) + + + + +#### Parameters + +* doc_id_ary + +* anno_id_ary + + + + + + + +### get_possible_annotators(proj_id) + + + + +#### Parameters + +* proj_id + + + + + + + +### get_project_annotators(proj_id) + + + + +#### Parameters + +* proj_id + + + + + + + +### add_project_annotator(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### make_project_annotator_active(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### project_annotator_allow_annotation(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### remove_project_annotator(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### reject_project_annotator(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### get_annotation_timings(proj_id) + + + + +#### Parameters + +* proj_id + + + + + + + +### delete_annotation_change_history(annotation_change_history_id) + + + + +#### Parameters + +* annotation_change_history_id + + + + + + + +### get_annotation_task() + + +::: tip Description +Gets the annotator's current task, returns a dictionary about the annotation task that contains all the information + needed to render the Annotate view. +::: + + + + + + + + +### get_annotation_task_with_id(annotation_id) + + +::: tip Description +Get annotation task dictionary for a specific annotation_id, must belong to the annotator (or is a manager or above) +::: + + + +#### Parameters + +* annotation_id + + + + + + + +### complete_annotation_task(annotation_id,annotation_data,elapsed_time) + + +::: tip Description +Complete the annotator's current task +::: + + + +#### Parameters + +* annotation_id + +* annotation_data + +* elapsed_time + + + + + + + +### reject_annotation_task(annotation_id) + + +::: tip Description +Reject the annotator's current task +::: + + + +#### Parameters + +* annotation_id + + + + + + + +### change_annotation(annotation_id,new_data) + + +::: tip Description +Adds annotation data to history +::: + + + +#### Parameters + +* annotation_id + +* new_data + + + + + + + +### get_document(document_id) + + +::: tip Description +Obsolete: to be deleted +::: + + + +#### Parameters + +* document_id + + + + + + + +### get_annotation(annotation_id) + + +::: tip Description +Obsolete: to be deleted +::: + + + +#### Parameters + +* annotation_id + + + + + + + +### annotator_leave_project() + + +::: tip Description +Allow annotator to leave their currently associated project. +::: + + + + + + + + +### get_all_users() + + + + + + + + + +### get_user(username) + + + + +#### Parameters + +* username + + + + + + + +### admin_update_user(user_dict) + + + + +#### Parameters + +* user_dict + + + + + + + +### admin_update_user_password(username,password) + + + + +#### Parameters + +* username + +* password + + + + + + + +### get_endpoint_listing() + + + + + + + + + + + diff --git a/docs/versioned/0.4.0/developerguide/documentation.md b/docs/versioned/0.4.0/developerguide/documentation.md new file mode 100644 index 00000000..8f84e1bb --- /dev/null +++ b/docs/versioned/0.4.0/developerguide/documentation.md @@ -0,0 +1,53 @@ +# Managing and versioning documentation + +Documentation versioning is managed by the custom node script located at `docs/manage_versions.js`. Versions of the documentation can be archived and the entire documentation site can be built using the script. + +Various configuration parameters used for management of documentation versioning can be found in `docs/docs.config.js`. + +## Editing the documentation + +The latest version of the documentation is located at `/docs/docs`. The archived (versioned) documentation are located in `/docs/versioned/version_number`. + +Use the following command to live preview the latest version of the documentation: + +``` +npm run serve:docs +``` + +Note that this will not work with other versioned docs as they are managed as a separate site. To live preview versioned documentation use the command (replace version_num with the version you'd like to preview): + +``` +vuepress dev docs/versioned/version_num +``` + +## Creating a new documentation version + +To create a version of the documentation, run the command: + +``` +npm run docs:create_version +``` + +This creates a copy of the current set of documentation in `/docs/docs` and places it at `/docs/versioned/version_num`. The version number in `package.json` is used for the documentation version. + +Each set of documentation can be considered as a separate vuepress site. Each one has a `.vuepress/versions.json` file that contains the listing of all versions, allowing them to link to each other. + +Note: Versions can also be created manually by running the command: + +``` +# Replace version_num with the version you'd like to create +node docs/manage_versions.js create version_num +``` + + +## Building documentation site + +To build the documentation site, the previous documentation build command is used: + +``` +npm run build:docs +``` + +## Implementation of the version selector UI + +A partial override of the default Vuepress theme was needed to add a custom component the navigation bar. The modified version of the `NavBar` component can be found in `/docs/docs/.vuepress/theme/components/NavBar.vue`. The modified NavBar uses the `VersionSelector` (`/docs/docs/.vuepress/theme/components/VersionSelector.vue`) component which reads from the `.vuepress/versions.json` from each set of documentation. diff --git a/docs/versioned/0.4.0/developerguide/releases.md b/docs/versioned/0.4.0/developerguide/releases.md new file mode 100644 index 00000000..ec14d1f2 --- /dev/null +++ b/docs/versioned/0.4.0/developerguide/releases.md @@ -0,0 +1,18 @@ +# Managing Releases + +*These instructions are primarily intended for the maintainers of Teamware.* + +Note: Releases are always made from the `master` branch of the repository. + +## Steps to making a release + +1. **Update the changelog** - This has to be done manually, go through any pull requests to `dev` since the last release. + - In github pull requests page, use the search term `is:pr merged:>=yyyy-mm-dd` to find all merged PR from the date since the last version change. + - Include the changes in the `CHANGELOG.md` file; the changelog section _MUST_ begin with a level-two heading that starts with the relevant version number in square brackets (`## [N.M.P] Optional descriptive suffix`) as the GitHub workflow that creates a release from the eventual tag depends on this pattern to find the right release notes. Each main item within the changelog should have a link to the originating PR e.g. \[#123\](https://github.com/GateNLP/gate-teamware/pull/123). +1. **Update and check the version numbers** - from the teamware directory run `python version.py check` to check whether all version numbers are up to date. If not, update the master `VERSION` file and run `python version.py update` to update all other version numbers and commit the result. Note that `version.py` requires `pyyaml` for reading `CITATION.cff`, `pyyaml` is included in Teamware's dependencies. +1. **Create a version of the documentation** - Run `npm run docs:create_version`, this will archive the current version of the documentation using the version number in `package.json`. +1. **Create a pull request from `dev` to `master`** including any changes to `CHANGELOG.md`, `VERSION`. +1. **Create a tag** - Once the dev-to-master pull request has been merged, create a tag from the resulting `master` branch named `vN.M.P` (i.e. the new version number prefixed with the letter `v`). This will trigger two GitHub workflows: + - one that builds versioned Docker images for this release and pushes them to `ghcr.io`, updating the `latest` image tag to point to the new release + - one that creates a "release" on GitHub with the necessary artifacts to make the `https://gate.ac.uk/get-teamware.sh` installation mechanism work correctly. The release notes for this release will be generated by extracting the matching section from `CHANGELOG.md`. +1. **Update the Helm chart** - Create a new branch on [https://github.com/GateNLP/charts](https://github.com/GateNLP/charts) to update the `appVersion` of the `gate-teamware` Helm chart to match the version that was just created by the tag workflow. You must also update the chart `version`, bumping the major version number if the new chart is not backwards-compatible with the old. Submit a pull request to the `main` branch, which will publish the new chart when it is merged. diff --git a/docs/versioned/0.4.0/developerguide/testing.md b/docs/versioned/0.4.0/developerguide/testing.md new file mode 100644 index 00000000..578501b2 --- /dev/null +++ b/docs/versioned/0.4.0/developerguide/testing.md @@ -0,0 +1,182 @@ +# Testing +All the tests can be run using the following command: + +```bash +npm run test +``` + +## Backend Testing +Pytest is used for testing the backend. + +```bash +npm run test:backend +``` + +### Backend test files + +* Unit test files are located in `/backend/tests` + +## Frontend testing +[Jest](https://jestjs.io/) is used for frontend testing. +The [Vue testing-library](https://testing-library.com/docs/vue-testing-library/intro/) is used for testing +Vue components. + +```bash +npm run test:frontend +``` + +### Frontend test files + +* Frontend test files are located in `/fontend/tests/unit` and should the extension `.spec.js` + +### Testing JS functions + +```javascript +describe("Description of a group of tests to be run", () =>{ + + beforeAll(() =>{ + //The code here is run before each test + }) + + it("A single test's description", async () =>{ + + // Assertions are done with the expect() function e.g. + let funcOutput = 30 + 10 + expect(funcOutput).toBe(40) + + + }) +}) + +``` + +### Mocking JS classes + +This is an example of a mock harness for the JRPCClient class. + +A mock file is created inside a ``__mock__`` directory placed next to the file that's being mocked, e.g. +for our JRPCClient class at `/frontend/src/jrpc/index.js`, the mock file is `/frontend/src/jrpc/__mock__/index.js`. + + +Inside the mock file `/frontend/src/jrpc/__mock__/index.js`: +```javascript +// Mocking jrpc/index.js +//Mocking the JRPCClient class +//Replacing the call function with a custom mockCall function +export const mockCall = jest.fn(()=> 30); +const mock = jest.fn().mockImplementation(() => { + return {call: mockCall}; +}); + +export default mock; +``` + + +Inside the test file `*.spec.js`: +```javascript +import JRPCClient from "@/jrpc"; +jest.mock('@/jrpc') + +import store from '@/store' +//Example on how to mock the jrpc call + +describe("Vuex functions testing", () =>{ + + beforeAll(() =>{ + + //Re-implement custom mock call implementation if needed + JRPCClient.mockImplementation(()=>{ + return { + call(){ + return 50 + } + } + }) + + }) + + it("testfunc", async () =>{ + + const noutput = await store.dispatch("testnormal") + expect(noutput).toBe("Hello world") + + const aoutput = await store.dispatch("testasync") + expect(aoutput).toBe("Hello world") + + const rpc = new JRPCClient("/") + const result = await rpc.call("some param") + expect(result).toBe(50) + + }) +}) +``` + +### Testing Vue components + + +```javascript +//Example of how a component could be tested +import { render, fireEvent } from '@testing-library/vue' + + +import HelloWorld from '@/components/HelloWorld.vue' + +//Testing a component e.g. HelloWorld +describe('HelloWorld.vue', () => { + + it('renders props.msg when passed', () => { + const msg = 'new message' + const { getByText } = render(HelloWorld) + + getByText("Installed CLI Plugins") + }) +}) + +``` + + +## Integration testing +[Cypress](https://www.cypress.io/) is used for integration testing. + +The integration settings are located at `teamware/settings/integration.py` + +To run the integration test: +```bash +npm run test:integration +``` + +The test can also be run in **interactive mode** using: + +```bash +npm run serve:cypressintegration +``` + +### Integration test files +Files related to integration testing are located in `/cypress` + +* Test files are located in the `/cypress/integration` directory and should have the extension `.spec.js`. + +### Re-seeding the database + +The command `npm run migrate:integration` resets the database and performs migration, use with `beforeEach` to run it +before every test case in a suite: + +```js +describe('Example test suite', () => { + + beforeEach(() => { + // Resets the database every time before + // the test is run + cy.exec('npm run migrate:integration') + }) + + it('Test case 1', () => { + // Test something + }) + + it('Test case 2', () => { + // Test something + }) +}) +``` + diff --git a/docs/versioned/0.4.0/img/gate-teamware-logo.svg b/docs/versioned/0.4.0/img/gate-teamware-logo.svg new file mode 100644 index 00000000..12385947 --- /dev/null +++ b/docs/versioned/0.4.0/img/gate-teamware-logo.svg @@ -0,0 +1,79 @@ + + + + diff --git a/docs/versioned/0.4.0/manageradminguide/README.md b/docs/versioned/0.4.0/manageradminguide/README.md new file mode 100644 index 00000000..7a70a19f --- /dev/null +++ b/docs/versioned/0.4.0/manageradminguide/README.md @@ -0,0 +1,45 @@ +# GATE Teamware Overview + +## User roles + +There are three types of users in GATE Teamware, [annotators](#annotators), [managers](#managers) +and [admins](#admins). + +### Annotators + +Annotator is the default role when signing up to Teamware. An annotator can be recruited into +annotation projects and annotate documents. + + +### Managers + +Managers can create, view and modify annotation projects. They can also recruit annotators to a project. + +### Admins + +Admins, on top of what managers can do, they can also manage the users in the system and elevate them as +managers or admins. + +## Annotation Projects, Documents and Annotations + +Projects, documents and annotations form the core of the application. + +### Projects + +An annotation project contains a configuration of how annotations are to be captured, the documents and its +annotations and the recruited annotators. + + +### Documents + +A document in application refers to an individual set of arbitrary text that's to be annotated. A document +is stored as arbitrary JSON object and can represent various things such as, a single post (e.g. a tweet +or a post from reddit), a pair of source post and reply or a part of a HTML web page. + + +### Annotations + +An annotation represents a single annotation task against a single document. Like the document, +an annotation is stored as an arbitrary JSON object and can have any arbitrary structure. + + diff --git a/docs/versioned/0.4.0/manageradminguide/annotators_management.md b/docs/versioned/0.4.0/manageradminguide/annotators_management.md new file mode 100644 index 00000000..cb4c79b0 --- /dev/null +++ b/docs/versioned/0.4.0/manageradminguide/annotators_management.md @@ -0,0 +1,13 @@ +# Annotators management + +The **Annotators** tab in the **Project management** page allows the viewing and management of annotators in the project. + +Add annotators to the project by clicking on the list of names in the right column. Current annotators +can be removed by clicking on the names in the left column. Removing annotators does not delete their +completed annotations but will stop their current pending annotation task. + +An annotator can only be recruited into **one project at a time**. + +Once an annotator has annotated a proportion of documents in the project (specified in project configuration), they will +be deemed to have completed all their annotation tasks and automatically be removed the project. This frees them to be +recruited in another project. diff --git a/docs/versioned/0.4.0/manageradminguide/config_examples.js b/docs/versioned/0.4.0/manageradminguide/config_examples.js new file mode 100644 index 00000000..e167b81e --- /dev/null +++ b/docs/versioned/0.4.0/manageradminguide/config_examples.js @@ -0,0 +1,251 @@ +export default { + config1: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + } + ], + config2: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + }, + { + "name": "opinion", + "type": "text", + "title": "What's your opinion of the above text?", + "optional": true + } + ], + configDisplay: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + } + ], + configDisplayHtmlNoHtml: [ + { + "name": "htmldisplay", + "type": "html", + "text": "No HTML: {{text}}
HTML: {{{text}}}" + } + ], + configDisplayCustomFieldnames: [ + { + "name": "htmldisplay", + "type": "html", + "text": "Custom field: {{customField}}
Another custom field: {{{anotherCustomField}}}
Subfield: {{{subfield.subfieldContent}}}" + } + ], + configDisplayPreserveNewlines: [ + { + "name": "htmldisplay", + "type": "html", + "text": "
{{text}}
" + } + ], + configTextInput: [ + { + "name": "mylabel", + "type": "text", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configTextarea: [ + { + "name": "mylabel", + "type": "textarea", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configRadio: [ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "orientation": "vertical", //Optional - default is "horizontal" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configCheckbox: [ + { + "name": "mylabel", + "type": "checkbox", + "optional": true, //Optional - Set if validation is not required + "orientation": "horizontal", //Optional - "horizontal" (default) or "vertical" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "minSelected": 1, //Optional - Specify the minimum number of options that must be selected + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configSelector: [ + { + "name": "mylabel", + "type": "selector", + "optional": true, //Optional - Set if validation is not required + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configRadioDict: [ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "options": { // The options can be specified as a dictionary, ordering is not guaranteed + "value1": "Text to show user 1", + "value2": "Text to show user 2", + "value3": "Text to show user 3", + }, + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + + configDbpediaExample: [ + { + "name": "uri", + "type": "radio", + "title": "Select the most appropriate URI", + "options":[ + {"fromDocument": "candidates"}, + {"value": "none", "label": "None of the above"}, + {"value": "unknown", "label": "Cannot be determined without more context"} + ] + } + ], + docDbpediaExample: { + "text": "President Bush visited the air base yesterday...", + "candidates": [ + { + "value": "http://dbpedia.org/resource/George_W._Bush", + "label": "George W. Bush (Jnr)" + }, + { + "value": "http://dbpedia.org/resource/George_H._W._Bush", + "label": "George H. W. Bush (Snr)" + } + ] + }, + + + doc1: {text: "Sometext with html"}, + doc2: { + customField: "Content of custom field.", + anotherCustomField: "Content of another custom field.", + subfield: { + subfieldContent: "Content of a subfield." + } + }, + docPlainText: { + "text": "This is some text\n\nIt has line breaks that we want to preserve." + }, + configPreAnnotation: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "radio", + "type": "radio", + "title": "Test radio input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test radio description" + }, + { + "name": "checkbox", + "type": "checkbox", + "title": "Test checkbox input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test checkbox description" + }, + { + "name": "text", + "type": "text", + "title": "Test text input", + "description": "Test text description" + } + + ], + docPreAnnotation: { + "id": 12345, + "text": "Example document text", + "preannotation": { + "radio": "val1", + "checkbox": ["val1", "val3"], + "text": "Pre-annotation text value" + } + } + + +} diff --git a/docs/versioned/0.4.0/manageradminguide/documents_annotations_management.md b/docs/versioned/0.4.0/manageradminguide/documents_annotations_management.md new file mode 100644 index 00000000..ac010c9f --- /dev/null +++ b/docs/versioned/0.4.0/manageradminguide/documents_annotations_management.md @@ -0,0 +1,272 @@ +# Documents & Annotations + +The **Documents & Annotations** tab in the **Project management** page allows the viewing and management of documents +and annotations related to the project. + +## Document & Annotation status + +### Annotation status + +Annotations can be in 1 of 5 states: + +* Annotation is completed - The annotator has completed this annotation task. +* Annotation is rejected - The annotator has chosen to not annotate the document. +* Annotation is timed out - The annotation task was not completed within the time specified in the project's configuration. The task is freed and can be assigned to another annotator. +* Annotation is aborted - The annotation task was aborted due to reasons other than timing out, such as when an annotator with a pending task is removed from a project. +* Annotation is pending - The annotator has started the annotation task but has not completed it. + +### Document status + +Documents also display a list of its current annotation status: + +* 1 - Number of completed annotations in the document. +* 1 - Number of rejected annotations in the document. +* 1 - Number of timed out annotations in the document. +* 1 - Number of aborted annotations in the document. +* 1 - Number of pending annotations in the document. + +## Importing documents + +Documents can be imported using the **Import** button. The supported file types are: + +* `.json` - The app expects a list of documents (represented as a dictionary object) + e.g. `[{"id": 1, "text": "Text1"}, ...]`. +* `.jsonl` - The app expects one document (represented as a dictionary object) per line. +* `.csv` - File must have a header row. It will be internally converted to JSON format. +* `.zip` - Can contain any number of `.json,.jsonl and .csv` files inside. + +### Importing documents with pre-annotation + +In the `Project Configurations` page, it is possible to set a field in which Teamware will look for pre-annotation. If +the field is found inside the document then the annotation form will be pre-filled with data provided in the document. + +The format for pre-annotation is exactly the same as the annotation output. You can see an example of generated +annotation by filling out the form in the `Annotation Preview` and observing the values in +the `Annotation Output Preview`. + + +For an example project configuration shown below, there are three captured labels named `radio`, `checkbox` and `text`: + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "radio", + "type": "radio", + "title": "Test radio input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test radio description" + }, + { + "name": "checkbox", + "type": "checkbox", + "title": "Test checkbox input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test checkbox description" + }, + { + "name": "text", + "type": "text", + "title": "Test text input", + "description": "Test text description" + } +] +``` + +On the `Project Configuration` page, if the `Pre-annotation` field is set to `preannotation`, the annotation form will be pre-filled with +the content provided in the `preannotation` field of the document e.g.: + +```json +{ + "id": 12345, + "text": "Example document text", + "preannotation": { + "radio": "val1", + "checkbox": [ + "val1", + "val3" + ], + "text": "Pre-annotation text value" + } +} +``` + +The example of the pre-filled form can be seen by clicking on the `Preview` tab above. + + + + + + +### Importing Training and Test documents + +When importing documents for the training and testing phase, Teamware expects a field/column (called `gold` by default) +that contains the correct annotation response for each label and, only for training documents, an explanation. + +For example, if we're expecting a multi-choice label for doing sentiment classification with a widget named `sentiment` +and choice of `postive`, `negative` and `neutrual`: + +```js +[ + { + "text": "What's my sentiment", + "gold": { + "sentiment": { + "value": "positive", // For this document, the correct value is postive + "explanation": "Because..." // Explanation is only given in the traiing phase and are optional in the test documents + } + } + } +] +``` + +in csv: + +| text | gold.sentiment.value | gold.sentiment.explanation | +| --- | --- | --- | +| What's my sentiment | positive | Because... | + +### Guidance on CSV column headings + +It is recommended that: + +* Spaces are not used in column headings, use dash (`-`), underscore (`_`) or camel case (e.g. fieldName) instead. +* The dot/full stop (`.`) is used to indicate hierarchical information so don't use it if that's not what's intended. + Explanation on this feature is given below. + +Documents imported from a CSV files are converted to JSON for use internally in Teamware, the reverse is true when +converting back to CSV. To allow a CSV to represent a hierarchical structure, a dot notation is used to indicate a +sub-field. + +In the following example, we can see that `gold` has a child field named `sentiment` which then has a child field +named `value`: + +| text | gold.sentiment.value | gold.sentiment.explanation | +| --- | --- | --- | +| What's my sentiment | positive | Because... | + +The above column headers will generate the following JSON: + +```js +[ + { + "text": "What's my sentiment", + "gold": { + "sentiment": { + "value": "positive", // For this document, the correct value is postive + "explanation": "Because..." // Explanation is only given in the traiing phase and are optional in the test documents + } + } + } +] +``` + +## Exporting documents + +Documents and annotations can be exported using the **Export** button. A zip file is generated containing files with 500 +documents each. You can choose how documents are exported: + +* `.json` & `.jsonl` - JSON or JSON Lines files can be generated in the format of: + * `raw` - Exports unmodified JSON. If you've originally uploaded in GATE format then choose this option. + + An additional field named `annotation_sets` is added for storing annotations. The annotations are laid out in the + same way as GATE JSON format. For example if a document has been annotated by `user1` with labels and values + `text`:`Annotation text`, `radio`:`val3`, and `checkbox`:`["val2", "val4"]`: + + ```json + { + "id": 32, + "text": "Document text", + "text2": "Document text 2", + "feature1": "Feature text", + "annotation_sets":{ + "user1":{ + "name":"user1", + "annotations":[ + { + "type":"Document", + "start":0, + "end":10, + "id":0, + "features":{ + "label":{ + "text":"Annotation text", + "radio":"val3", + "checkbox":[ + "val2", + "val4" + ] + } + } + } + ], + "next_annid":1 + } + } + } + ``` + + * `gate` - Convert documents to GATE JSON format and export. A `name` field is added that takes the ID value from the + ID field specified in the project configuration. Fields apart from `text` and the ID field specified in the project + config are placed in the `features` field. An `annotation_sets` field is added for storing annotations. + + For example in the case of this uploaded JSON document: + ```json + { + "id": 32, + "text": "Document text", + "text2": "Document text 2", + "feature1": "Feature text" + } + ``` + The generated output is as follows. The annotations are formatted same as the `raw` output above: + ```json + { + "name": 32, + "text": "Document text", + "features": { + "text2": "Document text 2", + "feature1": "Feature text" + }, + "offset_type":"p", + "annotation_sets": {...} + } + ``` +* `.csv` - The JSON documents will be flattened to csv's column based format. Annotations are added as additional + columns with the header of `annotations.username.label`. + +## Deleting documents and annotations + +It is possible to click on the top left of corner of documents and annotations to select it, then click on the +**Delete** button to delete them. + +::: tip + +Selecting a document also selects all its associated annotations. + +::: + + + diff --git a/docs/versioned/0.4.0/manageradminguide/project_config.md b/docs/versioned/0.4.0/manageradminguide/project_config.md new file mode 100644 index 00000000..99b26e34 --- /dev/null +++ b/docs/versioned/0.4.0/manageradminguide/project_config.md @@ -0,0 +1,495 @@ +--- +sidebarDepth: 3 +--- + +# Project configuration + +The **Configuration** tab in the **Project management** page allows you to change project settings including what +annotations are captured. + +Project configurations can be imported and exported in the format of a JSON file. + +The project can be also be cloned (have configurations copied to a new project). Note that cloning does not copy +documents, annotations or annotators to the new project. + +## Configuration fields + +* **Name** - The name of this annotation project. +* **Description** - The description of this annotation project that will be shown to annotators. Supports markdown and + HTML. +* **Annotator guideline** - The description of this annotation project that will be shown to annotators. Supports + markdown and HTML. +* **Annotations per document** - The project completes when each document in this annotation project have this many + number of valid annotations. When a project completes, all project annotators will be un-recruited and be allowed to + annotate other projects. +* **Maximum proportion of documents annotated per annotator (between 0 and 1)** - A single annotator cannot annotate + more than this proportion of documents. +* **Timeout for pending annotation tasks (minutes)** - Specify the number of minutes a user has to complete an + annotation task (i.e. annotating a single document). +* **Reject documents** - Switching this off will mean that annotators for this project will be unable to choose to reject documents. +* **Document ID field** - The field in your uploaded documents that is used as a unique identifier. GATE's json format + uses the name field. You can use a dot limited key path to access subfields e.g. enter features.name to get the id + from the object `{'features':{'name':'nameValue'}}` +* **Training stage enable/disable** - Enable or disable training stage, allows testing documents to be uploaded to the project. +* **Test stage enable/disable** - Enable or disable testing stage, allows test documents to be uploaded to the project. +* **Auto elevate to annotator** - The option works in combination with the training and test stage options, see table below for the behaviour: + + | Training stage | Testing stage | Auto elevate to annotator | Desciption | + | --- | --- | --- | --- | + | Disabled | Disabled | Enabled/Disabled | User allowed to annotate without manual approval. | + | Enabled | Disabled | Disabled | Manual approval required. | + | Disabled | Enabled | Disabled | " | + | Enabled | Disabled | Enabled | User always allowed to annotate after training phase completed | + | Disabled | Enabled | Enabled | User automatically allowed to annotate after passing test, if user fails test they have to be manually approved. | + | Enabled | Enabled | Enabled | " | + +* **Test pass proportion** - The proportion of correct test annotations to be automatically allowed to annotate documents. +* **Gold standard field** - The field in document's JSON/column that contains the ideal annotation values and explanation for the annotation. +* **Pre-annotation** - Pre-fill the form with annotation provided in the specified field. See [Importing Documents with pre-annotation](./documents_annotations_management.md#importing-documents-with-pre-annotation) section for more detail. + +## Anotation configuration + +The annotation configuration takes a `json` string for configuring how the document is displayed to the user and types +of annotation will be collected. Here's an example configuration and a preview of how it is shown to annotators: + + + + +```json +// Example configuration +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + } +] +``` + + + +Within the configuration, it is possible to specify how your documents will be displayed. The **Document input preview** +box can be used to provide a sample of your document for rendering of the preview. + +```json +// Example contents for the Document input preview +{ + "text": "Sometext with html" +} +``` + + + +The above configuration displays the value from the `text` field from the document to be annotated. It then shows a set +of 3 radio inputs that allows the user to select a Negative, Neutral, or Positive sentiment with the label +name `sentiment`. + + + +All fields **require** the properties **name** and **type**, it is used to name our label and determine the type of +input/display to be shown to the user respectively. + +Another field can be added to collect more information, e.g. a text field for opinions: + + + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + }, + { + "name": "opinion", + "type": "text", + "title": "What's your opinion of the above text?", + "optional": true + } + +] +``` + + + +Note that for the above case, the `optional` field is added ensure that allows user to not have to input any value. +This `optional` field can be used on all components. + +Some fields are available to configure which are specific to components, e.g. the `options` field are only available for +the `radio`, `checkbox` and `selector` components. See details below on the usage of each specific component. + +The captured annotation results in a JSON dictionary, an example can be seen in the **Annotation output preview** box. +The annotation is linked to a Document and is converted to a GATE JSON annotation format when exported. + +### Displaying text + + + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" // The text that will be displayed + } +] +``` + + + +The `htmldisplay` widget allows you to display the text you want annotated. It accepts almost full range of HTML +input which gives full styling flexibility. + +Any field/column from the document can be inserted by surrounding a field/column name with double or +triple curly brackets. Double curly brackets renders text as-is and triple curly brackets accepts HTML string: + + + +Input: + +```json +{ + "text": "Sometext with html" +} +``` + +Configuration, showing the same field/column in document as-is or as HTML: +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "No HTML: {{text}}
HTML: {{{text}}}" + } +] +``` + +
+ +The widget makes no assumption about your document structure and any field/column names can be used, +even sub-fields by using the dot notation e.g. `parentField.childField`: + + + +JSON input: + +```json +{ + "customField": "Content of custom field.", + "anotherCustomField": "Content of another custom field.", + "subfield": { + "subfieldContent": "Content of a subfield." + } +} +``` + +or in csv + +| customField | anotherCustomField | subfield.subfieldContent | +| --- | --- | --- | +| Content of custom field. | Content of another custom field. | Content of a subfield. | + + +Configuration, showing the same field/column in document as-is or as HTML: +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "Custom field: {{customField}}
Another custom field: {{{anotherCustomField}}}
Subfield: {{{subfield.subfieldContent}}}" + } +] +``` + +
+ +If your documents are plain text and include line breaks that need to be preserved when rendering, this can be achieved by using a special HTML wrapper which sets the [`white-space` CSS property](https://developer.mozilla.org/en-US/docs/Web/CSS/white-space). + + + +**Document** + +```json +{ + "text": "This is some text\n\nIt has line breaks that we want to preserve." +} +``` + +**Project configuration** + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "
{{text}}
" + } +] +``` + +
+ +`white-space: pre-line` preserves line breaks but collapses other whitespace down to a single space, `white-space: pre-wrap` would preserve all whitespace including indentation at the start of a line, but would still wrap lines that are too long for the available space. + +### Text input + + + +```json +[ + { + "name": "mylabel", + "type": "text", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Textarea input + + + +```json +[ + { + "name": "mylabel", + "type": "textarea", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Radio input + + + +```json +[ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "orientation": "vertical", //Optional - default is "horizontal" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Checkbox input + + + +```json +[ + { + "name": "mylabel", + "type": "checkbox", + "optional": true, //Optional - Set if validation is not required + "orientation": "horizontal", //Optional - "horizontal" (default) or "vertical" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "minSelected": 1, //Optional - Overrides optional field. Specify the minimum number of options that must be selected + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Selector input + + + +```json +[ + { + "name": "mylabel", + "type": "selector", + "optional": true, //Optional - Set if validation is not required + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Alternative way to provide options for radio, checkbox and selector + +A dictionary (key value pairs) and also be provided to the `options` field of the radio, checkbox and selector widgets +but note that the ordering of the options are **not guaranteed** as javascript does not sort dictionaries by +the order in which keys are added. + + + +```json +[ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "options": { // The options can be specified as a dictionary, ordering is not guaranteed + "value1": "Text to show user 1", + "value2": "Text to show user 2", + "value3": "Text to show user 3" + }, + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Dynamic options for radio, checkbox and selector + +All the examples above have a "static" list of available options for the radio, checkbox and selector widgets, where the complete options list is enumerated in the project configuration and every document offers the same set of options. However it is also possible to take some or all of the options from the _document_ data rather than the _configuration_ data. For example: + + + +**Project configuration** + +```json +[ + { + "name": "uri", + "type": "radio", + "title": "Select the most appropriate URI", + "options":[ + {"fromDocument": "candidates"}, + {"value": "none", "label": "None of the above"}, + {"value": "unknown", "label": "Cannot be determined without more context"} + ] + } +] +``` + +**Document** + +```json +{ + "text": "President Bush visited the air base yesterday...", + "candidates": [ + { + "value": "http://dbpedia.org/resource/George_W._Bush", + "label": "George W. Bush (Jnr)" + }, + { + "value": "http://dbpedia.org/resource/George_H._W._Bush", + "label": "George H. W. Bush (Snr)" + } + ] +} +``` + + + +`"fromDocument"` is a dot-separated property path leading to the location within each document where the additional options can be found, for example `"fromDocument":"candidates"` looks for a top-level property named `candidates` in each document, `"fromDocument": "options.custom"` would look for a property named `options` which is itself an object with a property named `custom`. The target property in the document may be in any of the following forms: + +- an array _of objects_, each with `value` and `label` properties, exactly as in the static configuration format - this is the format used in the example above +- an array _of strings_, where the same string will be used as both the value and the label for that option +- an arbitrary ["dictionary"](#options-as-dict) object mapping values to labels +- a _single string_, which is parsed into a list of options + +The "single string" alternative is designed to be easier to use when [importing documents](documents_annotations_management.md#importing-documents) from CSV files. It allows you to provide any number of options in a _single_ CSV column value. Within the column the options are separated by semicolons, and each option is of the form `value=label`. Whitespace around the delimiters is ignored, both between options and between the value and label of a single option. For example given CSV document data of + +| text | options | +|-----------------|---------------------------------------------------| +| Favourite fruit | `apple=Apples; orange = Oranges; kiwi=Kiwi fruit` | + +a `{"fromDocument": "options"}` configuration would produce the equivalent of + +```json +[ + {"value": "apple", "label": "Apples"}, + {"value": "orange", "label": "Oranges"}, + {"value": "kiwi", "label": "Kiwi fruit"} +] +``` + +If your values or labels may need to contain the default separator characters `;` or `=` you can select different separators by adding extra properties to the configuration: + +```json +{"fromDocument": "options", "separator": "~~", "valueLabelSeparator": "::"} +``` + +| text | options | +|-----------------|------------------------------------------------------| +| Favourite fruit | `apple::Apples ~~ orange::Oranges ~~ kiwi::Kiwi fruit` | + +The separators can be more than one character, and you can set `"valueLabelSeparator":""` to disable label splitting altogether and just use the value as its own label. + +### Mixing static and dynamic options + +Static and `fromDocument` options may be freely interspersed in any order, so you can have a fully-dynamic set of options by specifying _only_ a `fromDocument` entry with no static options, or you can have static options that are listed first followed by dynamic options, or dynamic options first followed by static, etc. + + diff --git a/docs/versioned/0.4.0/manageradminguide/project_management.md b/docs/versioned/0.4.0/manageradminguide/project_management.md new file mode 100644 index 00000000..f1fd0cfe --- /dev/null +++ b/docs/versioned/0.4.0/manageradminguide/project_management.md @@ -0,0 +1,38 @@ +# Annotation Project Management + +## Project Listing +Clicking on the `Projects` link in the top navigation bar takes you to a contains a list of existing +projects. The project names are shown along with their summaries. Clicking on a project name will +take you to the project management page. + + +## Project Management Page + +The project management page contains all the functionalities to manage an annotation project. The page +is composed of three main tabs: + +* [Configuration](project_config.md) - Configure project settings including what annotations are captured. +* [Documents & Annotation](documents_annotations_management.md) - Manage documents and annotations. Upload documents, see contents of a document's annotations and import/export documents. +* [Annotators](annotators_management.md) - Manage the recruitment of annotators. + +::: warning + +Annotators can only be recruited to an annotation project after it has been configured and documents +are uploaded to the project. + +::: + + +## Project status icons +In the **Project listing** and **Project management page**, icon badges are used to provide a quick overview of the project's status: + +* 1 - Number of completed annotations in the project. +* 1 - Number of rejected annotations in the project. +* 1 - Number of timed out annotations in the project. +* 1 - Number of aborted annotations in the project. +* 1 - Number of pending annotations in the project. +* 2/60 - Number of occupied annotation tasks over number of total tasks in the project. +* 20/5/10 - Number of documents, training documents and test documents in the project. +* 1 - Number of annotators recruited in the project. Annotators are removed from the project when they have completed all annotation tasks in their quota. + + diff --git a/docs/versioned/2.0.0/.vuepress/components/AnnotationRendererPreview.vue b/docs/versioned/2.0.0/.vuepress/components/AnnotationRendererPreview.vue new file mode 100644 index 00000000..868d9c54 --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/components/AnnotationRendererPreview.vue @@ -0,0 +1,85 @@ + + + + + diff --git a/docs/versioned/2.0.0/.vuepress/components/DisplayVersion.vue b/docs/versioned/2.0.0/.vuepress/components/DisplayVersion.vue new file mode 100644 index 00000000..03ec07ed --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/components/DisplayVersion.vue @@ -0,0 +1,21 @@ + + + + + diff --git a/docs/versioned/2.0.0/.vuepress/config.js b/docs/versioned/2.0.0/.vuepress/config.js new file mode 100644 index 00000000..36b12c03 --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/config.js @@ -0,0 +1,41 @@ +const versionData = require("./versions.json") +const path = require("path"); +module.exports = context => ({ + title: 'GATE Teamware Documentation', + description: 'Documentation for GATE Teamware', + base: versionData.base, + themeConfig: { + nav: [ + {text: 'Home', link: '/'}, + {text: 'Annotators', link: '/annotatorguide/'}, + {text: 'Managers & Admins', link: '/manageradminguide/'}, + {text: 'Developer', link: '/developerguide/'} + ], + sidebar: { + '/manageradminguide/': [ + "", + "project_management", + "project_config", + "documents_annotations_management", + "annotators_management" + ], + '/developerguide/': [ + '', + 'testing', + 'releases', + 'documentation', + "api_docs", + + ], + }, + }, + configureWebpack: { + resolve: { + alias: { + '@': path.resolve(__dirname, versionData.frontendSource) + } + } + }, + + +}) diff --git a/docs/versioned/2.0.0/.vuepress/enhanceApp.js b/docs/versioned/2.0.0/.vuepress/enhanceApp.js new file mode 100644 index 00000000..e7aadadf --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/enhanceApp.js @@ -0,0 +1,17 @@ +import Vue from 'vue' +import {BootstrapVue, BootstrapVueIcons, IconsPlugin} from 'bootstrap-vue' + +import 'bootstrap/dist/css/bootstrap.css' +import 'bootstrap-vue/dist/bootstrap-vue.css' + +Vue.use(BootstrapVue) +Vue.use(BootstrapVueIcons) + +export default ({ + Vue, // the version of Vue being used in the VuePress app + options, // the options for the root Vue instance + router, // the router instance for the app + siteData // site metadata +}) => { + +} diff --git a/docs/versioned/2.0.0/.vuepress/theme/components/Navbar.vue b/docs/versioned/2.0.0/.vuepress/theme/components/Navbar.vue new file mode 100644 index 00000000..c3b966db --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/theme/components/Navbar.vue @@ -0,0 +1,143 @@ + + + + + diff --git a/docs/versioned/2.0.0/.vuepress/theme/components/VersionSelector.vue b/docs/versioned/2.0.0/.vuepress/theme/components/VersionSelector.vue new file mode 100644 index 00000000..4cfb5eb9 --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/theme/components/VersionSelector.vue @@ -0,0 +1,33 @@ + + + + + diff --git a/docs/versioned/2.0.0/.vuepress/theme/index.js b/docs/versioned/2.0.0/.vuepress/theme/index.js new file mode 100644 index 00000000..b91b8a57 --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/theme/index.js @@ -0,0 +1,3 @@ +module.exports = { + extend: '@vuepress/theme-default' +} diff --git a/docs/versioned/2.0.0/.vuepress/versions.json b/docs/versioned/2.0.0/.vuepress/versions.json new file mode 100644 index 00000000..9f538bce --- /dev/null +++ b/docs/versioned/2.0.0/.vuepress/versions.json @@ -0,0 +1,23 @@ +{ + "current": "2.0.0", + "base": "/gate-teamware/2.0.0/", + "versions": [ + { + "text": "0.3.0", + "value": "/gate-teamware/0.3.0/" + }, + { + "text": "0.4.0", + "value": "/gate-teamware/0.4.0/" + }, + { + "text": "2.0.0", + "value": "/gate-teamware/2.0.0/" + }, + { + "text": "development", + "value": "/gate-teamware/development/" + } + ], + "frontendSource": "../../../../frontend/src" +} \ No newline at end of file diff --git a/docs/versioned/2.0.0/README.md b/docs/versioned/2.0.0/README.md new file mode 100644 index 00000000..3586c291 --- /dev/null +++ b/docs/versioned/2.0.0/README.md @@ -0,0 +1,138 @@ +# GATE Teamware + +![GATE Teamware logo](./img/gate-teamware-logo.svg "GATE Teamware logo") + +A web application for collaborative document annotation. + +This is a documentation for Teamware version: + +## Key Features +* Free and open source software. +* Configure annotation options using a highly flexible JSON config. +* Set limits on proportions of a task that annotators can annotate. +* Import existing annotations as CSV or JSON. +* Export annotations as CSV or JSON. +* Annotation instructions and document rendering supports markdown and HTML. + +## Getting started +A quickstart guide for annotators is [available here](annotatorguide). + +To use an existing instance of GATE Teamware as a project manager or admin, find instructions in the [Managers and Admins guide](manageradminguide). + +Documentation on deploying your own instance can be found in the [Developer Guide](developerguide). + +## Installation Guide + +### Quick Start + +The simplest way to deploy your own copy of GATE Teamware is to use Docker Compose on Linux or Mac. Installation on Windows is possible but not officially supported - you need to be able to run `bash` shell scripts for the quick-start installer. + +1. Install Docker - [Docker Engine](https://docs.docker.com/engine/) for Linux servers or [Docker Desktop](https://docs.docker.com/desktop/) for Mac. +2. Install [Docker Compose](https://github.com/docker/compose), if your Docker does not already include it (Compose is included by default with Docker Desktop) +3. Download the [installation script](https://gate.ac.uk/get-teamware.sh) into an empty directory, run it and follow the instructions. + +``` +mkdir gate-teamware +cd gate-teamware +curl -LO https://gate.ac.uk/get-teamware.sh +bash ./get-teamware.sh +``` + +This will make the Teamware application available as `http://localhost:8076`, with the option to expose it as a public `https://` URL if your server is directly internet-accessible - for production use we recommend deploying Teamware with a suitable internet-facing reverse proxy, or use Kubernetes as described below. + +### Deployment using Kubernetes + +A Helm chart to deploy Teamware on Kubernetes is published to the GATE team public charts repository. The chart requires [Helm](https://helm.sh) version 3.7 or later, and is compatible with Kubernetes version 1.23 or later. Earlier Kubernetes versions back to 1.19 _may_ work provided autoscaling is not enabled, but these have not been tested. + +The following quick start instructions assume you have a compatible Kubernetes cluster and a working installation of `kubectl` and `helm` (3.7 or later) with permission to create all the necessary resource types in your target namespace. + +First generate a random "secret key" for the Django application. This must be at least 50 random characters, a quick way to do this is + +``` +# 42 random bytes base64 encoded becomes 56 random characters +kubectl create secret generic -n {namespace} django-secret \ + --from-literal="secret-key=$( openssl rand -base64 42 )" +``` + +Add the GATE charts repository to your Helm configuration: + +``` +helm repo add gate https://repo.gate.ac.uk/repository/charts +helm repo update +``` + +Create a `values.yaml` file with the key settings required for teamware. The following is a minimal set of values for a typical installation: + +```yaml +# Public-facing web hostname of the teamware application, the public +# URL will be https://{hostName} +hostName: teamware.example.com + +email: + # "From" address on emails sent by Teamware + adminAddress: admin@teamware.example.com + # Send email via an SMTP server - alternatively "gmail" to use GMail API + backend: "smtp" + smtp: + host: mail.example.com + # You will also need to set user and passwordSecret if your + # mail server requires authentication + +privacyPolicy: + # Contact details of the host and administrator of the teamware + # instance, if no admin defined, defaults to the host values. + host: + # Name of the host + name: "Service Host" + # Host's physical address + address: "123 Example Street, City. Country." + # A method of contacting the host, field supports HTML for e.g. linking to a form + contact: "Email" + admin: + name: "Dr. Service Admin" + address: "Department of Example Studies, University of Example, City. Country." + contact: "Email" + +backend: + # Name of the random secret you created above + djangoSecret: django-secret + +# Initial "super user" created on the first install. These are just +# the *initial* settings, you can (and should!) change the password +# once Teamware is up and running +superuser: + email: me@example.com + username: admin + password: changeme +``` + +Some of these may be omitted or others may be required depending on the setup of your specific cluster - see the [chart README](https://github.com/GateNLP/charts/blob/main/gate-teamware/README.md) and the chart's own values file (which you can retrieve with `helm show values gate/gate-teamware`) for full details. In particular these values assume: + +- your cluster has an ingress controller, with a default ingress class configured, and that controller has a default TLS certificate that is compatible with your chosen hostname (e.g. a `*.example.com` wildcard) +- your cluster has a default storageClass configured to provision PVCs, and at least 8 GB of available PV capacity +- you can send email via an SMTP server with no authentication +- the default GATE Teamware terms and privacy documents are suitable for your deployment and compliant with the laws of your location. If this is not the case you can supply your own custom policy documents in a ConfigMap +- you do not need to back up your PostgreSQL database - the chart does include the option to store backups in Amazon S3 or another compatible object store, see the full README for details + +Once you have created your values file, you can install the chart or upgrade an existing installation using + +``` +helm upgrade --install gate-teamware gate/gate-teamware \ + --namespace {namespace} --values teamware-values.yaml +``` + + +## Bug reports and feature requests +Please make bug reports and feature requests as Issues on the [GATE Teamware GitHub repo](https://github.com/GATENLP/gate-teamware). + +# Using Teamware +Teamware is developed by the [GATE](https://gate.ac.uk) team, an academic research group at The University of Sheffield. As a result, future funding relies on evidence of the impact that the software provides. If you use Teamware, please let us know using the contact form at [gate.ac.uk](https://gate.ac.uk/g8/contact). Please include details on grants, publications, commercial products etc. Any information that can help us to secure future funding for our work is greatly appreciated. + +## Citation +For published work that has used Teamware, please cite this repository. One way is to include a citation such as: + +> Karmakharm, T., Wilby, D., Roberts, I., & Bontcheva, K. (2022). GATE Teamware (Version 0.1.4) [Computer software]. https://github.com/GateNLP/gate-teamware + +Please use the `Cite this repository` button at the top of the [project's GitHub repository](https://github.com/GATENLP/gate-teamware) to get an up to date citation. + +The Teamware version can be found on the 'About' page of your Teamware instance. diff --git a/docs/versioned/2.0.0/annotatorguide/README.md b/docs/versioned/2.0.0/annotatorguide/README.md new file mode 100644 index 00000000..4a3ddae2 --- /dev/null +++ b/docs/versioned/2.0.0/annotatorguide/README.md @@ -0,0 +1,27 @@ +# Annotators Quickstart + +Annotating a project: + +* After signing up to the site, notify the owner of the annotation project you've been recruited of + your username. This will allow them to add you as an annotator to a project. +* After you've been recruited to a project, click on the `Annotate` link on the navigation bar at the + top of the page to start annotating. +* You will be shown the details about the project you're annotating along with a set of form(s) to capture + your annotation. Ensure you've read the Annotator guideline fully before starting the annotation process. +* You can then start annotating documents one at a time. Click on `Submit` to confirm the completion of + annotation, `Clear` to start again or `Reject` to skip the particular document. Be aware some projects + do not allow you to skip documents. +* Once you've finished annotating a certain number of documents in a project (specified by the project + manager) your task will be deemed complete, and you will be able to be recruited into another annotation + project. + +## Deleting your account + +At any time you can choose to stop participating and delete your account. You can do this by: + +* Click on your username in the top right corner and then `Account`. +* Click on `Delete my account`. +* When deleting your account, by default your personal information will be removed but your annotations will remain on the system. To completely remove all of your annotations, click on the checkbox next to `Also remove any annotations, projects and documents that I own:`. +* Click the `Unlock` button. +* Then click `Delete` to remove your account. + diff --git a/docs/versioned/2.0.0/developerguide/README.md b/docs/versioned/2.0.0/developerguide/README.md new file mode 100644 index 00000000..0696512c --- /dev/null +++ b/docs/versioned/2.0.0/developerguide/README.md @@ -0,0 +1,283 @@ +# Developer guide + +## Architecture +``` +├── .github/workflows/ # github actions workflow files +├── teamware/ # Django project +│   └── settings/ +├── backend/ # Django app +├── charts/ # Helm charts for Kubernetes +├── cypress/ # integration test configurations +├── docs/ # documentation +├── examples/ # example data files +├── frontend/ # all frontend, in VueJS framework +├── nginx/ # Nginx configurations +| +# Top level directory contains scripts for management and deployment, +# main project package.json, python requirements, docker configs +├── build-images.sh +├── deploy.sh +├── create-django-db.sh +├── docker-compose.yml +├── Dockerfile +├── generate-docker-env.sh +├── manage.py +├── migrate-integration.sh +├── package.json +├── package-lock.json +├── pytest.ini +├── README.md +├── requirements-dev.txt +├── requirements.txt +└── run-server.sh + +``` + +## Installation for development + +The service depends on a combination of python and javascript libraries. We recommend developing inside a `conda` conda environment as it is able to install +python libraries and nodejs which is used to install javascript libraries. + +* Install anaconda/miniconda +* Create a blank virtual conda env + ```bash + $ conda create -n teamware python=3.9 + ``` +* Activate conda environment + ```bash + $ source activate teamware + # or + $ conda activate teamware + ``` +* Install python dependencies in conda environment using pip + ```bash + (teamware)$ pip install -r requirements.txt -r requirements-dev.txt + ``` +* Install nodejs, postgresql and openssl in the conda environment + ```bash + (teamware)$ conda install -y -c conda-forge postgresql=14.* + (teamware)$ conda install -y -c conda-forge nodejs=14.* + ``` +* Install nodejs dependencies + ```bash + (teamware)$ npm install + ``` + +Set up a new postgreSQL database and user for development: +``` +# Create a new directory for the db data and initialise +mkdir -p pgsql/data +initdb -D pgsql/data + +# Launch postgres in the background +postgres -p 5432 -D pgsql/data & + +# Create a DB user, you'll be prompted to input password, "password" is the default in teamware/settings/base.py for development +createuser -p 5432 -P user --createdb + +# Create a rumours_db with rumours as user +createdb -p 5432 -O user teamware_db + +# Migrate & create database tables +python manage.py migrate + +# create a new superuser - when prompted enter a username and password for the db superuser +python manage.py createsuperuser +``` + +## Updating packages +To update packages after a merge, run the following commands: + +```bash +# Activate the conda environment +source activate teamware +# Update any packages changed in the python requirements.txt and requirements-dev.txt files +pip install -r requirements.txt -r requirements-dev.txt +# Update any packages changed in package.json +npm install +``` + +## Development server +The application uses django's dev server to serve page contents and run the RPC API, it also uses Vue CLI's +development server to serve dynamic assets such as javascript or stylesheets allowing for hot-reloading +during development. + +To run both servers together: + + ```bash + npm run serve + ``` + +To run separately: + +* Django server + ```bash + npm run serve:backend + ``` +* Vue CLI dev server + ```bash + npm run serve:frontend + ``` + +## Deploying a development version using Docker +Deployment is via [docker-compose](https://docs.docker.com/compose/), using [NGINX](https://www.nginx.com/) to serve static content, a separate [postgreSQL](https://hub.docker.com/_/postgres) service containing the database and a database backup service (see `docker-compose.yml` for details). Pre-built images can be run using most versions of Docker but _building_ images requires `docker buildx`, which means either Docker Desktop or version 19.03 or later of Docker Engine. + +1. Run `./generate-docker-env.sh` to create a `.env` file containing randomly generated secrets which are mounted as environment variables into the container. See [below](#env-config) for details. + +2. Then build the images via: + ```bash + ./build-images.sh + ``` + +3. then deploy the stack with + + ```bash + ./deploy.sh production # (or prod) to deploy with production settings + ./deploy.sh staging # (or stag) to deploy with staging settings + ``` + +To bring the stack down, run `docker-compose down`, using the `-v` flag to destroy the database volume (be careful with this). + +### Configuration using environment variables (.env file) + +To allow the app to be easily configured between instances especially inside containers, many of the app's configuration can be done through environment variables. + +Run `./generate-docker-env.sh` to generate a `.env` file with all configurable environment parameters. + +To set values for your own deployment, add values to the variables in `.env`, most existing values will be kept after running `generate-docker-env.sh`, see comments in `.env` for specific details. Anything that is left blank will be filled with a default value. Passwords and keys are filled with auto-generated random values. + +Existing `.env` files are copied into a new file named `saved-env.` by `generate-docker-env.sh`. + +### Backups + +In a docker-compose based deployment, backups of the database are managed by the service `pgbackups` which uses the [`prodrigestivill/postgres-backup-local:12`](https://hub.docker.com/r/prodrigestivill/postgres-backup-local) image. +By default, backups are taken of the database daily, and the `docker-compose.yml` contains settings for the number of backups kept under the options for the `pgbackups` service. +Backups are stored as a gzipped SQL dump from the database. + +#### Taking a manual backup + +A shell script is provided for manually triggering a backup snapshot. +From the main project directory run + +```sh +$ ./backup_manual.sh +``` + +This uses the `pgbackups` service and all settings and envrionment variables it is configured with in `docker-compose.yml`, so backups will be taken to the same location as configured for the main backup schedule. + +#### Restoring from a backup +1. Locate the backup file (`*.sql.gz`) on your system that you would like to restore from. +2. Make sure that the stack is down, from the main project directory run `docker-commpose down`. +3. Run the backup restore shell script, passing in the path to your backup file as the only argument: + +```sh +$ ./backup_restore.sh path/to/my/backup.sql.gz +``` + +This will first launch the database container, then via Django's `dbshell` command, running in the `backend` service, execute a number of SQL commands before and after running all the SQL from the backup file. + +4. Redeploy the stack, via `./deploy.sh staging` or `./deploy.sh production`, whichever is the case. +5. The database *should* be restored. + +## Configuration + +### Django settings files + +Django settings are located in `teamware/settings` folder. The app will use `base.py` setting by default +and this must be overridden depending on use. + +### Database +A SQLite3 database is used during development and during integration testing. + +For staging and production, postgreSQL is used, running from a `postgres-12` docker container. Settings are found in `teamware/settings/base.py` and `deployment.py` as well as being set as environment variables by `./generate-docker-env.sh` and passed to the container as configured in `docker-compose.yml`. + +In Kubernetes deployments the PostgreSQL database is installed using the Bitnami `postresql` public chart. + + +### Sending E-mail +It's recommended to specify e-mail configurations through environment variables (`.env`). As these settings will include username and passwords that should not be tracked by version control. + +#### E-mail using SMTP +SMTP is supported as standard in Django, add the following configurations with your own details +to the list of environment variables: + +```bash +DJANGO_EMAIL_BACKEND='django.core.mail.backends.smtp.EmailBackend' +DJANGO_EMAIL_HOST='myserver.com' +DJANGO_EMAIL_PORT=22 +DJANGO_EMAIL_HOST_USER='username' +DJANGO_EMAIL_HOST_PASSWORD='password' +``` + +#### E-mail using Google API +The [django-gmailapi-backend](https://github.com/dolfim/django-gmailapi-backend) library +has been added to allow sending of mail through Google's API as sending through SMTP is disabled as standard. + +Unlike with SMTP, Google's API requires OAuth authentication which means a project and a credential has to be +created through Google's cloud console. + +* More information on the Gmail API: [https://developers.google.com/gmail/api/guides/sending](https://developers.google.com/gmail/api/guides/sending) +* OAuth credentials for sending emails: [https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough](https://github.com/google/gmail-oauth2-tools/wiki/OAuth2DotPyRunThrough) + +This package includes the script linked in the documentation above, which simplifies the setup of the API credentials. The following outlines the key steps: + +1. Create a project in the Google developer console, [https://console.cloud.google.com/](https://console.cloud.google.com/) +2. Enable the Gmail API +3. Create OAuth 2.0 credentials, you'll likely want to create a `Desktop` +4. Create a valid refresh_token using the helper script included in the package: + ```bash + gmail_oauth2 --generate_oauth2_token \ + --client_id="" \ + --client_secret="" \ + --scope="https://www.googleapis.com/auth/gmail.send" + ``` +5. Add the created credentials and tokens to the environment variable as shown below: + ```bash + DJANGO_EMAIL_BACKEND='gmailapi_backend.mail.GmailBackend' + DJANGO_GMAIL_API_CLIENT_ID='google_assigned_id' + DJANGO_GMAIL_API_CLIENT_SECRET='google_assigned_secret' + DJANGO_GMAIL_API_REFRESH_TOKEN='google_assigned_token' + ``` + + +#### Teamware Privacy Policy and Terms & Conditions + +Teamware includes a default privacy policy and terms & conditions, which are required for running the application. + +The default privacy policy is intended to be compliant with UK GDPR regulations, which may comply with the rights of users of your deployment, however it is your responsibility to ensure that this is the case. + +If the default privacy policy covers your use case, then you will need to include configuration for a few contact details. + +Contact details are required for the **host** and the **administrator**: the **host** is the organisation or individual responsible for managing the deployment of the teamware instance and the **administrator** is the organisation or individual responsible for managing users, projects and data on the instance. In many cases these roles will be filled by the same organisation or individual, so in this case specifying just the **host** details is sufficient. + +For deployment from source, set the following environment variables: + +* `PP_HOST_NAME` +* `PP_HOST_ADDRESS` +* `PP_HOST_CONTACT` +* `PP_ADMIN_NAME` +* `PP_ADMIN_ADDRESS` +* `PP_ADMIN_CONTACT` + +For deployment using docker-compose, set these values in `.env`. + +If the host and administrator are the same, you can just set the `PP_HOST_*` variables above which will be used for both. + +##### Including a custom Privacy Policy and/or Terms & Conditions + +If the default privacy policy or terms & conditions do not cover your use case, you can easily replace these with your own documents. + +If deploying from source, include markdown (`.md`) files in a `custom-policies` directory in the project root with the exact names `custom-policies/privacy-policy.md` and/or `custom-policies/terms-and-conditions.md` which will be rendered at the corresponding pages on the running web app. If you are not familiar with the Markdown language there are a number of free WYSIWYG-style editor tools available including [StackEdit](https://stackedit.io/app) (browser based) and [Zettlr](https://www.zettlr.com) (desktop app). + +If deploying with docker compose, place the `custom-policies` directory at the same location as the `docker-compose.yml` file before running `./deploy.sh` as above. + +An example custom privacy policy file contents might look like: + +```md +# Organisation X Teamware Privacy Policy +... +... +## Definitions of Roles and Terminology +... +... +``` \ No newline at end of file diff --git a/docs/versioned/2.0.0/developerguide/api_docs.md b/docs/versioned/2.0.0/developerguide/api_docs.md new file mode 100644 index 00000000..03964b37 --- /dev/null +++ b/docs/versioned/2.0.0/developerguide/api_docs.md @@ -0,0 +1,1086 @@ +--- +sidebarDepth: 3 +--- + +# API Documentation + +## Using the JSONRPC endpoints + +::: tip +A single endpoint is used for all API requests, located at `/rpc` +::: + +The API used in the app complies to JSON-RPC 2.0 spec. Requests should always be sent with `POST` and +contain a JSON request object in the body. The response will also be in the form of a JSON object. + +For example, to call the method `subtract(a, b)`. Send `POST` a post request to `/rpc` with the following JSON +in the body: + +```json +{ + "jsonrpc":"2.0", + "method":"subtract", + "params":[ + 42, + 23 + ], + "id":1 +} +``` + +Variables are passed as a list to the `params` field, in this case `a=42` and `b=23`. The `id` field in the top +level of the request object refers to the message ID, this ID value will be matched in the response, +it does not affect the method that is being called. + +The response will be as follows: + +```json +{ + "jsonrpc":"2.0", + "result":19, + "id":1 +} +``` + +In the case of errors, the response will contain an `error` field with error `code` and error `message`: + +```json +{ + "jsonrpc":"2.0", + "error":{ + "code":-32601, + "message":"Method not found" + }, + "id":"1" +} +``` + +The following are error codes used in the app: + +```python +PARSE_ERROR = -32700 +INVALID_REQUEST = -32600 +METHOD_NOT_FOUND = -32601 +INVALID_PARAMS = -32602 +INTERNAL_ERROR = -32603 +AUTHENTICATION_ERROR = -32000 +UNAUTHORIZED_ERROR = -32001 +``` + +## API Listing + + + +### initialise() + + +::: tip Description +Provide the initial context information to initialise the Teamware app + + context_object: + user: + isAuthenticated: bool + isManager: bool + isAdmin: bool + configs: + docFormatPref: bool + global_configs: + allowUserDelete: bool +::: + + + + + + + + +### is_authenticated() + + +::: tip Description +Checks that the current user has logged in. +::: + + + + + + + + +### login(payload) + + + + +#### Parameters + +* payload + + + + + + + +### logout() + + + + + + + + + +### register(payload) + + + + +#### Parameters + +* payload + + + + + + + +### generate_user_activation(username) + + + + +#### Parameters + +* username + + + + + + + +### activate_account(username,token) + + + + +#### Parameters + +* username + +* token + + + + + + + +### generate_password_reset(username) + + + + +#### Parameters + +* username + + + + + + + +### reset_password(username,token,new_password) + + + + +#### Parameters + +* username + +* token + +* new_password + + + + + + + +### change_password(payload) + + + + +#### Parameters + +* payload + + + + + + + +### change_email(payload) + + + + +#### Parameters + +* payload + + + + + + + +### set_user_receive_mail_notifications(do_receive_notifications) + + + + +#### Parameters + +* do_receive_notifications + + + + + + + +### set_user_document_format_preference(doc_preference) + + + + +#### Parameters + +* doc_preference + + + + + + + +### get_user_details() + + + + + + + + + +### get_user_annotated_projects() + + +::: tip Description +Gets a list of projects that the user has annotated +::: + + + + + + + + +### get_user_annotations_in_project(project_id,current_page,page_size) + + +::: tip Description +Gets a list of documents in a project where the user has performed annotations in. + :param project_id: The id of the project to query + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + + + + + + + +### user_delete_personal_information() + + + + + + + + + +### user_delete_account() + + + + + + + + + +### create_project() + + + + + + + + + +### delete_project(project_id) + + + + +#### Parameters + +* project_id + + + + + + + +### update_project(project_dict) + + + + +#### Parameters + +* project_dict + + + + + + + +### get_project(project_id) + + + + +#### Parameters + +* project_id + + + + + + + +### clone_project(project_id) + + + + +#### Parameters + +* project_id + + + + + + + +### import_project_config(pk,project_dict) + + + + +#### Parameters + +* pk + +* project_dict + + + + + + + +### export_project_config(pk) + + + + +#### Parameters + +* pk + + + + + + + +### get_projects(current_page,page_size,filters) + + +::: tip Description +Gets the list of projects. Query result can be limited by using current_page and page_size and sorted + by using filters. + + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter option used to search project, currently only string is used to search + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* current_page + +* page_size + +* filters + + + + + + + +### get_project_documents(project_id,current_page,page_size,filters) + + +::: tip Description +Gets the list of documents and its annotations. Query result can be limited by using current_page and page_size + and sorted by using filters + + :param project_id: The id of the project that the documents belong to, is a required variable + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter currently only searches for ID of documents + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + +* filters + + + + + + + +### get_project_test_documents(project_id,current_page,page_size,filters) + + +::: tip Description +Gets the list of documents and its annotations. Query result can be limited by using current_page and page_size + and sorted by using filters + + :param project_id: The id of the project that the documents belong to, is a required variable + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter currently only searches for ID of documents + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + +* filters + + + + + + + +### get_project_training_documents(project_id,current_page,page_size,filters) + + +::: tip Description +Gets the list of documents and its annotations. Query result can be limited by using current_page and page_size + and sorted by using filters + + :param project_id: The id of the project that the documents belong to, is a required variable + :param current_page: A 1-indexed page count + :param page_size: The maximum number of items to return per query + :param filters: Filter currently only searches for ID of documents + for project title + :returns: Dictionary of items and total count after filter is applied {"items": [], "total_count": int} +::: + + + +#### Parameters + +* project_id + +* current_page + +* page_size + +* filters + + + + + + + +### add_project_document(project_id,document_data) + + + + +#### Parameters + +* project_id + +* document_data + + + + + + + +### add_project_test_document(project_id,document_data) + + + + +#### Parameters + +* project_id + +* document_data + + + + + + + +### add_project_training_document(project_id,document_data) + + + + +#### Parameters + +* project_id + +* document_data + + + + + + + +### add_document_annotation(doc_id,annotation_data) + + + + +#### Parameters + +* doc_id + +* annotation_data + + + + + + + +### get_annotations(project_id) + + +::: tip Description +Serialize project annotations as GATENLP format JSON using the python-gatenlp interface. +::: + + + +#### Parameters + +* project_id + + + + + + + +### delete_documents_and_annotations(doc_id_ary,anno_id_ary) + + + + +#### Parameters + +* doc_id_ary + +* anno_id_ary + + + + + + + +### get_possible_annotators(proj_id) + + + + +#### Parameters + +* proj_id + + + + + + + +### get_project_annotators(proj_id) + + + + +#### Parameters + +* proj_id + + + + + + + +### add_project_annotator(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### make_project_annotator_active(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### project_annotator_allow_annotation(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### remove_project_annotator(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### reject_project_annotator(proj_id,username) + + + + +#### Parameters + +* proj_id + +* username + + + + + + + +### get_annotation_timings(proj_id) + + + + +#### Parameters + +* proj_id + + + + + + + +### delete_annotation_change_history(annotation_change_history_id) + + + + +#### Parameters + +* annotation_change_history_id + + + + + + + +### get_annotation_task() + + +::: tip Description +Gets the annotator's current task, returns a dictionary about the annotation task that contains all the information + needed to render the Annotate view. +::: + + + + + + + + +### get_annotation_task_with_id(annotation_id) + + +::: tip Description +Get annotation task dictionary for a specific annotation_id, must belong to the annotator (or is a manager or above) +::: + + + +#### Parameters + +* annotation_id + + + + + + + +### complete_annotation_task(annotation_id,annotation_data,elapsed_time) + + +::: tip Description +Complete the annotator's current task +::: + + + +#### Parameters + +* annotation_id + +* annotation_data + +* elapsed_time + + + + + + + +### reject_annotation_task(annotation_id) + + +::: tip Description +Reject the annotator's current task +::: + + + +#### Parameters + +* annotation_id + + + + + + + +### change_annotation(annotation_id,new_data) + + +::: tip Description +Adds annotation data to history +::: + + + +#### Parameters + +* annotation_id + +* new_data + + + + + + + +### get_document(document_id) + + +::: tip Description +Obsolete: to be deleted +::: + + + +#### Parameters + +* document_id + + + + + + + +### get_annotation(annotation_id) + + +::: tip Description +Obsolete: to be deleted +::: + + + +#### Parameters + +* annotation_id + + + + + + + +### annotator_leave_project() + + +::: tip Description +Allow annotator to leave their currently associated project. +::: + + + + + + + + +### get_all_users() + + + + + + + + + +### get_user(username) + + + + +#### Parameters + +* username + + + + + + + +### admin_update_user(user_dict) + + + + +#### Parameters + +* user_dict + + + + + + + +### admin_update_user_password(username,password) + + + + +#### Parameters + +* username + +* password + + + + + + + +### admin_delete_user_personal_information(username) + + + + +#### Parameters + +* username + + + + + + + +### admin_delete_user(username) + + + + +#### Parameters + +* username + + + + + + + +### get_privacy_policy_details() + + + + + + + + + +### get_endpoint_listing() + + + + + + + + + + + diff --git a/docs/versioned/2.0.0/developerguide/documentation.md b/docs/versioned/2.0.0/developerguide/documentation.md new file mode 100644 index 00000000..3de09904 --- /dev/null +++ b/docs/versioned/2.0.0/developerguide/documentation.md @@ -0,0 +1,61 @@ +# Managing and versioning documentation + +Documentation versioning is managed by the custom node script located at `docs/manage_versions.js`. Versions of the documentation can be archived and the entire documentation site can be built using the script. + +Various configuration parameters used for management of documentation versioning can be found in `docs/docs.config.js`. + +## Installing dependencies required to serve the documentation site + +The documentation uses vuepress and other libraries which has to be installed separately running the following command from the root of the project: + +```bash +npm run install:docs +``` + +## Editing the documentation + +The latest version of the documentation is located at `/docs/docs`. The archived (versioned) documentation are located in `/docs/versioned/version_number`. + +Use the following command to live preview the latest version of the documentation: + +``` +npm run serve:docs +``` + +Note that this will not work with other versioned docs as they are managed as a separate site. To live preview versioned documentation use the command (replace version_num with the version you'd like to preview): + +``` +vuepress dev docs/versioned/version_num +``` + +## Creating a new documentation version + +To create a version of the documentation, run the command: + +``` +npm run docs:create_version +``` + +This creates a copy of the current set of documentation in `/docs/docs` and places it at `/docs/versioned/version_num`. The version number in `package.json` is used for the documentation version. + +Each set of documentation can be considered as a separate vuepress site. Each one has a `.vuepress/versions.json` file that contains the listing of all versions, allowing them to link to each other. + +Note: Versions can also be created manually by running the command: + +``` +# Replace version_num with the version you'd like to create +node docs/manage_versions.js create version_num +``` + + +## Building documentation site + +To build the documentation site, the previous documentation build command is used: + +``` +npm run build:docs +``` + +## Implementation of the version selector UI + +A partial override of the default Vuepress theme was needed to add a custom component the navigation bar. The modified version of the `NavBar` component can be found in `/docs/docs/.vuepress/theme/components/NavBar.vue`. The modified NavBar uses the `VersionSelector` (`/docs/docs/.vuepress/theme/components/VersionSelector.vue`) component which reads from the `.vuepress/versions.json` from each set of documentation. diff --git a/docs/versioned/2.0.0/developerguide/releases.md b/docs/versioned/2.0.0/developerguide/releases.md new file mode 100644 index 00000000..ec14d1f2 --- /dev/null +++ b/docs/versioned/2.0.0/developerguide/releases.md @@ -0,0 +1,18 @@ +# Managing Releases + +*These instructions are primarily intended for the maintainers of Teamware.* + +Note: Releases are always made from the `master` branch of the repository. + +## Steps to making a release + +1. **Update the changelog** - This has to be done manually, go through any pull requests to `dev` since the last release. + - In github pull requests page, use the search term `is:pr merged:>=yyyy-mm-dd` to find all merged PR from the date since the last version change. + - Include the changes in the `CHANGELOG.md` file; the changelog section _MUST_ begin with a level-two heading that starts with the relevant version number in square brackets (`## [N.M.P] Optional descriptive suffix`) as the GitHub workflow that creates a release from the eventual tag depends on this pattern to find the right release notes. Each main item within the changelog should have a link to the originating PR e.g. \[#123\](https://github.com/GateNLP/gate-teamware/pull/123). +1. **Update and check the version numbers** - from the teamware directory run `python version.py check` to check whether all version numbers are up to date. If not, update the master `VERSION` file and run `python version.py update` to update all other version numbers and commit the result. Note that `version.py` requires `pyyaml` for reading `CITATION.cff`, `pyyaml` is included in Teamware's dependencies. +1. **Create a version of the documentation** - Run `npm run docs:create_version`, this will archive the current version of the documentation using the version number in `package.json`. +1. **Create a pull request from `dev` to `master`** including any changes to `CHANGELOG.md`, `VERSION`. +1. **Create a tag** - Once the dev-to-master pull request has been merged, create a tag from the resulting `master` branch named `vN.M.P` (i.e. the new version number prefixed with the letter `v`). This will trigger two GitHub workflows: + - one that builds versioned Docker images for this release and pushes them to `ghcr.io`, updating the `latest` image tag to point to the new release + - one that creates a "release" on GitHub with the necessary artifacts to make the `https://gate.ac.uk/get-teamware.sh` installation mechanism work correctly. The release notes for this release will be generated by extracting the matching section from `CHANGELOG.md`. +1. **Update the Helm chart** - Create a new branch on [https://github.com/GateNLP/charts](https://github.com/GateNLP/charts) to update the `appVersion` of the `gate-teamware` Helm chart to match the version that was just created by the tag workflow. You must also update the chart `version`, bumping the major version number if the new chart is not backwards-compatible with the old. Submit a pull request to the `main` branch, which will publish the new chart when it is merged. diff --git a/docs/versioned/2.0.0/developerguide/testing.md b/docs/versioned/2.0.0/developerguide/testing.md new file mode 100644 index 00000000..578501b2 --- /dev/null +++ b/docs/versioned/2.0.0/developerguide/testing.md @@ -0,0 +1,182 @@ +# Testing +All the tests can be run using the following command: + +```bash +npm run test +``` + +## Backend Testing +Pytest is used for testing the backend. + +```bash +npm run test:backend +``` + +### Backend test files + +* Unit test files are located in `/backend/tests` + +## Frontend testing +[Jest](https://jestjs.io/) is used for frontend testing. +The [Vue testing-library](https://testing-library.com/docs/vue-testing-library/intro/) is used for testing +Vue components. + +```bash +npm run test:frontend +``` + +### Frontend test files + +* Frontend test files are located in `/fontend/tests/unit` and should the extension `.spec.js` + +### Testing JS functions + +```javascript +describe("Description of a group of tests to be run", () =>{ + + beforeAll(() =>{ + //The code here is run before each test + }) + + it("A single test's description", async () =>{ + + // Assertions are done with the expect() function e.g. + let funcOutput = 30 + 10 + expect(funcOutput).toBe(40) + + + }) +}) + +``` + +### Mocking JS classes + +This is an example of a mock harness for the JRPCClient class. + +A mock file is created inside a ``__mock__`` directory placed next to the file that's being mocked, e.g. +for our JRPCClient class at `/frontend/src/jrpc/index.js`, the mock file is `/frontend/src/jrpc/__mock__/index.js`. + + +Inside the mock file `/frontend/src/jrpc/__mock__/index.js`: +```javascript +// Mocking jrpc/index.js +//Mocking the JRPCClient class +//Replacing the call function with a custom mockCall function +export const mockCall = jest.fn(()=> 30); +const mock = jest.fn().mockImplementation(() => { + return {call: mockCall}; +}); + +export default mock; +``` + + +Inside the test file `*.spec.js`: +```javascript +import JRPCClient from "@/jrpc"; +jest.mock('@/jrpc') + +import store from '@/store' +//Example on how to mock the jrpc call + +describe("Vuex functions testing", () =>{ + + beforeAll(() =>{ + + //Re-implement custom mock call implementation if needed + JRPCClient.mockImplementation(()=>{ + return { + call(){ + return 50 + } + } + }) + + }) + + it("testfunc", async () =>{ + + const noutput = await store.dispatch("testnormal") + expect(noutput).toBe("Hello world") + + const aoutput = await store.dispatch("testasync") + expect(aoutput).toBe("Hello world") + + const rpc = new JRPCClient("/") + const result = await rpc.call("some param") + expect(result).toBe(50) + + }) +}) +``` + +### Testing Vue components + + +```javascript +//Example of how a component could be tested +import { render, fireEvent } from '@testing-library/vue' + + +import HelloWorld from '@/components/HelloWorld.vue' + +//Testing a component e.g. HelloWorld +describe('HelloWorld.vue', () => { + + it('renders props.msg when passed', () => { + const msg = 'new message' + const { getByText } = render(HelloWorld) + + getByText("Installed CLI Plugins") + }) +}) + +``` + + +## Integration testing +[Cypress](https://www.cypress.io/) is used for integration testing. + +The integration settings are located at `teamware/settings/integration.py` + +To run the integration test: +```bash +npm run test:integration +``` + +The test can also be run in **interactive mode** using: + +```bash +npm run serve:cypressintegration +``` + +### Integration test files +Files related to integration testing are located in `/cypress` + +* Test files are located in the `/cypress/integration` directory and should have the extension `.spec.js`. + +### Re-seeding the database + +The command `npm run migrate:integration` resets the database and performs migration, use with `beforeEach` to run it +before every test case in a suite: + +```js +describe('Example test suite', () => { + + beforeEach(() => { + // Resets the database every time before + // the test is run + cy.exec('npm run migrate:integration') + }) + + it('Test case 1', () => { + // Test something + }) + + it('Test case 2', () => { + // Test something + }) +}) +``` + diff --git a/docs/versioned/2.0.0/img/gate-teamware-logo.svg b/docs/versioned/2.0.0/img/gate-teamware-logo.svg new file mode 100644 index 00000000..12385947 --- /dev/null +++ b/docs/versioned/2.0.0/img/gate-teamware-logo.svg @@ -0,0 +1,79 @@ + + + + diff --git a/docs/versioned/2.0.0/manageradminguide/README.md b/docs/versioned/2.0.0/manageradminguide/README.md new file mode 100644 index 00000000..7a70a19f --- /dev/null +++ b/docs/versioned/2.0.0/manageradminguide/README.md @@ -0,0 +1,45 @@ +# GATE Teamware Overview + +## User roles + +There are three types of users in GATE Teamware, [annotators](#annotators), [managers](#managers) +and [admins](#admins). + +### Annotators + +Annotator is the default role when signing up to Teamware. An annotator can be recruited into +annotation projects and annotate documents. + + +### Managers + +Managers can create, view and modify annotation projects. They can also recruit annotators to a project. + +### Admins + +Admins, on top of what managers can do, they can also manage the users in the system and elevate them as +managers or admins. + +## Annotation Projects, Documents and Annotations + +Projects, documents and annotations form the core of the application. + +### Projects + +An annotation project contains a configuration of how annotations are to be captured, the documents and its +annotations and the recruited annotators. + + +### Documents + +A document in application refers to an individual set of arbitrary text that's to be annotated. A document +is stored as arbitrary JSON object and can represent various things such as, a single post (e.g. a tweet +or a post from reddit), a pair of source post and reply or a part of a HTML web page. + + +### Annotations + +An annotation represents a single annotation task against a single document. Like the document, +an annotation is stored as an arbitrary JSON object and can have any arbitrary structure. + + diff --git a/docs/versioned/2.0.0/manageradminguide/annotators_management.md b/docs/versioned/2.0.0/manageradminguide/annotators_management.md new file mode 100644 index 00000000..cb4c79b0 --- /dev/null +++ b/docs/versioned/2.0.0/manageradminguide/annotators_management.md @@ -0,0 +1,13 @@ +# Annotators management + +The **Annotators** tab in the **Project management** page allows the viewing and management of annotators in the project. + +Add annotators to the project by clicking on the list of names in the right column. Current annotators +can be removed by clicking on the names in the left column. Removing annotators does not delete their +completed annotations but will stop their current pending annotation task. + +An annotator can only be recruited into **one project at a time**. + +Once an annotator has annotated a proportion of documents in the project (specified in project configuration), they will +be deemed to have completed all their annotation tasks and automatically be removed the project. This frees them to be +recruited in another project. diff --git a/docs/versioned/2.0.0/manageradminguide/config_examples.js b/docs/versioned/2.0.0/manageradminguide/config_examples.js new file mode 100644 index 00000000..e167b81e --- /dev/null +++ b/docs/versioned/2.0.0/manageradminguide/config_examples.js @@ -0,0 +1,251 @@ +export default { + config1: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + } + ], + config2: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + }, + { + "name": "opinion", + "type": "text", + "title": "What's your opinion of the above text?", + "optional": true + } + ], + configDisplay: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + } + ], + configDisplayHtmlNoHtml: [ + { + "name": "htmldisplay", + "type": "html", + "text": "No HTML: {{text}}
HTML: {{{text}}}" + } + ], + configDisplayCustomFieldnames: [ + { + "name": "htmldisplay", + "type": "html", + "text": "Custom field: {{customField}}
Another custom field: {{{anotherCustomField}}}
Subfield: {{{subfield.subfieldContent}}}" + } + ], + configDisplayPreserveNewlines: [ + { + "name": "htmldisplay", + "type": "html", + "text": "
{{text}}
" + } + ], + configTextInput: [ + { + "name": "mylabel", + "type": "text", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configTextarea: [ + { + "name": "mylabel", + "type": "textarea", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configRadio: [ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "orientation": "vertical", //Optional - default is "horizontal" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configCheckbox: [ + { + "name": "mylabel", + "type": "checkbox", + "optional": true, //Optional - Set if validation is not required + "orientation": "horizontal", //Optional - "horizontal" (default) or "vertical" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "minSelected": 1, //Optional - Specify the minimum number of options that must be selected + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configSelector: [ + { + "name": "mylabel", + "type": "selector", + "optional": true, //Optional - Set if validation is not required + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + configRadioDict: [ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "options": { // The options can be specified as a dictionary, ordering is not guaranteed + "value1": "Text to show user 1", + "value2": "Text to show user 2", + "value3": "Text to show user 3", + }, + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message then field is validated", //Optional + "valError": "Error message when field fails is validation" //Optional + } + ], + + configDbpediaExample: [ + { + "name": "uri", + "type": "radio", + "title": "Select the most appropriate URI", + "options":[ + {"fromDocument": "candidates"}, + {"value": "none", "label": "None of the above"}, + {"value": "unknown", "label": "Cannot be determined without more context"} + ] + } + ], + docDbpediaExample: { + "text": "President Bush visited the air base yesterday...", + "candidates": [ + { + "value": "http://dbpedia.org/resource/George_W._Bush", + "label": "George W. Bush (Jnr)" + }, + { + "value": "http://dbpedia.org/resource/George_H._W._Bush", + "label": "George H. W. Bush (Snr)" + } + ] + }, + + + doc1: {text: "Sometext with html"}, + doc2: { + customField: "Content of custom field.", + anotherCustomField: "Content of another custom field.", + subfield: { + subfieldContent: "Content of a subfield." + } + }, + docPlainText: { + "text": "This is some text\n\nIt has line breaks that we want to preserve." + }, + configPreAnnotation: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "radio", + "type": "radio", + "title": "Test radio input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test radio description" + }, + { + "name": "checkbox", + "type": "checkbox", + "title": "Test checkbox input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test checkbox description" + }, + { + "name": "text", + "type": "text", + "title": "Test text input", + "description": "Test text description" + } + + ], + docPreAnnotation: { + "id": 12345, + "text": "Example document text", + "preannotation": { + "radio": "val1", + "checkbox": ["val1", "val3"], + "text": "Pre-annotation text value" + } + } + + +} diff --git a/docs/versioned/2.0.0/manageradminguide/documents_annotations_management.md b/docs/versioned/2.0.0/manageradminguide/documents_annotations_management.md new file mode 100644 index 00000000..ac010c9f --- /dev/null +++ b/docs/versioned/2.0.0/manageradminguide/documents_annotations_management.md @@ -0,0 +1,272 @@ +# Documents & Annotations + +The **Documents & Annotations** tab in the **Project management** page allows the viewing and management of documents +and annotations related to the project. + +## Document & Annotation status + +### Annotation status + +Annotations can be in 1 of 5 states: + +* Annotation is completed - The annotator has completed this annotation task. +* Annotation is rejected - The annotator has chosen to not annotate the document. +* Annotation is timed out - The annotation task was not completed within the time specified in the project's configuration. The task is freed and can be assigned to another annotator. +* Annotation is aborted - The annotation task was aborted due to reasons other than timing out, such as when an annotator with a pending task is removed from a project. +* Annotation is pending - The annotator has started the annotation task but has not completed it. + +### Document status + +Documents also display a list of its current annotation status: + +* 1 - Number of completed annotations in the document. +* 1 - Number of rejected annotations in the document. +* 1 - Number of timed out annotations in the document. +* 1 - Number of aborted annotations in the document. +* 1 - Number of pending annotations in the document. + +## Importing documents + +Documents can be imported using the **Import** button. The supported file types are: + +* `.json` - The app expects a list of documents (represented as a dictionary object) + e.g. `[{"id": 1, "text": "Text1"}, ...]`. +* `.jsonl` - The app expects one document (represented as a dictionary object) per line. +* `.csv` - File must have a header row. It will be internally converted to JSON format. +* `.zip` - Can contain any number of `.json,.jsonl and .csv` files inside. + +### Importing documents with pre-annotation + +In the `Project Configurations` page, it is possible to set a field in which Teamware will look for pre-annotation. If +the field is found inside the document then the annotation form will be pre-filled with data provided in the document. + +The format for pre-annotation is exactly the same as the annotation output. You can see an example of generated +annotation by filling out the form in the `Annotation Preview` and observing the values in +the `Annotation Output Preview`. + + +For an example project configuration shown below, there are three captured labels named `radio`, `checkbox` and `text`: + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "radio", + "type": "radio", + "title": "Test radio input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test radio description" + }, + { + "name": "checkbox", + "type": "checkbox", + "title": "Test checkbox input", + "options": [ + {"value": "val1", "label": "Value 1"}, + {"value": "val2", "label": "Value 2"}, + {"value": "val3", "label": "Value 4"}, + {"value": "val4", "label": "Value 5"} + ], + "description": "Test checkbox description" + }, + { + "name": "text", + "type": "text", + "title": "Test text input", + "description": "Test text description" + } +] +``` + +On the `Project Configuration` page, if the `Pre-annotation` field is set to `preannotation`, the annotation form will be pre-filled with +the content provided in the `preannotation` field of the document e.g.: + +```json +{ + "id": 12345, + "text": "Example document text", + "preannotation": { + "radio": "val1", + "checkbox": [ + "val1", + "val3" + ], + "text": "Pre-annotation text value" + } +} +``` + +The example of the pre-filled form can be seen by clicking on the `Preview` tab above. + + + + + + +### Importing Training and Test documents + +When importing documents for the training and testing phase, Teamware expects a field/column (called `gold` by default) +that contains the correct annotation response for each label and, only for training documents, an explanation. + +For example, if we're expecting a multi-choice label for doing sentiment classification with a widget named `sentiment` +and choice of `postive`, `negative` and `neutrual`: + +```js +[ + { + "text": "What's my sentiment", + "gold": { + "sentiment": { + "value": "positive", // For this document, the correct value is postive + "explanation": "Because..." // Explanation is only given in the traiing phase and are optional in the test documents + } + } + } +] +``` + +in csv: + +| text | gold.sentiment.value | gold.sentiment.explanation | +| --- | --- | --- | +| What's my sentiment | positive | Because... | + +### Guidance on CSV column headings + +It is recommended that: + +* Spaces are not used in column headings, use dash (`-`), underscore (`_`) or camel case (e.g. fieldName) instead. +* The dot/full stop (`.`) is used to indicate hierarchical information so don't use it if that's not what's intended. + Explanation on this feature is given below. + +Documents imported from a CSV files are converted to JSON for use internally in Teamware, the reverse is true when +converting back to CSV. To allow a CSV to represent a hierarchical structure, a dot notation is used to indicate a +sub-field. + +In the following example, we can see that `gold` has a child field named `sentiment` which then has a child field +named `value`: + +| text | gold.sentiment.value | gold.sentiment.explanation | +| --- | --- | --- | +| What's my sentiment | positive | Because... | + +The above column headers will generate the following JSON: + +```js +[ + { + "text": "What's my sentiment", + "gold": { + "sentiment": { + "value": "positive", // For this document, the correct value is postive + "explanation": "Because..." // Explanation is only given in the traiing phase and are optional in the test documents + } + } + } +] +``` + +## Exporting documents + +Documents and annotations can be exported using the **Export** button. A zip file is generated containing files with 500 +documents each. You can choose how documents are exported: + +* `.json` & `.jsonl` - JSON or JSON Lines files can be generated in the format of: + * `raw` - Exports unmodified JSON. If you've originally uploaded in GATE format then choose this option. + + An additional field named `annotation_sets` is added for storing annotations. The annotations are laid out in the + same way as GATE JSON format. For example if a document has been annotated by `user1` with labels and values + `text`:`Annotation text`, `radio`:`val3`, and `checkbox`:`["val2", "val4"]`: + + ```json + { + "id": 32, + "text": "Document text", + "text2": "Document text 2", + "feature1": "Feature text", + "annotation_sets":{ + "user1":{ + "name":"user1", + "annotations":[ + { + "type":"Document", + "start":0, + "end":10, + "id":0, + "features":{ + "label":{ + "text":"Annotation text", + "radio":"val3", + "checkbox":[ + "val2", + "val4" + ] + } + } + } + ], + "next_annid":1 + } + } + } + ``` + + * `gate` - Convert documents to GATE JSON format and export. A `name` field is added that takes the ID value from the + ID field specified in the project configuration. Fields apart from `text` and the ID field specified in the project + config are placed in the `features` field. An `annotation_sets` field is added for storing annotations. + + For example in the case of this uploaded JSON document: + ```json + { + "id": 32, + "text": "Document text", + "text2": "Document text 2", + "feature1": "Feature text" + } + ``` + The generated output is as follows. The annotations are formatted same as the `raw` output above: + ```json + { + "name": 32, + "text": "Document text", + "features": { + "text2": "Document text 2", + "feature1": "Feature text" + }, + "offset_type":"p", + "annotation_sets": {...} + } + ``` +* `.csv` - The JSON documents will be flattened to csv's column based format. Annotations are added as additional + columns with the header of `annotations.username.label`. + +## Deleting documents and annotations + +It is possible to click on the top left of corner of documents and annotations to select it, then click on the +**Delete** button to delete them. + +::: tip + +Selecting a document also selects all its associated annotations. + +::: + + + diff --git a/docs/versioned/2.0.0/manageradminguide/project_config.md b/docs/versioned/2.0.0/manageradminguide/project_config.md new file mode 100644 index 00000000..99b26e34 --- /dev/null +++ b/docs/versioned/2.0.0/manageradminguide/project_config.md @@ -0,0 +1,495 @@ +--- +sidebarDepth: 3 +--- + +# Project configuration + +The **Configuration** tab in the **Project management** page allows you to change project settings including what +annotations are captured. + +Project configurations can be imported and exported in the format of a JSON file. + +The project can be also be cloned (have configurations copied to a new project). Note that cloning does not copy +documents, annotations or annotators to the new project. + +## Configuration fields + +* **Name** - The name of this annotation project. +* **Description** - The description of this annotation project that will be shown to annotators. Supports markdown and + HTML. +* **Annotator guideline** - The description of this annotation project that will be shown to annotators. Supports + markdown and HTML. +* **Annotations per document** - The project completes when each document in this annotation project have this many + number of valid annotations. When a project completes, all project annotators will be un-recruited and be allowed to + annotate other projects. +* **Maximum proportion of documents annotated per annotator (between 0 and 1)** - A single annotator cannot annotate + more than this proportion of documents. +* **Timeout for pending annotation tasks (minutes)** - Specify the number of minutes a user has to complete an + annotation task (i.e. annotating a single document). +* **Reject documents** - Switching this off will mean that annotators for this project will be unable to choose to reject documents. +* **Document ID field** - The field in your uploaded documents that is used as a unique identifier. GATE's json format + uses the name field. You can use a dot limited key path to access subfields e.g. enter features.name to get the id + from the object `{'features':{'name':'nameValue'}}` +* **Training stage enable/disable** - Enable or disable training stage, allows testing documents to be uploaded to the project. +* **Test stage enable/disable** - Enable or disable testing stage, allows test documents to be uploaded to the project. +* **Auto elevate to annotator** - The option works in combination with the training and test stage options, see table below for the behaviour: + + | Training stage | Testing stage | Auto elevate to annotator | Desciption | + | --- | --- | --- | --- | + | Disabled | Disabled | Enabled/Disabled | User allowed to annotate without manual approval. | + | Enabled | Disabled | Disabled | Manual approval required. | + | Disabled | Enabled | Disabled | " | + | Enabled | Disabled | Enabled | User always allowed to annotate after training phase completed | + | Disabled | Enabled | Enabled | User automatically allowed to annotate after passing test, if user fails test they have to be manually approved. | + | Enabled | Enabled | Enabled | " | + +* **Test pass proportion** - The proportion of correct test annotations to be automatically allowed to annotate documents. +* **Gold standard field** - The field in document's JSON/column that contains the ideal annotation values and explanation for the annotation. +* **Pre-annotation** - Pre-fill the form with annotation provided in the specified field. See [Importing Documents with pre-annotation](./documents_annotations_management.md#importing-documents-with-pre-annotation) section for more detail. + +## Anotation configuration + +The annotation configuration takes a `json` string for configuring how the document is displayed to the user and types +of annotation will be collected. Here's an example configuration and a preview of how it is shown to annotators: + + + + +```json +// Example configuration +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + } +] +``` + + + +Within the configuration, it is possible to specify how your documents will be displayed. The **Document input preview** +box can be used to provide a sample of your document for rendering of the preview. + +```json +// Example contents for the Document input preview +{ + "text": "Sometext with html" +} +``` + + + +The above configuration displays the value from the `text` field from the document to be annotated. It then shows a set +of 3 radio inputs that allows the user to select a Negative, Neutral, or Positive sentiment with the label +name `sentiment`. + + + +All fields **require** the properties **name** and **type**, it is used to name our label and determine the type of +input/display to be shown to the user respectively. + +Another field can be added to collect more information, e.g. a text field for opinions: + + + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + }, + { + "name": "opinion", + "type": "text", + "title": "What's your opinion of the above text?", + "optional": true + } + +] +``` + + + +Note that for the above case, the `optional` field is added ensure that allows user to not have to input any value. +This `optional` field can be used on all components. + +Some fields are available to configure which are specific to components, e.g. the `options` field are only available for +the `radio`, `checkbox` and `selector` components. See details below on the usage of each specific component. + +The captured annotation results in a JSON dictionary, an example can be seen in the **Annotation output preview** box. +The annotation is linked to a Document and is converted to a GATE JSON annotation format when exported. + +### Displaying text + + + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" // The text that will be displayed + } +] +``` + + + +The `htmldisplay` widget allows you to display the text you want annotated. It accepts almost full range of HTML +input which gives full styling flexibility. + +Any field/column from the document can be inserted by surrounding a field/column name with double or +triple curly brackets. Double curly brackets renders text as-is and triple curly brackets accepts HTML string: + + + +Input: + +```json +{ + "text": "Sometext with html" +} +``` + +Configuration, showing the same field/column in document as-is or as HTML: +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "No HTML: {{text}}
HTML: {{{text}}}" + } +] +``` + +
+ +The widget makes no assumption about your document structure and any field/column names can be used, +even sub-fields by using the dot notation e.g. `parentField.childField`: + + + +JSON input: + +```json +{ + "customField": "Content of custom field.", + "anotherCustomField": "Content of another custom field.", + "subfield": { + "subfieldContent": "Content of a subfield." + } +} +``` + +or in csv + +| customField | anotherCustomField | subfield.subfieldContent | +| --- | --- | --- | +| Content of custom field. | Content of another custom field. | Content of a subfield. | + + +Configuration, showing the same field/column in document as-is or as HTML: +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "Custom field: {{customField}}
Another custom field: {{{anotherCustomField}}}
Subfield: {{{subfield.subfieldContent}}}" + } +] +``` + +
+ +If your documents are plain text and include line breaks that need to be preserved when rendering, this can be achieved by using a special HTML wrapper which sets the [`white-space` CSS property](https://developer.mozilla.org/en-US/docs/Web/CSS/white-space). + + + +**Document** + +```json +{ + "text": "This is some text\n\nIt has line breaks that we want to preserve." +} +``` + +**Project configuration** + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "
{{text}}
" + } +] +``` + +
+ +`white-space: pre-line` preserves line breaks but collapses other whitespace down to a single space, `white-space: pre-wrap` would preserve all whitespace including indentation at the start of a line, but would still wrap lines that are too long for the available space. + +### Text input + + + +```json +[ + { + "name": "mylabel", + "type": "text", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Textarea input + + + +```json +[ + { + "name": "mylabel", + "type": "textarea", + "optional": true, //Optional - Set if validation is not required + "regex": "regex string", //Optional - When specified, the regex pattern will used to validate the text + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Radio input + + + +```json +[ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "orientation": "vertical", //Optional - default is "horizontal" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Checkbox input + + + +```json +[ + { + "name": "mylabel", + "type": "checkbox", + "optional": true, //Optional - Set if validation is not required + "orientation": "horizontal", //Optional - "horizontal" (default) or "vertical" + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "minSelected": 1, //Optional - Overrides optional field. Specify the minimum number of options that must be selected + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Selector input + + + +```json +[ + { + "name": "mylabel", + "type": "selector", + "optional": true, //Optional - Set if validation is not required + "options": [ // The options that the user is able to select from + {"value": "value1", "label": "Text to show user 1"}, + {"value": "value2", "label": "Text to show user 2"}, + {"value": "value3", "label": "Text to show user 3"} + ], + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Alternative way to provide options for radio, checkbox and selector + +A dictionary (key value pairs) and also be provided to the `options` field of the radio, checkbox and selector widgets +but note that the ordering of the options are **not guaranteed** as javascript does not sort dictionaries by +the order in which keys are added. + + + +```json +[ + { + "name": "mylabel", + "type": "radio", + "optional": true, //Optional - Set if validation is not required + "options": { // The options can be specified as a dictionary, ordering is not guaranteed + "value1": "Text to show user 1", + "value2": "Text to show user 2", + "value3": "Text to show user 3" + }, + "title": "Title string", //Optional + "description": "Description string", //Optional + "valSuccess": "Success message when the field is validated", //Optional + "valError": "Error message when the field fails validation" //Optional + } +] +``` + + + +### Dynamic options for radio, checkbox and selector + +All the examples above have a "static" list of available options for the radio, checkbox and selector widgets, where the complete options list is enumerated in the project configuration and every document offers the same set of options. However it is also possible to take some or all of the options from the _document_ data rather than the _configuration_ data. For example: + + + +**Project configuration** + +```json +[ + { + "name": "uri", + "type": "radio", + "title": "Select the most appropriate URI", + "options":[ + {"fromDocument": "candidates"}, + {"value": "none", "label": "None of the above"}, + {"value": "unknown", "label": "Cannot be determined without more context"} + ] + } +] +``` + +**Document** + +```json +{ + "text": "President Bush visited the air base yesterday...", + "candidates": [ + { + "value": "http://dbpedia.org/resource/George_W._Bush", + "label": "George W. Bush (Jnr)" + }, + { + "value": "http://dbpedia.org/resource/George_H._W._Bush", + "label": "George H. W. Bush (Snr)" + } + ] +} +``` + + + +`"fromDocument"` is a dot-separated property path leading to the location within each document where the additional options can be found, for example `"fromDocument":"candidates"` looks for a top-level property named `candidates` in each document, `"fromDocument": "options.custom"` would look for a property named `options` which is itself an object with a property named `custom`. The target property in the document may be in any of the following forms: + +- an array _of objects_, each with `value` and `label` properties, exactly as in the static configuration format - this is the format used in the example above +- an array _of strings_, where the same string will be used as both the value and the label for that option +- an arbitrary ["dictionary"](#options-as-dict) object mapping values to labels +- a _single string_, which is parsed into a list of options + +The "single string" alternative is designed to be easier to use when [importing documents](documents_annotations_management.md#importing-documents) from CSV files. It allows you to provide any number of options in a _single_ CSV column value. Within the column the options are separated by semicolons, and each option is of the form `value=label`. Whitespace around the delimiters is ignored, both between options and between the value and label of a single option. For example given CSV document data of + +| text | options | +|-----------------|---------------------------------------------------| +| Favourite fruit | `apple=Apples; orange = Oranges; kiwi=Kiwi fruit` | + +a `{"fromDocument": "options"}` configuration would produce the equivalent of + +```json +[ + {"value": "apple", "label": "Apples"}, + {"value": "orange", "label": "Oranges"}, + {"value": "kiwi", "label": "Kiwi fruit"} +] +``` + +If your values or labels may need to contain the default separator characters `;` or `=` you can select different separators by adding extra properties to the configuration: + +```json +{"fromDocument": "options", "separator": "~~", "valueLabelSeparator": "::"} +``` + +| text | options | +|-----------------|------------------------------------------------------| +| Favourite fruit | `apple::Apples ~~ orange::Oranges ~~ kiwi::Kiwi fruit` | + +The separators can be more than one character, and you can set `"valueLabelSeparator":""` to disable label splitting altogether and just use the value as its own label. + +### Mixing static and dynamic options + +Static and `fromDocument` options may be freely interspersed in any order, so you can have a fully-dynamic set of options by specifying _only_ a `fromDocument` entry with no static options, or you can have static options that are listed first followed by dynamic options, or dynamic options first followed by static, etc. + + diff --git a/docs/versioned/2.0.0/manageradminguide/project_management.md b/docs/versioned/2.0.0/manageradminguide/project_management.md new file mode 100644 index 00000000..f1fd0cfe --- /dev/null +++ b/docs/versioned/2.0.0/manageradminguide/project_management.md @@ -0,0 +1,38 @@ +# Annotation Project Management + +## Project Listing +Clicking on the `Projects` link in the top navigation bar takes you to a contains a list of existing +projects. The project names are shown along with their summaries. Clicking on a project name will +take you to the project management page. + + +## Project Management Page + +The project management page contains all the functionalities to manage an annotation project. The page +is composed of three main tabs: + +* [Configuration](project_config.md) - Configure project settings including what annotations are captured. +* [Documents & Annotation](documents_annotations_management.md) - Manage documents and annotations. Upload documents, see contents of a document's annotations and import/export documents. +* [Annotators](annotators_management.md) - Manage the recruitment of annotators. + +::: warning + +Annotators can only be recruited to an annotation project after it has been configured and documents +are uploaded to the project. + +::: + + +## Project status icons +In the **Project listing** and **Project management page**, icon badges are used to provide a quick overview of the project's status: + +* 1 - Number of completed annotations in the project. +* 1 - Number of rejected annotations in the project. +* 1 - Number of timed out annotations in the project. +* 1 - Number of aborted annotations in the project. +* 1 - Number of pending annotations in the project. +* 2/60 - Number of occupied annotation tasks over number of total tasks in the project. +* 20/5/10 - Number of documents, training documents and test documents in the project. +* 1 - Number of annotators recruited in the project. Annotators are removed from the project when they have completed all annotation tasks in their quota. + + diff --git a/examples/documents_radio_fromDocument.json b/examples/documents_radio_fromDocument.json new file mode 100644 index 00000000..e0654adb --- /dev/null +++ b/examples/documents_radio_fromDocument.json @@ -0,0 +1,30 @@ +[ + { + "id": "1", + "text": "What is your favourite fruit?", + "choices": [ + {"value": "apple", "label": "Apple"}, + {"value": "orange", "label": "Orange"}, + {"value": "kiwi", "label": "Kiwi fruit"} + ] + }, + { + "id": "2", + "text": "What is your favourite shade of green?", + "choices": { + "#808000": "Olive", + "#80ff00": "Chartreuse", + "#98FB98": "Mint" + } + }, + { + "id": "3", + "text": "Which of these languages do you find easiest to speak?", + "choices": ["English", "Mandarin", "Portuguese"] + }, + { + "id": "4", + "text": "What is your preferred computing platform?", + "choices": "win=Windows; mac=Apple; linux=Linux (e.g. Ubuntu)" + } +] \ No newline at end of file diff --git a/examples/project_config_radio_fromDocument.json b/examples/project_config_radio_fromDocument.json new file mode 100644 index 00000000..c161c787 --- /dev/null +++ b/examples/project_config_radio_fromDocument.json @@ -0,0 +1,18 @@ +[ + { + "name": "question", + "type": "html", + "title": "Question", + "text": "{{text}}" + }, + { + "name": "answer", + "type": "radio", + "title": "Answer", + "orientation": "vertical", + "options": [ + {"fromDocument": "choices"}, + {"value": "none", "label": "No preference"} + ] + } +] \ No newline at end of file diff --git a/frontend/src/AnnotationApp.vue b/frontend/src/AnnotationApp.vue index fa13225d..2f0f0cf9 100644 --- a/frontend/src/AnnotationApp.vue +++ b/frontend/src/AnnotationApp.vue @@ -2,14 +2,16 @@
+
+ + diff --git a/frontend/src/main.js b/frontend/src/main.js index f85e51da..7475b69c 100644 --- a/frontend/src/main.js +++ b/frontend/src/main.js @@ -42,7 +42,7 @@ Vue.filter('datetime', function (dateString) { async function initialiseApp() { //Ensure authentication status is checked before we actually start the app //so things that depends on user's logged in status works properly (e.g. routing) - await store.dispatch("is_authenticated") + await store.dispatch("initialise") new Vue({ router, diff --git a/frontend/src/router/index.js b/frontend/src/router/index.js index caa15884..1a2afd17 100644 --- a/frontend/src/router/index.js +++ b/frontend/src/router/index.js @@ -21,6 +21,18 @@ const routes = [ component: () => import(/* webpackChunkName: "about" */ '../views/About.vue'), meta: {guest: true}, }, + { + path: '/privacypolicy', + name: 'Privacy Policy', + component: () => import('../views/PrivacyPolicy.vue'), + meta: {guest: true}, + }, + { + path: '/terms', + name: 'Terms & Conditions', + component: () => import('../views/TermsAndConditions.vue'), + meta: {guest: true}, + }, { path: '/login', name: 'Login', @@ -83,6 +95,12 @@ const routes = [ component: () => import('../views/ManageUsers'), meta: {requiresAdmin: true}, }, + { + path: '/cookies', + name: 'Cookies', + component: () => import('../views/Cookies'), + meta: {guest: true}, + }, ] diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index cd4455b2..e90b6bdd 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -17,7 +17,11 @@ export default new Vuex.Store({ isAdmin: false, isActivated: false, docFormatPref: "JSON", + }, + global_configs: { + allowUserDelete: false, + } }, getters:{ @@ -29,6 +33,9 @@ export default new Vuex.Store({ docFormatPref(state){ return state.user.docFormatPref }, + allowUserDelete(state){ + return state.global_configs.allowUserDelete + } }, mutations: { activateUser(state){ @@ -43,12 +50,27 @@ export default new Vuex.Store({ }, updateDocFormatPref(state, preference){ state.user.docFormatPref = preference + }, + updateAllowUserDelete(state, doAllow){ + state.global_configs.allowUserDelete = doAllow } }, actions: { updateUser({commit}, params) { commit("updateUser", params); }, + async initialise({dispatch,commit}){ + try{ + let response = await rpc.call("initialise"); + dispatch("updateUser", response.user) + commit("updateDocFormatPref", response.configs.docFormatPref) + commit("updateAllowUserDelete", response.global_configs.allowUserDelete) + + }catch(e){ + console.log(e) + throw e + } + }, async login({dispatch, commit}, params) { try{ const payload = { @@ -72,6 +94,15 @@ export default new Vuex.Store({ await rpc.call("logout"); commit("updateUser", params); }, + async getPrivacyPolicyDetails() { + try{ + let response = await rpc.call("get_privacy_policy_details"); + return response + }catch (e){ + console.error(e) + throw e + } + }, async register({dispatch, commit}, params) { try{ const payload = { @@ -178,7 +209,6 @@ export default new Vuex.Store({ throw e } }, - async getUser({dispatch, commit}) { try{ let user = await rpc.call("get_user_details"); @@ -189,6 +219,22 @@ export default new Vuex.Store({ console.error(e); } }, + async deletePersonalInformation({dispatch, commit}) { + try{ + let response = await rpc.call("user_delete_personal_information"); + dispatch("logout") + }catch (e){ + console.error(e); + } + }, + async deleteAccount({dispatch, commit}) { + try{ + let response = await rpc.call("user_delete_account"); + dispatch("logout") + }catch (e){ + console.error(e); + } + }, async adminGetUser({dispatch, commit}, username) { try{ @@ -224,7 +270,22 @@ export default new Vuex.Store({ throw e } }, - + async adminDeleteUserPersonalInformation({dispatch, commit}, username){ + try{ + await rpc.call("admin_delete_user_personal_information", username) + }catch (e){ + console.error(e) + throw e + } + }, + async adminDeleteUser({dispatch, commit}, username){ + try{ + await rpc.call("admin_delete_user", username) + }catch (e){ + console.error(e) + throw e + } + }, async getAllUsers({dispatch,commit}){ try { let users = await rpc.call("get_all_users"); diff --git a/frontend/src/utils/annotations.js b/frontend/src/utils/annotations.js index 95ce51c2..22699bf3 100644 --- a/frontend/src/utils/annotations.js +++ b/frontend/src/utils/annotations.js @@ -1,14 +1,79 @@ -export function generateBVOptions(options) { +import {getValueFromKeyPath} from "@/utils/dict"; + +export function generateBVOptions(options, document) { let optionsList = [] if (Array.isArray(options)) { - for( let i in options){ - const option = options[i] - optionsList.push({ - value: option.value, - text: option.label - }) + for (let option of options){ + if (document && option && typeof option === "object" + && typeof option.fromDocument === "string") { + // one or more options taken from the document data + const propertyPath = option.fromDocument + // fromDocument is supposed to be a dot-separated path, but if + // the whole path is found as a top-level property name then use + // it, i.e. try opt['foo.bar.baz'] first, then opt.foo.bar.baz + // second + let optionsFromDocument = (propertyPath in document) ? + document[propertyPath] : getValueFromKeyPath(document, propertyPath); + + if(typeof optionsFromDocument === "string") { + // single string - treat it as a delimited list of options where + // each option may be a pair of value and label. For example, + // with the default delimiters + // + // orange=Orange; kiwi=Kiwi fruit + // + // maps to + // + // [{"value":"orange", "label":"Orange"}, + // {"value":"kiwi", "label":"Kiwi fruit"}] + // + // Whitespace around the delimiters is ignored + const optionSeparator = (typeof option.separator === 'string' ? option.separator : ';'); + if (optionSeparator) { + optionsFromDocument = optionsFromDocument.split(optionSeparator); + } else { + optionsFromDocument = [optionsFromDocument]; + } + optionsFromDocument = optionsFromDocument.map(s => s.trim()); + + const valueLabelSeparator = + (typeof option.valueLabelSeparator === 'string' ? option.valueLabelSeparator : '='); + if(valueLabelSeparator) { + optionsFromDocument = optionsFromDocument.map(opt => { + // can't use String.prototype.split here as we want everything after the first + // occurrence of the separator to be used as the value, even when that includes + // more instances of the separator string + const sepIndex = opt.indexOf(valueLabelSeparator); + if(sepIndex >= 0) { + return { + value: opt.substring(0, sepIndex).trim(), + label: opt.substring(sepIndex + valueLabelSeparator.length).trim(), + }; + } else { + // no separator - use whole item as both value and label + return opt; + } + }); + } + } + optionsList.push(...generateBVOptions(optionsFromDocument)); + } else if (option !== null && typeof option !== "undefined") { + // single option + if (typeof option === 'string') { + optionsList.push({ + value: option, + text: option, + }) + } else if ("value" in option) { + optionsList.push({ + value: option.value, + text: ("label" in option ? option.label : option.value), + }) + } // else invalid option, so ignore + } } } else { + // a dictionary mapping value to label for (let optionKey in options) { optionsList.push({ value: optionKey, diff --git a/frontend/src/views/About.vue b/frontend/src/views/About.vue index 188b7d5e..d5d6283f 100644 --- a/frontend/src/views/About.vue +++ b/frontend/src/views/About.vue @@ -2,16 +2,16 @@

About GATE Teamware

-

Using Teamware

+

Teamware is developed by the GATE team, an academic research group at The University of Sheffield. As a result, future funding relies on evidence of the impact that the software provides. If you use Teamware, please let us know using the contact form at gate.ac.uk. Please include details on grants, publications, commercial products etc. Any information that can help us to secure future funding for our work is greatly appreciated.

-

Citation

+

For published work that has used Teamware, please cite the project's GitHub repository. One way is to include a citation such as:

@@ -23,9 +23,14 @@ Please use the Cite this repository button at the top of the project's GitHub repository to get an up to date citation. -

Version

- This is Teamware version {{ appVersion }}. +

+ This instance is running Teamware version {{ appVersion }}. +

+ +

Source Code & Licensing

+

+ Teamware is free open source software available at the project's GitHub repository under the GNU Affero General Public License v3.

diff --git a/frontend/src/views/Annotate.vue b/frontend/src/views/Annotate.vue index 540dc8ac..d77eabb0 100644 --- a/frontend/src/views/Annotate.vue +++ b/frontend/src/views/Annotate.vue @@ -272,7 +272,7 @@ export default { //Fills the annotation renderer with data if (this.$refs.annotationRenderer) { this.$refs.annotationRenderer.clearForm() - if (this.currentAnnotationTask.annotation_data != null) + if (this.currentAnnotationTask && this.currentAnnotationTask.annotation_data != null) this.$refs.annotationRenderer.setAnnotationData(this.currentAnnotationTask.annotation_data) } diff --git a/frontend/src/views/Cookies.vue b/frontend/src/views/Cookies.vue new file mode 100644 index 00000000..a8543237 --- /dev/null +++ b/frontend/src/views/Cookies.vue @@ -0,0 +1,61 @@ + + + + + diff --git a/frontend/src/views/ManageUsers.vue b/frontend/src/views/ManageUsers.vue index 50892a1b..39e5c758 100644 --- a/frontend/src/views/ManageUsers.vue +++ b/frontend/src/views/ManageUsers.vue @@ -105,8 +105,30 @@ Generate activation e-mail + + + Delete user account + + + + + Warning, this action is permanent! The personal details in the account will be removed and the user will no longer be able to login to the system using this account. + + + Also remove any annotations, projects and documents associated with the user: + + + + + + @@ -114,13 +136,14 @@ diff --git a/frontend/src/views/PrivacyPolicy.vue b/frontend/src/views/PrivacyPolicy.vue new file mode 100644 index 00000000..f705e2cc --- /dev/null +++ b/frontend/src/views/PrivacyPolicy.vue @@ -0,0 +1,157 @@ + + diff --git a/frontend/src/views/Register.vue b/frontend/src/views/Register.vue index 33f42d16..2c866837 100644 --- a/frontend/src/views/Register.vue +++ b/frontend/src/views/Register.vue @@ -19,6 +19,8 @@ +

By registering to use Teamware, you confirm that you are over 18 years of age and have read and agreed to Teamware's privacy policy and terms & conditions.

+