From 8aaf1c4d99dfb3ea2297eb8b7ce13342aec78d12 Mon Sep 17 00:00:00 2001 From: Ian Roberts Date: Fri, 5 May 2023 11:31:07 +0100 Subject: [PATCH 01/35] Test that removing or rejecting an annotator from a project should clear their "pending" annotations --- backend/tests/test_models.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 33a3fba1..0dfa9bcb 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -411,6 +411,26 @@ def test_reject_annotator(self): self.assertEqual(AnnotatorProject.COMPLETED, annotator_project.status) self.assertEqual(True, annotator_project.rejected) + def test_remove_annotator_clears_pending(self): + annotator = self.annotators[0] + # Start a task - should be one pending annotation + self.project.get_annotator_task(annotator) + self.assertEqual(1, annotator.annotations.filter(status=Annotation.PENDING).count()) + + # remove annotator from project - pending annotations should be cleared + self.project.remove_annotator(annotator) + self.assertEqual(0, annotator.annotations.filter(status=Annotation.PENDING).count()) + + def test_reject_annotator_clears_pending(self): + annotator = self.annotators[0] + # Start a task - should be one pending annotation + self.project.get_annotator_task(annotator) + self.assertEqual(1, annotator.annotations.filter(status=Annotation.PENDING).count()) + + # reject annotator from project - pending annotations should be cleared + self.project.reject_annotator(annotator) + self.assertEqual(0, annotator.annotations.filter(status=Annotation.PENDING).count()) + def test_num_documents(self): self.assertEqual(self.project.num_documents, self.num_docs) From 43bca74cbd80d70ef77e5214903a8b632bf142e0 Mon Sep 17 00:00:00 2001 From: Ian Roberts Date: Fri, 5 May 2023 11:31:25 +0100 Subject: [PATCH 02/35] Errors should be raised, not returned --- backend/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/models.py b/backend/models.py index f0b93af7..f5b6ed35 100644 --- a/backend/models.py +++ b/backend/models.py @@ -623,7 +623,7 @@ def get_current_annotator_task(self, user): annotation = current_annotations.first() if annotation.document.project != self: - return RuntimeError( + raise RuntimeError( "The annotation doesn't belong to this project! Annotator should only work on one project at a time") return annotation From aa949ee5249a7c05b0a799bc7fce87de5a7008a0 Mon Sep 17 00:00:00 2001 From: Ian Roberts Date: Fri, 5 May 2023 11:31:47 +0100 Subject: [PATCH 03/35] Clear pending annotation(s) when rejecting a user from a project Fixes #361 --- backend/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/models.py b/backend/models.py index f5b6ed35..f4b7d47f 100644 --- a/backend/models.py +++ b/backend/models.py @@ -485,6 +485,9 @@ def reject_annotator(self, user, finished_time=timezone.now()): annotator_project.status = AnnotatorProject.COMPLETED annotator_project.rejected = True annotator_project.save() + + Annotation.clear_all_pending_user_annotations(user) + except ObjectDoesNotExist: raise Exception(f"User {user.username} is not an annotator of the project.") From 97c387fdf3a388d2194eb38f8f3ac90ce42a700e Mon Sep 17 00:00:00 2001 From: davidwilby Date: Thu, 11 May 2023 14:02:28 +0100 Subject: [PATCH 04/35] correct DOI --- CITATION.cff | 2 +- README.md | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/CITATION.cff b/CITATION.cff index e3e14ade..c2dabc4c 100644 --- a/CITATION.cff +++ b/CITATION.cff @@ -26,7 +26,7 @@ identifiers: - description: The collection of archived snapshots of all versions of GATE Teamware 2 type: doi - value: 10.5281/zenodo.7821718 + value: 10.5281/zenodo.7899193 keywords: - NLP - machine learning diff --git a/README.md b/README.md index b7d9927f..c76a92b9 100644 --- a/README.md +++ b/README.md @@ -2,13 +2,13 @@ ![](/frontend/public/static/img/gate-teamware-logo.svg "GATE Teamware") -[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7821718.svg)](https://doi.org/10.5281/zenodo.7821718) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.7899193.svg)](https://doi.org/10.5281/zenodo.7899193) A web application for collaborative document annotation. Full documentation can be [found here][docs]. -GATE teamware provides a flexible web app platform for managing classification of documents by human annotators. +GATE Teamware provides a flexible web app platform for managing classification of documents by human annotators. ## Key Features * Configure annotation options using a highly flexible JSON config. @@ -63,7 +63,7 @@ Teamware is developed by the [GATE](https://gate.ac.uk) team, an academic resear ## Citation For published work that has used Teamware, please cite this repository. One way is to include a citation such as: -> Karmakharm, T., Wilby, D., Roberts, I., & Bontcheva, K. (2022). GATE Teamware (Version 0.1.4) [Computer software]. https://github.com/GateNLP/gate-teamware +> Karmakharm, T., Wilby, D., Roberts, I., & Bontcheva, K. (2022). GATE Teamware (Version 2.1.0) [Computer software]. https://github.com/GateNLP/gate-teamware Please use the `Cite this repository` button at the top of the [project's GitHub repository](https://github.com/GATENLP/gate-teamware) to get an up to date citation. From 1ab0cf2d9ff53bddb0b548c7ca78f1d478874c08 Mon Sep 17 00:00:00 2001 From: davidwilby Date: Thu, 11 May 2023 15:06:38 +0100 Subject: [PATCH 05/35] manage version in readme citation section as well --- version.py | 38 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/version.py b/version.py index 6fcd81b1..48fe0199 100644 --- a/version.py +++ b/version.py @@ -1,11 +1,13 @@ import json import yaml +import re import sys PACKAGE_JSON_FILE_PATH = "package.json" DOCS_PACKAGE_JSON_FILE_PATH = "docs/package.json" CITATION_FILE_PATH = "CITATION.cff" MASTER_VERSION_FILE = "VERSION" +README_FILE_PATH = "README.md" def check(): """ @@ -22,12 +24,15 @@ def check(): citation_file = yaml.safe_load(f) citation_version = citation_file['version'] - print(f"CITATION.cff version is {citation_version}") + print(f"{CITATION_FILE_PATH} version is {citation_version}") + + readme_version = get_readme_version(README_FILE_PATH) + print(f"{README_FILE_PATH} version is {readme_version}") master_version = get_master_version() print(f"VERSION file version is {master_version}") - if js_version != master_version or docs_js_version != master_version or citation_version != master_version: + if js_version != master_version or docs_js_version != master_version or citation_version != master_version or readme_version != master_version: print("One or more versions does not match") sys.exit(1) else: @@ -40,6 +45,20 @@ def get_package_json_version(file_path: str) -> str: js_version = package_json['version'] return js_version +def get_readme_version(file_path: str) -> str: + with open(file_path, 'r') as f: + readme_text = f.read() + + match = re.search(r'\(Version (.*)\)', readme_text) + + if match is None: + print(f"No version found in {README_FILE_PATH}.") + return + elif len(match.groups()) > 1: + print(f"{len(match.groups())} matches found in {README_FILE_PATH}, expected 1.") + return + else: + return match.groups(1)[0] def get_master_version(): with open(MASTER_VERSION_FILE, "r") as f: @@ -56,6 +75,8 @@ def update(): update_package_json_version(DOCS_PACKAGE_JSON_FILE_PATH, master_version) + update_readme_version(README_FILE_PATH, master_version) + with open(CITATION_FILE_PATH, "r") as f: citation_file = yaml.safe_load(f) print(f"Writing master version {master_version} to {CITATION_FILE_PATH}") @@ -73,6 +94,19 @@ def update_package_json_version(file_path:str, version_no:str): package_json['version'] = version_no json.dump(package_json, f, indent=2) +def update_readme_version(file_path:str, version_no:str): + with open(file_path, 'r') as f: + readme_text = f.read() + + readme_text = re.sub( + r'\(Version (.*)\)', + f'(Version {version_no})', + readme_text + ) + + with open(file_path, 'w') as f: + f.write(readme_text) + if __name__ == "__main__": if sys.argv[1] == 'check': From 664f8da808c3b7f631a50be82e1e0ff96f51c937 Mon Sep 17 00:00:00 2001 From: davidwilby Date: Thu, 11 May 2023 15:20:43 +0100 Subject: [PATCH 06/35] use safer regex pattern --- version.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/version.py b/version.py index 48fe0199..c12d33ac 100644 --- a/version.py +++ b/version.py @@ -8,6 +8,7 @@ CITATION_FILE_PATH = "CITATION.cff" MASTER_VERSION_FILE = "VERSION" README_FILE_PATH = "README.md" +README_VERSION_REGEX = r"\(Version ([^)]*)\)" def check(): """ @@ -49,7 +50,7 @@ def get_readme_version(file_path: str) -> str: with open(file_path, 'r') as f: readme_text = f.read() - match = re.search(r'\(Version (.*)\)', readme_text) + match = re.search(README_VERSION_REGEX, readme_text) if match is None: print(f"No version found in {README_FILE_PATH}.") @@ -99,7 +100,7 @@ def update_readme_version(file_path:str, version_no:str): readme_text = f.read() readme_text = re.sub( - r'\(Version (.*)\)', + README_VERSION_REGEX, f'(Version {version_no})', readme_text ) From 0d2e116a4967293cf450e83633397f986fab9c51 Mon Sep 17 00:00:00 2001 From: davidwilby Date: Thu, 11 May 2023 15:45:19 +0100 Subject: [PATCH 07/35] enable version.py update to take version number as argument --- docs/docs/developerguide/releases.md | 2 +- version.py | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 6 deletions(-) diff --git a/docs/docs/developerguide/releases.md b/docs/docs/developerguide/releases.md index ec14d1f2..8a08d12b 100644 --- a/docs/docs/developerguide/releases.md +++ b/docs/docs/developerguide/releases.md @@ -9,7 +9,7 @@ Note: Releases are always made from the `master` branch of the repository. 1. **Update the changelog** - This has to be done manually, go through any pull requests to `dev` since the last release. - In github pull requests page, use the search term `is:pr merged:>=yyyy-mm-dd` to find all merged PR from the date since the last version change. - Include the changes in the `CHANGELOG.md` file; the changelog section _MUST_ begin with a level-two heading that starts with the relevant version number in square brackets (`## [N.M.P] Optional descriptive suffix`) as the GitHub workflow that creates a release from the eventual tag depends on this pattern to find the right release notes. Each main item within the changelog should have a link to the originating PR e.g. \[#123\](https://github.com/GateNLP/gate-teamware/pull/123). -1. **Update and check the version numbers** - from the teamware directory run `python version.py check` to check whether all version numbers are up to date. If not, update the master `VERSION` file and run `python version.py update` to update all other version numbers and commit the result. Note that `version.py` requires `pyyaml` for reading `CITATION.cff`, `pyyaml` is included in Teamware's dependencies. +1. **Update and check the version numbers** - from the teamware directory run `python version.py check` to check whether all version numbers are up to date. If not, update the master `VERSION` file and run `python version.py update` to update all other version numbers and commit the result. Alternatively, run `python version.py update ` where `` is the version number to update to, e.g. `python version.py update 2.1.0`. Note that `version.py` requires `pyyaml` for reading `CITATION.cff`, `pyyaml` is included in Teamware's dependencies. 1. **Create a version of the documentation** - Run `npm run docs:create_version`, this will archive the current version of the documentation using the version number in `package.json`. 1. **Create a pull request from `dev` to `master`** including any changes to `CHANGELOG.md`, `VERSION`. 1. **Create a tag** - Once the dev-to-master pull request has been merged, create a tag from the resulting `master` branch named `vN.M.P` (i.e. the new version number prefixed with the letter `v`). This will trigger two GitHub workflows: diff --git a/version.py b/version.py index c12d33ac..51faf08f 100644 --- a/version.py +++ b/version.py @@ -1,3 +1,4 @@ +import argparse import json import yaml import re @@ -15,6 +16,8 @@ def check(): Intended for use in CI pipelines, checks versions in files and exits with non-zero exit code if they don't match. """ + print("Checking versions...") + js_version = get_package_json_version(PACKAGE_JSON_FILE_PATH) print(f"package.json version is {js_version}") @@ -66,11 +69,15 @@ def get_master_version(): master_version = f.readline().strip() return master_version -def update(): +def update(master_version:str = None): """ Updates all versions to match the master version file. """ - master_version = get_master_version() + if master_version is None: + master_version = get_master_version() + else: + with open(MASTER_VERSION_FILE, 'w') as f: + f.write(master_version) update_package_json_version(PACKAGE_JSON_FILE_PATH, master_version) @@ -111,10 +118,14 @@ def update_readme_version(file_path:str, version_no:str): if __name__ == "__main__": if sys.argv[1] == 'check': - print("Checking versions...") + + if len(sys.argv) > 2: + print('WARNING: Additional arguments not supported for "check"') check() elif sys.argv[1] == 'update': - print("Updating versions...") - update() + if len(sys.argv) > 2: + update(sys.argv[2]) + else: + update() else: print(f"Unknown function {sys.argv[1]}, available functions are 'check' and 'update'.") From 59d3617a9ef4d98f99009a1838d1bfdc4f24131d Mon Sep 17 00:00:00 2001 From: Twin Karmakharm Date: Fri, 12 May 2023 15:32:17 +0100 Subject: [PATCH 08/35] Added fadein transition --- frontend/src/views/Annotate.vue | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/frontend/src/views/Annotate.vue b/frontend/src/views/Annotate.vue index 80b1e75f..a19f9f0a 100644 --- a/frontend/src/views/Annotate.vue +++ b/frontend/src/views/Annotate.vue @@ -103,6 +103,7 @@ :doc_preannotation_field="currentAnnotationTask.document_pre_annotation_field" :allow_document_reject="isLatestTask() && currentAnnotationTask.allow_document_reject" :clear_after_submit="clearFormAfterSubmit" + :class="{documentChanged : annotationRendererTransitionEnabled}" @submit="submitHandler" @reject="rejectHandler" > @@ -188,6 +189,7 @@ export default { showLeaveProjectModal: false, showStageIntroCard: false, showThankyouCard: false, + annotationRendererTransitionEnabled: false, } }, @@ -195,6 +197,12 @@ export default { ...mapActions(["getUserAnnotationTask", "getUserAnnotationTaskWithID", "completeUserAnnotationTask", "rejectUserAnnotationTask", "annotatorLeaveProject", "changeAnnotation"]), + triggerAnnotationRendererTransition(){ + this.annotationRendererTransitionEnabled = true + setTimeout(()=>{ + this.annotationRendererTransitionEnabled = false + }, 500) + }, getAnnotationContainerBgClass() { return { "mt-4": true, @@ -233,6 +241,7 @@ export default { // Go to previous task in the history toPreviousTask() { if (this.hasPreviousTask()) { + this.triggerAnnotationRendererTransition() this.currentTaskIndex += 1 this.getCurrentTask() } @@ -240,12 +249,14 @@ export default { // Goes to next task in history toNextTask() { if (this.hasNextTask()) { + this.triggerAnnotationRendererTransition() this.currentTaskIndex -= 1 this.getCurrentTask() } }, // Goes to the latest task toLatestTask() { + this.triggerAnnotationRendererTransition() this.currentTaskIndex = 0 this.getCurrentTask() }, @@ -282,6 +293,7 @@ export default { } }, async submitHandler(value, time) { + this.triggerAnnotationRendererTransition() if (this.isLatestTask()) { // Complete a current task @@ -313,6 +325,8 @@ export default { }, async rejectHandler() { + this.triggerAnnotationRendererTransition() + try { await this.rejectUserAnnotationTask(this.annotationTask.annotation_id) } catch (e) { @@ -419,4 +433,17 @@ export default { } +.documentChanged { + animation: fadein 0.5s; +} + +@keyframes fadein { + from{ + opacity: 0; + } + to{ + opacity: 1.0; + } +} + From 0db010c074d0f54c05dbca7458b7bd962ba211ab Mon Sep 17 00:00:00 2001 From: Twin Karmakharm Date: Fri, 12 May 2023 15:41:05 +0100 Subject: [PATCH 09/35] Put transition trigger in getCurrentTask() instead --- frontend/src/views/Annotate.vue | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/frontend/src/views/Annotate.vue b/frontend/src/views/Annotate.vue index a19f9f0a..49c08d99 100644 --- a/frontend/src/views/Annotate.vue +++ b/frontend/src/views/Annotate.vue @@ -241,7 +241,6 @@ export default { // Go to previous task in the history toPreviousTask() { if (this.hasPreviousTask()) { - this.triggerAnnotationRendererTransition() this.currentTaskIndex += 1 this.getCurrentTask() } @@ -249,14 +248,12 @@ export default { // Goes to next task in history toNextTask() { if (this.hasNextTask()) { - this.triggerAnnotationRendererTransition() this.currentTaskIndex -= 1 this.getCurrentTask() } }, // Goes to the latest task toLatestTask() { - this.triggerAnnotationRendererTransition() this.currentTaskIndex = 0 this.getCurrentTask() }, @@ -287,13 +284,15 @@ export default { this.$refs.annotationRenderer.setAnnotationData(this.currentAnnotationTask.annotation_data) } + this.triggerAnnotationRendererTransition() + } catch (e) { toastError("Could not get annotation task", e, this) console.log(e) } }, async submitHandler(value, time) { - this.triggerAnnotationRendererTransition() + if (this.isLatestTask()) { // Complete a current task @@ -325,7 +324,6 @@ export default { }, async rejectHandler() { - this.triggerAnnotationRendererTransition() try { await this.rejectUserAnnotationTask(this.annotationTask.annotation_id) From 5ac430471dc20374f9283b919d6c5f01a9ab2838 Mon Sep 17 00:00:00 2001 From: Twin Karmakharm Date: Fri, 12 May 2023 17:21:11 +0100 Subject: [PATCH 10/35] Stop throwing error on initialise rpc call, added tests to check that page loads on bad status and blank response --- cypress/e2e/connection-error.spec.js | 31 ++++++++++++++++++++++++++++ frontend/src/jrpc/index.js | 3 ++- frontend/src/store/index.js | 2 +- 3 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 cypress/e2e/connection-error.spec.js diff --git a/cypress/e2e/connection-error.spec.js b/cypress/e2e/connection-error.spec.js new file mode 100644 index 00000000..d1c9ae41 --- /dev/null +++ b/cypress/e2e/connection-error.spec.js @@ -0,0 +1,31 @@ + + +describe("Test connection errors", () =>{ + + it("Test initialise with 404 response", () => { + + cy.intercept("POST", "/rpc/", + { + statusCode: 404, + body: { + + } + } + ).as('rpcCalls') // and assign an alias + cy.visit("/") + cy.contains("TEAMWARE").should("be.visible") + }) + + it("Test initialise with blank response", () =>{ + cy.intercept( + { + method: 'POST', // Route all GET requests + url: '/rpc/', // that have a URL that matches '/users/*' + }, + [] // and force the response to be: [] + ).as('rpcCalls') // and assign an alias + cy.visit("/") + cy.contains("TEAMWARE").should("be.visible") + }) + +}) diff --git a/frontend/src/jrpc/index.js b/frontend/src/jrpc/index.js index c62277ab..0b9c7187 100644 --- a/frontend/src/jrpc/index.js +++ b/frontend/src/jrpc/index.js @@ -73,8 +73,9 @@ class JRPCClient{ // Not a fully formed json-rpc response, may be a problem with the endpoint // or connection to the server // TODO: Do we care about specific connection errors? - const err = new Error("Unknown error") + const err = new Error("There appears to be a problem with the connection") err.code = JRPCClient.INTERNAL_ERROR + err.response = e.response throw err } } diff --git a/frontend/src/store/index.js b/frontend/src/store/index.js index e90b6bdd..3c8ead56 100644 --- a/frontend/src/store/index.js +++ b/frontend/src/store/index.js @@ -68,7 +68,7 @@ export default new Vuex.Store({ }catch(e){ console.log(e) - throw e + // Error is not thrown as this function is called before the UI is loaded } }, async login({dispatch, commit}, params) { From 89d0cd530efdeb6b1729f729fa6af07656751519 Mon Sep 17 00:00:00 2001 From: Ian Roberts Date: Tue, 16 May 2023 19:12:48 +0100 Subject: [PATCH 11/35] Acquire a row lock on the user before checking for an annotation task, since otherwise concurrent calls can cause a race condition and assign two different documents to the same user at the same time, causing errors later. Fixes #374 --- backend/models.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/backend/models.py b/backend/models.py index f4b7d47f..c3ab27ba 100644 --- a/backend/models.py +++ b/backend/models.py @@ -64,6 +64,13 @@ class ServiceUser(AbstractUser): agreed_privacy_policy = models.BooleanField(default=False) is_deleted = models.BooleanField(default=False) + def lock_user(self): + """ + Lock this user with a SELECT FOR UPDATE. This method must be called within a transaction, + the lock will be released when the transaction commits or rolls back. + """ + return type(self).objects.filter(id=self.id).select_for_update().get() + @property def has_active_project(self): return self.annotatorproject_set.filter(status=AnnotatorProject.ACTIVE).count() > 0 @@ -592,6 +599,9 @@ def get_annotator_task(self, user): user from annotator list if there's no more tasks or user reached quota. """ + # Lock required to prevent concurrent calls from assigning two different tasks + # to the same user + user = user.lock_user() annotation = self.get_current_annotator_task(user) if annotation: # User has existing task From 1c109f334478f1004f8d6c7078cc46dd24b205b1 Mon Sep 17 00:00:00 2001 From: Ian Roberts Date: Mon, 24 Apr 2023 19:06:08 +0100 Subject: [PATCH 12/35] Initial attempt at implementing #164 (conditional annotation) Added an "if" option to all annotation widgets, whose value is an expression that has access to the document data _and_ the current state of annotationOutput. Widgets are only rendered if their if expression evaluates to a truthy value. The conditionals are re-evaluated every time the annotation data changes, so widgets can appear or disappear depending on the current state. --- frontend/package-lock.json | 102 ++++++++++++++++++ frontend/package.json | 7 ++ .../src/components/AnnotationRenderer.vue | 63 ++++++++++- frontend/src/components/JsonEditor.vue | 3 +- 4 files changed, 171 insertions(+), 4 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index 18a82ac1..3ae2dc73 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -8,6 +8,12 @@ "name": "frontend", "version": "2.0.0", "dependencies": { + "@jsep-plugin/arrow": "^1.0.5", + "@jsep-plugin/new": "^1.0.3", + "@jsep-plugin/numbers": "^1.0.1", + "@jsep-plugin/object": "^1.2.1", + "@jsep-plugin/regex": "^1.0.3", + "@jsep-plugin/spread": "^1.0.2", "@vitejs/plugin-vue2": "^2.2.0", "axios": "^0.21.1", "bootstrap": "^4.6.0", @@ -17,6 +23,7 @@ "css-loader": "^5.2.1", "csvtojson": "^2.0.10", "js-cookie": "^2.2.1", + "jse-eval": "^1.5.2", "jsonl-parse-stringify": "^1.0.1", "jszip": "^3.7.1", "lodash": "^4.17.21", @@ -355,6 +362,72 @@ "@jridgewell/sourcemap-codec": "1.4.14" } }, + "node_modules/@jsep-plugin/arrow": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@jsep-plugin/arrow/-/arrow-1.0.5.tgz", + "integrity": "sha512-4Q9/6nETEf79DQdyynPk9G5CvYGw/TyRAw6IpkiIBm1z6eyDyjhcLjYxmBCqlKIUvjS8h8hfU8MzSjQRSntK5Q==", + "engines": { + "node": ">= 10.16.0" + }, + "peerDependencies": { + "jsep": "^0.4.0||^1.0.0" + } + }, + "node_modules/@jsep-plugin/new": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@jsep-plugin/new/-/new-1.0.3.tgz", + "integrity": "sha512-TsXGyHeK8yKinrNXirljPvbEVth/3YIFH7oufq2E96CAuRzxYS9jEOA0u9/kVe5su1RBzTJMXG+jlWZ4gwLMiQ==", + "engines": { + "node": ">= 10.16.0" + }, + "peerDependencies": { + "jsep": "^0.4.0||^1.0.0" + } + }, + "node_modules/@jsep-plugin/numbers": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/@jsep-plugin/numbers/-/numbers-1.0.1.tgz", + "integrity": "sha512-SdZgumrnEKcKSr1IA+yHY9RXwnGFO8BBh3hXHKxJUIodEsl1tFrKtapHGKtbi2jSX9RIig9sfumNJfI1/MYHng==", + "engines": { + "node": ">= 10.16.0" + }, + "peerDependencies": { + "jsep": "^0.4.0||^1.0.0" + } + }, + "node_modules/@jsep-plugin/object": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@jsep-plugin/object/-/object-1.2.1.tgz", + "integrity": "sha512-6YoZP80h2QFCuxyqj+OvoqEnTu2r5cSRpgpvGauWlvnevFP/F/dibpvXDpnHeqwT2FIzzvg47YOe3QD/UT8vJw==", + "engines": { + "node": ">= 10.16.0" + }, + "peerDependencies": { + "jsep": "^0.4.0||^1.0.0" + } + }, + "node_modules/@jsep-plugin/regex": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/@jsep-plugin/regex/-/regex-1.0.3.tgz", + "integrity": "sha512-XfZgry4DwEZvSFtS/6Y+R48D7qJYJK6R9/yJFyUFHCIUMEEHuJ4X95TDgJp5QkmzfLYvapMPzskV5HpIDrREug==", + "engines": { + "node": ">= 10.16.0" + }, + "peerDependencies": { + "jsep": "^0.4.0||^1.0.0" + } + }, + "node_modules/@jsep-plugin/spread": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@jsep-plugin/spread/-/spread-1.0.2.tgz", + "integrity": "sha512-yYUYcgfWnttw1QSwEu4PDDRKw/iT/BJcBhnDkxEuJOBmMa+W/PTNETBM9aoBEEglkYDAaprp58DYlU5stKCbFg==", + "engines": { + "node": ">= 10.16.0" + }, + "peerDependencies": { + "jsep": "^0.4.0||^1.0.0" + } + }, "node_modules/@kurkle/color": { "version": "0.3.2", "resolved": "https://registry.npmjs.org/@kurkle/color/-/color-0.3.2.tgz", @@ -3501,6 +3574,19 @@ "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", "dev": true }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -4526,6 +4612,22 @@ } } }, + "node_modules/jse-eval": { + "version": "1.5.2", + "resolved": "https://registry.npmjs.org/jse-eval/-/jse-eval-1.5.2.tgz", + "integrity": "sha512-o2Uc6F5CnfXTx5l+BRWSSBr4e8cNgebuxAaIQghKqk6cda0TazUDf/1Y/KhjWHN6xptnaQA3cZxhvYVSJca/YQ==", + "dependencies": { + "jsep": "^1.2.0" + } + }, + "node_modules/jsep": { + "version": "1.3.8", + "resolved": "https://registry.npmjs.org/jsep/-/jsep-1.3.8.tgz", + "integrity": "sha512-qofGylTGgYj9gZFsHuyWAN4jr35eJ66qJCK4eKDnldohuUoQFbU3iZn2zjvEbd9wOAhP9Wx5DsAAduTyE1PSWQ==", + "engines": { + "node": ">= 10.16.0" + } + }, "node_modules/json-parse-even-better-errors": { "version": "2.3.1", "resolved": "https://registry.npmjs.org/json-parse-even-better-errors/-/json-parse-even-better-errors-2.3.1.tgz", diff --git a/frontend/package.json b/frontend/package.json index 953ef606..1e895611 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -13,6 +13,12 @@ "test": "run-s test:unit test:component" }, "dependencies": { + "@jsep-plugin/arrow": "^1.0.5", + "@jsep-plugin/new": "^1.0.3", + "@jsep-plugin/numbers": "^1.0.1", + "@jsep-plugin/object": "^1.2.1", + "@jsep-plugin/regex": "^1.0.3", + "@jsep-plugin/spread": "^1.0.2", "@vitejs/plugin-vue2": "^2.2.0", "axios": "^0.21.1", "bootstrap": "^4.6.0", @@ -22,6 +28,7 @@ "css-loader": "^5.2.1", "csvtojson": "^2.0.10", "js-cookie": "^2.2.1", + "jse-eval": "^1.5.2", "jsonl-parse-stringify": "^1.0.1", "jszip": "^3.7.1", "lodash": "^4.17.21", diff --git a/frontend/src/components/AnnotationRenderer.vue b/frontend/src/components/AnnotationRenderer.vue index 831cd9bc..751bec53 100644 --- a/frontend/src/components/AnnotationRenderer.vue +++ b/frontend/src/components/AnnotationRenderer.vue @@ -1,6 +1,6 @@ @@ -77,6 +90,10 @@ export default { DocumentType, answerBgColor: {}, conditions: [], + exceptions: { + parse: [], + evaluate: [] + }, } }, props: { @@ -118,6 +135,9 @@ export default { clear_after_submit: { default: true, type: Boolean + }, + show_expression_errors: { + default: null } }, computed: { @@ -128,6 +148,7 @@ export default { return null }, shownElements() { + this.exceptions.evaluate = [] if (!this.config) { return []; } @@ -137,7 +158,11 @@ export default { document: this.document, annotation: this.annotationData, }); - } catch (_) { + } catch (e) { + this.exceptions.evaluate.push({ + config: elemConfig, + error: e + }) // treat error as "don't show" return false; } @@ -183,15 +208,20 @@ export default { const truePredicate = () => true; this.validation = {} this.conditions = [] + this.exceptions.parse = [] for (let elemConfig of config) { this.validation[elemConfig.name] = null let thisElementPredicate = truePredicate; if (elemConfig["if"]) { try { thisElementPredicate = compile(elemConfig["if"]) - } catch (_) { + } catch (e) { // error compiling expression -> treat the same as if the element // did not have an "if" at all, and always show it. + this.exceptions.parse.push({ + config: elemConfig, + error: e + }) } } this.conditions.push(thisElementPredicate) diff --git a/frontend/src/components/ProjectConfiguration.vue b/frontend/src/components/ProjectConfiguration.vue index 2af40077..b358d718 100644 --- a/frontend/src/components/ProjectConfiguration.vue +++ b/frontend/src/components/ProjectConfiguration.vue @@ -169,6 +169,7 @@ From 4a4b1130a67780aacbe86830cc20473cb4decedd Mon Sep 17 00:00:00 2001 From: Ian Roberts Date: Thu, 11 May 2023 00:45:46 +0100 Subject: [PATCH 22/35] Added an example of a conditional widget that uses document as well as annotation data, and added discussion of error handling and how to deal with unset annotation values --- .../docs/manageradminguide/config_examples.js | 44 +++++++++++ docs/docs/manageradminguide/project_config.md | 73 +++++++++++++++++++ 2 files changed, 117 insertions(+) diff --git a/docs/docs/manageradminguide/config_examples.js b/docs/docs/manageradminguide/config_examples.js index 9114c5f6..d6e40454 100644 --- a/docs/docs/manageradminguide/config_examples.js +++ b/docs/docs/manageradminguide/config_examples.js @@ -223,6 +223,50 @@ export default { "valError": "Please specify a URI (starting http:, https: or urn:)" } ], + configConditional2: [ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + }, + { + "name": "reason", + "type": "text", + "title": "Why do you disagree with the suggested value?", + "if": "annotation.sentiment !== document.preanno.sentiment" + } + ], + docsConditional2: [ + { + "text": "I love the thing!", + "preanno": { + "sentiment": "positive" + } + }, + { + "text": "I hate the thing!", + "preanno": { + "sentiment": "negative" + } + }, + { + "text": "The thing is ok, I guess...", + "preanno": { + "sentiment": "neutral" + } + } + ], doc1: {text: "Sometext with html"}, diff --git a/docs/docs/manageradminguide/project_config.md b/docs/docs/manageradminguide/project_config.md index fc31bb38..d3d4ea3d 100644 --- a/docs/docs/manageradminguide/project_config.md +++ b/docs/docs/manageradminguide/project_config.md @@ -566,6 +566,62 @@ The following simple example shows how you might implement an "Other (please spe ``` +Note that validation rules (such as `optional`, `minSelected` or `regex`) are not applied to components that are hidden by an `if` expression - hidden components will never be included in the annotation output, even if they would be considered "required" had they been visible. + +Components can also be made conditional on properties of the _document_, or a combination of the document and the annotation values, for example + + + +**Project configuration** + +```json +[ + { + "name": "htmldisplay", + "type": "html", + "text": "{{{text}}}" + }, + { + "name": "sentiment", + "type": "radio", + "title": "Sentiment", + "description": "Please select a sentiment of the text above.", + "options": [ + {"value": "negative", "label": "Negative"}, + {"value": "neutral", "label": "Neutral"}, + {"value": "positive", "label": "Positive"} + ] + }, + { + "name": "reason", + "type": "text", + "title": "Why do you disagree with the suggested value?", + "if": "annotation.sentiment !== document.preanno.sentiment" + } +] +``` + +**Documents** + +```json +[ + { + "text": "I love the thing!", + "preanno": { "sentiment": "positive" } + }, + { + "text": "I hate the thing!", + "preanno": { "sentiment": "negative" } + }, + { + "text": "The thing is ok, I guess...", + "preanno": { "sentiment": "neutral" } + } +] +``` + + + The full list of supported constructions is as follows: - the `annotation` variable refers to the current state of the annotation components for this document @@ -599,6 +655,23 @@ The full list of supported constructions is as follows: - `all(e in document.scores, e.value < 0.7)` (assuming `scores` is an object mapping labels to scores, e.g. `{"scores": {"positive": 0.5, "negative": 0.3}}`) - when testing a predicate against an _object_ each entry has `.key` and `.value` properties giving the key and value of the current entry - on a null, undefined or empty array/object, `any` will return _false_ (since there are no items that pass the test) and `all` will return _true_ (since there are no items that _fail_ the test) + - the predicate is optional - `any(arrayExpression)` resolves to `true` if any item in the array has a value that JavaScript considers to be "truthy", i.e. anything other than the number 0, the empty string, null or undefined. So `any(annotation.myCheckbox)` is a convenient way to check whether _at least one_ option has been selected in a `checkbox` component. + +If the `if` expression for a particular component is _syntactically invalid_ (missing operands, mis-matched brackets, etc.) then the condition will be ignored and the component will always be displayed as though it did not have an `if` expression at all. Conversely, if the expression is valid but an error occurs while _evaluating_ it, this will be treated the same as if the expression returned `false`, and the associated component will not be displayed. The behaviour is this way around as the most common reason for errors during evaluation is attempting to refer to annotation components that have not yet been filled in - if this is not appropriate in your use case you must account for the possibility within your expression. For example, suppose `confidence` is a `radio` or `selector` component with values ranging from 1 to 5, then another component that declares + +``` +"if": "annotation.confidence && annotation.confidence < 4"` +``` + +will hide this component if `confidence` is unset, displaying it only if `confidence` is set to a value less than 4, whereas + +``` +"if": "!annotation.confidence || annotation.confidence < 4" +``` + +will hide this component only if `confidence` is actually _set_ to a value of 4 or greater - it will _show_ this component if `confidence` is unset. Either approach may be correct depending on your project's requirements. + +To assist managers in authoring project configurations with `if` conditions, the "preview" mode on the project configuration page will display details of any errors that occur when parsing the expressions, or when evaluating them against the **Document input preview** data. You are encouraged to test your expressions thoroughly against a variety of inputs to ensure they behave as intended, before opening your project to annotators.