Init for MetadataManager (#67)

lib/workload/stateless/metadata_manager/src/.gitignore

@@ -137,3 +137,5 @@ dist
 dbschema/edgeql-js
 dbschema/queries.*
 dbschema/interfaces.ts
+
+asset/

lib/workload/stateless/metadata_manager/src/Dockerfile

@@ -0,0 +1,14 @@
+FROM node:20
+WORKDIR /app
+
+COPY Makefile package.json yarn.lock tsconfig.json jest.config.js ./
+COPY src/ ./src/
+COPY tests/ ./tests/
+
+COPY dbschema/queries ./dbschema/queries
+COPY dbschema/migrations ./dbschema/migrations
+COPY dbschema/*.esdl ./dbschema/
+
+RUN make install
+
+ENTRYPOINT [ "make", "start", "EDGEDB_HOST=edgedb" ]

lib/workload/stateless/metadata_manager/src/Makefile

@@ -0,0 +1,59 @@
+EDGEDB_HOST ?= localhost
+
+# Installation
+install:
+	@yarn install
+
+# To run docker compose
+up:
+	@docker compose up -d
+
+down:
+	@docker compose down
+
+stop:
+	@docker compose down
+
+# Build the app for deployment
+
+edgetypes:
+	@npx generate queries --target ts --file --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure # pragma: allowlist secret
+	@npx generate edgeql-js --target ts --output-dir dbschema/edgeql-js --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure # pragma: allowlist secret
+	@npx generate interfaces --file dbschema/interfaces.ts --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure # pragma: allowlist secret
+
+build: edgetypes
+	@rm -rf asset
+	@mkdir -p asset
+	@yarn run build-src
+	@cd dist && zip -r ../asset/src.zip .
+
+	@mkdir -p nodejs
+	@cp -r node_modules ./nodejs
+	@zip -r asset/dependency.zip nodejs
+	@rm -rf nodejs
+
+
+# Testing
+test:
+	@yarn test
+
+# Starting server
+start: edgetypes
+	@yarn start
+start-scenario-1: edgetypes
+	@yarn run insert-scenario-1
+	@yarn start
+
+# Inserting scenarios
+insert-scenario-2:
+	@yarn run insert-scenario-2
+
+# Shortcut of edgedb cli
+edgedb:
+	@edgedb --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure # pragma: allowlist secret
+edgedb-ui:
+	@edgedb ui --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure --no-server-check # pragma: allowlist secret
+edgedb-migrate:
+	@edgedb migrate --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure # pragma: allowlist secret
+edgedb-migration-create:
+	@edgedb migration create --dsn edgedb://orcabus_admin:admin@$(EDGEDB_HOST):5656 --tls-security insecure # pragma: allowlist secret

lib/workload/stateless/metadata_manager/src/README.md

@@ -1,46 +1,171 @@
-# Metadata manager
+# Metadata Manager for OrcaBus
 
-The `MetadataManager` is one of the microservice within the UMCCR OrcaBus that manages the metadata records used for the
-BioInformatics pipelines. The service will be responsible to make records to stay up to date and able to sync records
-from multiple sources.
+One of the microservices in the Orcabus handles all the metadata information.
 
-## Development
+## Schema
 
-### Git Checkout
-As this microservice is part of a monorepo from the main OrcaBus project, the process Git Flow process
-will follow on the root of this project.
+The EdgeDb schema looks like this as follows. You could also open this view with the make command: `make edgedb-ui` (see development
+section below)
 
-### Development
+![schema](./docs/edgedb-schema.png)
+
+Relationship between schema
+
+![relationship](docs/relationship.drawio.svg)
+
+The mapping between the metadata spreadsheet to the schema are
+| Metadata Header   | Table                | Field Name |
+|-------------------|----------------------|------------|
+| SubjectID         | `metadata::Subject`  | internalId |
+| ExternalSubjectID | `metadata::Subject`  | externalId |
+| SampleID          | `metadata::Specimen` | internalId |
+| ExternalSampleID  | `metadata::Specimen` | externalId |
+| Source            | `metadata::Specimen` | source     |
+| LibraryID         | `metadata::Library`  | internalId |
+| Phenotype         | `metadata::Library`  | phenotype  |
+| Quality           | `metadata::Library`  | quality    |
+| Type              | `metadata::Library`  | type       |
+| Assay             | `metadata::Library`  | assay      |
+| Workflow          | `metadata::Library`  | workflow   |
+| Coverage (X)      | `metadata::Library`  | coverage   |
+
+Notes:
+
+- The `orcabusId` is constructed uniquely for ocabusId with the structure of prefixes either `spc.`, `lib.`, `sbj.`
+  (for specimen, library, and subject, respectively) and followed by [ulid](https://github.com/ulid/spec) id.
 
-#### Setup
+  e.g. `sbj.01HF6B4D6GHCYT3ZVF0S6087W3`
+- The audit event schema will record the generation of success/failure of an insertion/deletion.
+
+  ```curl
+  select audit::AuditEvent {*} filter .id=<uuid>'04ec4232-82be-11ee-870c-6b78b11f565f';
+  {
+    audit::SystemAuditEvent {
+      id: 04ec4232-82be-11ee-870c-6b78b11f565f,
+      actionCategory: C,
+      actionDescription: 'Insert new subject record: sbj.01HF6B4DJPWGYZK8RZDN9BK454',
+      actionOutcome: success,
+      details: Json("{\"id\": \"04f3ea1e-82be-11ee-870c-4f6a4d334d37\", \"orcaBusId\": \"sbj.01HF6B4DJPWGYZK8RZDN9BK454\", \"externalId\": \"EXTSUB002\", \"internalId\": \"SBJ220002\"}"),
+      updatedDateTime: <datetime>'2023-11-14T07:18:33.366109Z',
+    },
+  }
+  ```
 
-Change the terminal directory from the OrcaBus root to this directory:
 
+## Local Deployment
+
+Requirement:
+
+- Docker
+- Node.js with Yarn
+
+```bash
+docker -v
+Docker version 20.10.12, build e91ed5707e
+
+node -v
+v20.9.0
 ```
-cd lib/workload/stateless/metadata_manager/src
+
+### Installation
+
+To install dependency, yarn is needed for this purpose
+
+```bash
+npm i -g yarn
 ```
 
-Requirement for this project:
-  - nodejs
-  - edgedb (see <https://www.edgedb.com/install>)
+To install all dependencies, a make file is provided and can be done by:
 
-To install the dependency of this project
+```bash
+make install
 ```
-yarn
+
+### Start
+
+
+To start the database and server use the following make command. This will run the server and database in
+docker-compose. The server will run at port 8080. (http://localhost:8080).
+The database used here is [EdgeDb](https://www.edgedb.com/) which is a graph-relational database that uses SQL as their
+underlying database. The configuration for this has been bundled up into a docker-compose and could be run as follows.
+
+```bash
+make up
 ```
 
-Setup the database
+
+To start the server up use the following command
+
+```bash
+make start
 ```
-yarn init-db
-yarn migrate-db
+
+We could also start data by resetting and inserting some mock test data.
+
+```bash
+make start-scenario-1
 ```
 
-#### Build
-```yarn edgetypes```
+This should run at localhost port 8080. Valid endpoints are as follows
+
+```bash
+
+http://localhost:8080/subject
+http://localhost:8080/library
+http://localhost:8080/specimen
+
+
+<!-- GraphQL -->
+http://localhost:8080/graphql/explore (UI)
+http://localhost:8080/graphql (Only for graphql POST method)
+```
+
+If you need to serve from existing metadata locally. You could copy and paste
+the `.xlsx` file into [./src/test-data/scenarios/metadata.xlsx](./src/test-data/scenarios/metadata.xlsx).
+Then to dump and reload with this metadata spreadsheet run the following:
+
+```bash
+make insert-scenario-2
+```
+
+### Testing
+
+```bash
+make test
+
+Test Suites: 2 passed, 2 total
+Tests:       6 passed, 6 total
+Snapshots:   0 total
+Time:        36.073 s
+```
+
+### Build
+
+To build for deployment the following will generate assets that could be deployed to lambdas.
+For more details on the deployment [deploy-readme](./deploy/README.md).
+
+```bash
+make build 
+```
+
+### Development
+
+edgedb-cli is expected to be installed in the dev environment. See the [installation guide](https://www.edgedb.com/install#macos-homebrew) for details.
+
+Homebrew user:
+
+```curl
+brew tap edgedb/tap
+brew install edgedb-cli
+```
 
+Since the EdgeDb is running in a container, some EdgeDb commands will need to pass in their connection parameter
+to get connected with the container. Some make commands are added in the makefile for convenience.
 
-### Run
-```yarn watch```
+List available makefile command equivalent to edgedb-cli:
 
-#### Testing
-```yarn test```
+- `make edgedb` (`edgedb`)- to run EdgeDb
+- `make edgedb-ui` (`edgedb ui`) - to open EdgeDb UI (username: `orcabus_admin`; password: `admin`) // pragma: allowlist secret
+- `make edgedb-migrate` (`edgedb migrate`) - to apply new migration
+- `make edgedb-migration-create` (`edgedb migration create`) - to create new migrations
+- `make edgetypes` - to make EdgeDb files (e.g. `*.edgeql`) recognizable by typescript

lib/workload/stateless/metadata_manager/src/compose.yaml

@@ -0,0 +1,28 @@
+version: '3.8'
+services:
+  edgedb:
+    image: edgedb/edgedb:3.5
+    environment:
+      EDGEDB_DOCKER_LOG_LEVEL: 'debug'
+      EDGEDB_SERVER_SECURITY: insecure_dev_mode
+      EDGEDB_SERVER_ADMIN_UI: enabled
+      EDGEDB_SERVER_USER: 'orcabus_admin'
+      EDGEDB_SERVER_PASSWORD: 'admin' # pragma: allowlist secret
+    volumes:
+      - './dbschema:/dbschema'
+    ports:
+      - '5656:5656'
+
+  app:
+    build: .
+    environment:
+      # EdgedDb related config
+      EDGEDB_HOST: edgedb
+
+      # Fastify config
+      PORT: '8080'
+      HOST: '0.0.0.0'
+    ports:
+      - '8080:8080'
+    links:
+      - edgedb

lib/workload/stateless/metadata_manager/src/dbschema/audit.esdl

@@ -1,9 +1,10 @@
+using extension graphql;
 module audit {
 
   # NOTE: TODO How to query deleted library??
 
   # ActionType definition
-  # Ref: http://hl7.org/fhir/audit-even t-action
+  # Ref: http://hl7.org/fhir/audit-event-action
   scalar type ActionType extending enum<'C', 'R', 'U', 'D', 'E'>;
 
   # ActionOutcome definition
@@ -18,36 +19,18 @@ module audit {
     # any details will appear later in the details JSON
     required actionDescription: str;
 
-    # when this audit record has been made (should be close to occurredDateTime)
-    required recordedDateTime: datetime {
-      default := datetime_current();
-      readonly := true;
-    }
-    required occurredDateTime: datetime {
-      default := datetime_current();
-    }
-    required occurredDuration: duration {
-      rewrite insert, update using (
-        select __subject__.occurredDateTime - __subject__.recordedDateTime
-      )
-    }
-
     # when this audit record is last updated (e.g. completion/failure of an event)
     # Using `rewrite` to re-update this automatically
     required updatedDateTime: datetime {
       rewrite insert, update using (datetime_of_statement())
     }
 
-
     # bespoke JSON with details of the event
     details: json;
 
-    # Whether its expected that this audit event will be updated.
-    required inProgress: bool;
   }
 
   type SystemAuditEvent extending AuditEvent {
   }
 
-
 }