Merge pull request #535 from EyeSeeTea/development

Release 1.7.0

.github/workflows/main.yml

@@ -5,6 +5,7 @@ on:
 jobs:
     unit-tests:
         name: Unit tests
+        runs-on: self-hosted
         steps:
             - name: Checkout repository
               uses: actions/checkout@v2

deploy/auth.template.sh

@@ -0,0 +1 @@
+export auth='admin:PASSWORD'

deploy/clone-pro-to-training.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e -u -o pipefail
+
+# Actions:
+#
+#   - Save last projects for 60 days in training instance.
+#   - Clone instance in proj-dhis-prod to localhost.
+#   - Setup of the local docker + recover training projects
+
+# Requirements: Open vendorlink before running the script:
+#
+#   $ vendorlink sp-proj-dhis-test-00.clt1.theark.cloud sp-proj-dhis-prod-00.clt1.theark.cloud
+
+cd "$(dirname "$0")"
+source "./lib.sh"
+
+#run sp-proj-dhis-test-00.clt1.theark.cloud sudo usermod -a -G docker asanchez
+
+run boone-ip-pro bash push-pro-docker.sh
+#run boone-ip-test bash start-training-from-pro.sh

deploy/lib.sh

@@ -0,0 +1,71 @@
+#!/bin/bash
+script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-$0}")")"
+source "$script_dir/auth.sh"
+
+export image_pro="docker.eyeseetea.com/eyeseetea/dhis2-data:2.36.11.1-sp-ip-pro"
+export image_test="docker.eyeseetea.com/samaritans/dhis2-data:2.36.11.1-sp-ip-test"
+export image_dev="docker.eyeseetea.com/samaritans/dhis2-data:2.36.11.1-sp-ip-dev"
+export image_training="docker.eyeseetea.com/samaritans/dhis2-data:2.36.11.1-sp-ip-training"
+
+debug() {
+    echo "$@" >&2
+}
+
+get_url() {
+    local port=$1
+    echo "http://$auth@localhost:$port"
+}
+
+run() {
+    local host=$1
+    local command=$2
+    shift 2
+
+    debug "Copy deploy folder"
+    rsync -a . "$host":deploy/
+    debug "Run: $command $*"
+    ssh "$host" "cd deploy &&" "$command" "$@"
+}
+
+wait_for_dhis2_server() {
+    local url=$url
+    local port
+    port=$(echo "$url" | grep -o ':[[:digit:]]*$' | cut -d: -f2)
+    echo "Wait for server: port=$port"
+
+    while ! curl -sS -f "http://localhost:$port"; do
+        sleep 10
+    done
+}
+
+timestamp() {
+    date "+%Y-%m-%d_%H-%M"
+}
+
+get_user_ids() {
+    local url=$1 usernames=$2
+    local get_users_url="$url/api/users.json?filter=userCredentials.username:in:[${usernames}]"
+
+    curl -g -sS "$get_users_url" | jq -r '.users[].id'
+}
+
+get_user_role_ids() {
+    local url=$1 names=$2
+    local get_user_roles_url="$url/api/userRoles.json"
+
+    curl -G -g -sS "$get_user_roles_url" \
+        --data-urlencode "filter=name:in:[${names}]" | jq -r '.userRoles[].id'
+}
+
+change_server_name() {
+    local url=$1 title=$2
+    debug "Change server name: $title"
+    curl -sS -X POST "$url/api/systemSettings/applicationTitle" \
+        -d "$title" -H "Content-Type: application/json"
+}
+
+join_by() {
+    local IFS="$1"
+    shift
+    echo "$*"
+}

deploy/push-pro-docker.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+set -e -u -o pipefail
+
+name="docker.eyeseetea.com/eyeseetea/dhis2-data:2.36.11.1-sp-ip-pro"
+sql_filename="$(basename $name).sql.gz"
+
+echo "Dump DB: $name -> $sql_filename"
+sudo -u postgres pg_dump dhis2 | gzip >"$sql_filename"
+
+echo "Create d2-docker image: $name"
+sudo d2-docker create data --sql="$sql_filename" "$name" \
+    --apps-dir=/home/dhis/config/files/apps/ --documents-dir=/home/dhis/config/files/document/
+sudo docker push "$name"

deploy/start-training-from-pro.sh

@@ -0,0 +1,82 @@
+#!/bin/bash
+set -e -u -o pipefail
+
+cd "$(dirname "$0")"
+script_dir=$(pwd)
+source "./lib.sh"
+source "./tasks.sh"
+
+load_training_projects() {
+    local url=$1
+    cd "$script_dir/project-monitoring-app" || return 1
+    yarn ts-node src/scripts/projects.ts --url="$url" import training-projects.json
+}
+
+training_post_clone() {
+    local url=$1
+    set_email_password "$url"
+    enable_users "$url" "traindatareviewer,traindataviewer,traindataentry"
+    change_server_name "$url" "SP Platform - Training"
+    add_users_to_maintainer_roles "$url"
+    set_logos "$url" "$script_dir/training"
+}
+
+get_app_version() {
+    local url=$1
+    curl -sS -u "$auth" "$url/api/apps" |
+        jq '.[] | select(.key == "Data-Management-App").version' -r
+}
+
+get_project_monitoring_app_source() {
+    local url=$1
+    local repo_url="https://github.com/eyeseetea/project-monitoring-app"
+
+    app_version=$(get_app_version "$url")
+    cd "$script_dir" || return 1
+    git clone "$repo_url" "project-monitoring-app" || true
+    cd "project-monitoring-app" || return 1
+
+    git fetch
+    git checkout v"$app_version" -f
+    yarn install
+    yarn add [email protected]
+    yarn localize
+}
+
+save_last_training_projects() {
+    local url=$1
+    date=$(date --date="60 day ago" "+%Y-%m-%d")
+    cd "$script_dir/project-monitoring-app" || return 1
+    yarn ts-node src/scripts/projects.ts --url="$url" --from="$date" export training-projects.json
+}
+
+delete_projects() {
+    d2-docker run-sql -i $image_training "$script_dir/clone-scripts/create-guest-user.sql"
+    d2-docker run-sql -i $image_training "$script_dir/clone-scripts/sql-01.empty_data_tables_228.sql"
+    d2-docker run-sql -i $image_training "$script_dir/clone-scripts/sql-02.delete-projects.sql"
+}
+
+start_from_pro() {
+    d2-docker pull "$image_pro"
+    d2-docker commit "$image_training"
+    d2-docker copy "$image_training" "backup/sp-ip-training-$(timestamp)"
+    d2-docker stop "$image_training"
+    d2-docker copy "$image_pro" "$image_training"
+
+    sudo /usr/local/bin/start-dhis2-training
+
+    wait_for_dhis2_server "$url"
+}
+
+main() {
+    url=$(get_url 81)
+
+    get_project_monitoring_app_source "$url"
+    save_last_training_projects "$url"
+    start_from_pro
+    delete_projects
+    load_training_projects "$url"
+    training_post_clone "$url"
+}
+
+main "$@"

deploy/tasks.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+script_dir="$(dirname "$(readlink -f "${BASH_SOURCE[0]:-$0}")")"
+# shellcheck source=./lib.sh
+source "$script_dir/lib.sh"
+
+add_users_to_maintainer_roles() {
+    local url=$1
+    debug "Add users to maintainer roles"
+    local usernames="hlackey,topelt,bpipher,abong,rsaballe,ccampomanes,virikefe,ashettleroe"
+    local roles="Metadata Maintainer,User Maintainer"
+    local role_refs
+
+    role_refs=$(
+        get_user_role_ids "$url" "$roles" | while read -r role_id; do
+            echo "{\"id\":\"$role_id\"}"
+        done
+    )
+    local new_roles user_updated
+    # shellcheck disable=SC2086
+    new_roles="[$(join_by "," $role_refs)]"
+    debug "New roles: $new_roles"
+
+    get_user_ids "$url" "$usernames" | while read -r user_id; do
+        debug -n "Add roles ($new_roles) to user: ${user_id}... "
+
+        user_updated=$(curl -sS -X GET \
+            "$url/api/users/$user_id.json" | jq ".userCredentials.userRoles += ${new_roles}")
+        curl -H "Content-Type: application/json" -sS -X PUT \
+            "$url/api/users/$user_id" -d "$user_updated" | jq -r '.status'
+    done
+}
+
+enable_users() {
+    local url=$1 usernames=$2
+    debug "Enable training users: $usernames"
+
+    get_user_ids "$url" "$usernames" | while read -r user_id; do
+        debug "Enable user: ${user_id}"
+        curl -H "Content-Type: application/json" -sS -X PATCH \
+            "$url/api/users/$user_id" -d '{"userCredentials":{"disabled":false}}'
+    done
+}
+
+set_email_password() {
+    local url=$url
+    echo "Set email password"
+    curl -sS -H 'Content-Type: text/plain' -u "$auth" \
+        "$url/api/systemSettings/keyEmailPassword" \
+        -d 'RLIi96f3TJSBsV2h1IO6Vy52ToWzH0' | jq -r '.status'
+}
+
+set_logos() {
+    local url=$1 folder=$2
+
+    echo "Set logs: url=$url, folder=$folder"
+
+    curl -sS -F "file=@$folder/logo_front.png;type=image/png" \
+        -X POST -u "$auth" \
+        -H "Content-Type: multipart/form-data" \
+        "$url/api/staticContent/logo_front"
+
+    curl -sS -F "file=@$folder/logo_banner.png;type=image/png" \
+        -X POST -u "$auth" \
+        -H "Content-Type: multipart/form-data" \
+        "$url/api/staticContent/logo_banner"
+}

package.json

@@ -1,7 +1,7 @@
 {
     "name": "data-management-app",
     "description": "DHIS2 Data Management App",
-    "version": "1.6.2",
+    "version": "1.7.0",
     "license": "GPL-3.0",
     "author": "EyeSeeTea team",
     "homepage": ".",
@@ -51,7 +51,8 @@
         "striptags": "3.2.0",
         "styled-components": "^5.2.1",
         "styled-jsx": "3.4.1",
-        "word-wrap": "1.2.5"
+        "word-wrap": "1.2.5",
+        "xlsx": "^0.18.5"
     },
     "scripts": {
         "prestart": "yarn localize && d2-manifest package.json manifest.webapp",
@@ -131,7 +132,7 @@
         "jest": "26.6.3",
         "parse-typed-args": "^0.2.0",
         "prettier": "2.5.1",
-        "ts-node": "9.1.1",
+        "ts-node": "10.9.2",
         "typescript": "4.9.3",
         "wait-on": "5.2.1"
     },

src/CompositionRoot.ts

@@ -0,0 +1,36 @@
+import { DataElementD2Repository } from "./data/repositories/DataElementD2Repository";
+import { DataValueD2Repository } from "./data/repositories/DataValueD2Repository";
+import { DataValueExportJsonRepository } from "./data/repositories/DataValueExportJsonRepository";
+import { ExportDataElementJsonRepository } from "./data/repositories/ExportDataElementJsonRepository";
+import { ImportDataElementSpreadSheetRepository } from "./data/repositories/ImportDataElementSpreadSheetRepository";
+import { OrgUnitD2Repository } from "./data/repositories/OrgUnitD2Repository";
+import { ImportDataElementsUseCase } from "./domain/usecases/ImportDataElementsUseCase";
+import { Config } from "./models/Config";
+import { D2Api } from "./types/d2-api";
+
+export function getCompositionRoot(api: D2Api, config: Config) {
+    const dataValueRepository = new DataValueD2Repository(api);
+    const dataElementRepository = new DataElementD2Repository(api, config);
+    const importDataElementSpreadSheetRepository = new ImportDataElementSpreadSheetRepository(
+        api,
+        config
+    );
+    const exportDataElementJsonRepository = new ExportDataElementJsonRepository(api, config);
+    const dataValueExportRepository = new DataValueExportJsonRepository();
+    const orgUnitRepository = new OrgUnitD2Repository(api);
+
+    return {
+        dataElements: {
+            import: new ImportDataElementsUseCase(
+                importDataElementSpreadSheetRepository,
+                dataElementRepository,
+                exportDataElementJsonRepository,
+                dataValueRepository,
+                dataValueExportRepository,
+                orgUnitRepository
+            ),
+        },
+    };
+}
+
+export type CompositionRoot = ReturnType<typeof getCompositionRoot>;

src/data/AttributeValue.ts

@@ -0,0 +1,3 @@
+import { Ref } from "../domain/entities/Ref";
+
+export type AttributeValue = { attribute: Ref; value: string };

src/data/DataElementGroup.ts

@@ -0,0 +1,7 @@
+import { Code, NamedRef, Ref } from "../domain/entities/Ref";
+
+export interface DataElementGroup extends NamedRef {
+    code: Code;
+    shortName: string;
+    dataElements: Ref[];
+}

src/data/Ref.ts

@@ -0,0 +1 @@
+export type Identifiable = string;

src/data/SaveOptions.ts

@@ -0,0 +1,5 @@
+export type SaveOptions = { post: boolean };
+
+export function getImportModeFromOptions(persist: boolean): "COMMIT" | "VALIDATE" {
+    return persist ? "COMMIT" : "VALIDATE";
+}

src/data/common.ts

@@ -0,0 +1,22 @@
+import _ from "lodash";
+import { DataElement } from "../domain/entities/DataElement";
+
+export function getExistingAndNewDataElements(dataElements: DataElement[]) {
+    const existingDataElements = _(dataElements)
+        .filter(dataElement => dataElement.existing)
+        .value();
+
+    const existingDataElementsKeys = _(existingDataElements)
+        .keyBy(dataElement => dataElement.id)
+        .value();
+
+    const newDataElements = _(dataElements)
+        .filter(dataElement => !dataElement.existing)
+        .value();
+
+    const newDataElementsKeys = _(newDataElements)
+        .keyBy(dataElement => dataElement.id)
+        .value();
+
+    return { existingDataElements, existingDataElementsKeys, newDataElements, newDataElementsKeys };
+}