Merge branch 'LN-1620-metadata-error-states-support' into 'dev'

fix: improve error handling and UI elements for governance actions

See merge request voltaire/govtool-outcomes-pillar!52

backend/package.json

@@ -33,7 +33,9 @@
     "reflect-metadata": "^0.2.2",
     "rimraf": "^6.0.1",
     "rxjs": "^7.8.1",
-    "typeorm": "^0.3.20"
+    "typeorm": "^0.3.20",
+    "blakejs": "^1.2.1",
+    "jsonld": "^8.3.2"
   },
   "devDependencies": {
     "@nestjs/cli": "^11.0.4",

backend/src/enums/LoggerMessage.ts

@@ -0,0 +1,7 @@
+export enum LoggerMessage {
+  METADATA_VALIDATION_ERROR = "Metadata validation error",
+  METADATA_DATA = "Metadata data",
+  CANNOT_GET_METADATA_URL = "Cannot get metadata from URL",
+  PARSED_METADATA_BODY = "Parsed metadata body",
+  CANNOT_PARSE_METADATA_BODY = "Cannot parse metadata body",
+}

backend/src/enums/ValidationErrors.ts

@@ -0,0 +1,6 @@
+export enum MetadataValidationStatus {
+  URL_NOT_FOUND = "URL_NOT_FOUND",
+  INVALID_JSONLD = "INVALID_JSONLD",
+  INVALID_HASH = "INVALID_HASH",
+  INCORRECT_FORMAT = "INCORRECT_FORMAT",
+}

backend/src/governance-actions/governance-actions.controller.ts

@@ -1,5 +1,6 @@
 import { Controller, Get, Param, Query } from "@nestjs/common";
 import { GovernanceActionsService } from "./governance-actions.service";
+import { ValidateMetadataResult } from "src/types/validateMetadata";
 
 @Controller("governance-actions")
 export class GovernanceActionsController {
@@ -26,8 +27,11 @@ export class GovernanceActionsController {
   }
 
   @Get("/metadata")
-  findMetadata(@Query("url") url: string) {
-    return this.governanceActionsService.findMetadata(url);
+  findMetadata(
+    @Query("url") url: string,
+    @Query("hash") hash: string
+  ): Promise<ValidateMetadataResult> {
+    return this.governanceActionsService.getMetadata(url, hash);
   }
 
   @Get(":id")

backend/src/governance-actions/governance-actions.service.ts

@@ -1,12 +1,20 @@
-import { HttpException, Injectable } from "@nestjs/common";
+import { Injectable, Logger } from "@nestjs/common";
 import { InjectDataSource } from "@nestjs/typeorm";
 import { getGovernanceAction } from "src/queries/governanceAction";
 import { getGovernanceActions } from "src/queries/governanceActions";
 import { DataSource } from "typeorm";
 import { HttpService } from "@nestjs/axios";
-import { lastValueFrom } from "rxjs";
-import { map, catchError } from "rxjs/operators";
+import { firstValueFrom } from "rxjs";
+import { catchError, finalize } from "rxjs/operators";
 import { ConfigService } from "@nestjs/config";
+import { MetadataValidationStatus } from "src/enums/ValidationErrors";
+import { getStandard } from "src/utils/getStandard";
+import * as blake from "blakejs";
+import * as jsonld from "jsonld";
+import { validateMetadataStandard } from "src/utils/validateMetadataStandard";
+import { parseMetadata } from "src/utils/parseMetadata";
+import { LoggerMessage } from "src/enums/LoggerMessage";
+import { ValidateMetadataResult } from "src/types/validateMetadata";
 
 @Injectable()
 export class GovernanceActionsService {
@@ -37,7 +45,7 @@ export class GovernanceActionsService {
     ]);
   }
 
-  private convertIpfsToHttpUrl(url: string): string {
+  private processUrl(url: string): string {
     if (url.startsWith("ipfs://")) {
       const ipfsHash = url.replace("ipfs://", "");
       const gateway = this.configService.get<string>("IPFS_GATEWAY");
@@ -46,39 +54,102 @@ export class GovernanceActionsService {
     return url;
   }
 
-  async findMetadata(url: string) {
-    try {
-      const httpUrl = this.convertIpfsToHttpUrl(url);
+  async getMetadata(
+    url: string,
+    hash: string
+  ): Promise<ValidateMetadataResult> {
+    let metadataStatus: MetadataValidationStatus;
+    let metadata: Record<string, unknown>;
+    let standard;
 
-      const response$ = this.httpService.get(httpUrl).pipe(
-        map((response) => {
-          if (!response.data) {
-            throw new Error("No data found in the response");
-          }
-          const metadata = response.data;
-          return {
-            authors: metadata.authors || [],
-            hashAlgorithm: metadata.hashAlgorithm || "",
-            body: {
-              abstract: metadata.body?.abstract || "",
-              motivation: metadata.body?.motivation || "",
-              rationale: metadata.body?.rationale || "",
-              references: metadata.body?.references || [],
-              title: metadata.body?.title || "",
-              comment: metadata.body?.comment || "",
-              externalUpdates: metadata.body?.externalUpdates || [],
+    const httpUrl = this.processUrl(url);
+
+    try {
+      const response = await firstValueFrom(
+        this.httpService
+          .get(httpUrl, {
+            headers: {
+              "User-Agent": "GovTool/Metadata-Validation-Tool",
+              "Content-Type": "application/json",
             },
-          };
-        }),
-        catchError((error) => {
-          throw new Error(`Error: ${error.message}`);
-        })
+          })
+          .pipe(
+            finalize(() => Logger.log(`Fetching ${httpUrl} completed`)),
+            catchError((error) => {
+              Logger.error(error, JSON.stringify(error));
+              throw MetadataValidationStatus.URL_NOT_FOUND;
+            })
+          )
       );
 
-      return await lastValueFrom(response$);
+      const rawData = (response as any).data;
+
+      let parsedData;
+
+      if (typeof rawData !== "object") {
+        try {
+          parsedData = JSON.parse(rawData);
+        } catch (error) {
+          throw MetadataValidationStatus.INCORRECT_FORMAT;
+        }
+      } else {
+        parsedData = rawData;
+      }
+
+      if (!parsedData?.body) {
+        throw MetadataValidationStatus.INCORRECT_FORMAT;
+      }
+
+      if (!standard) {
+        standard = getStandard(parsedData);
+      }
+
+      if (standard) {
+        await validateMetadataStandard(parsedData.body, standard);
+        metadata = parseMetadata(parsedData.body);
+      }
+      const rawDataForHashing =
+        typeof rawData === "object" ? JSON.stringify(rawData) : rawData;
+      const hashedMetadata = blake.blake2bHex(rawDataForHashing, undefined, 32);
+
+      if (hashedMetadata !== hash) {
+        // Optionally validate on a parsed metadata
+        const hashedParsedMetadata = blake.blake2bHex(
+          JSON.stringify(parsedData, null, 2),
+          undefined,
+          32
+        );
+        if (hashedParsedMetadata !== hash) {
+          // Optional support for the canonized data hash
+          // Validate canonized data hash
+          const dataForCanonicalization =
+            typeof rawData === "object" ? rawData : JSON.parse(rawData);
+          const canonizedMetadata = await jsonld.canonize(
+            dataForCanonicalization,
+            {
+              safe: false,
+            }
+          );
+
+          const hashedCanonizedMetadata = blake.blake2bHex(
+            canonizedMetadata,
+            undefined,
+            32
+          );
+
+          if (hashedCanonizedMetadata !== hash) {
+            throw MetadataValidationStatus.INVALID_HASH;
+          }
+        }
+      }
     } catch (error) {
-      throw error;
+      Logger.error(LoggerMessage.METADATA_VALIDATION_ERROR, error);
+      if (Object.values(MetadataValidationStatus).includes(error)) {
+        metadataStatus = error;
+      }
     }
+
+    return { metadataStatus, metadataValid: !Boolean(metadataStatus), data: metadata };
   }
 
   findOne(id: string) {

backend/src/types/validateMetadata.ts

@@ -0,0 +1,12 @@
+import { MetadataValidationStatus } from "src/enums/ValidationErrors";
+
+export enum MetadataStandard {
+  CIP108 = "CIP108",
+  CIP119 = "CIP119",
+}
+
+export type ValidateMetadataResult = {
+  metadataStatus?: MetadataValidationStatus;
+  metadataValid: boolean;
+  data?: any;
+};

backend/src/utils/getFieldValue.ts

@@ -0,0 +1,22 @@
+/**
+ * Retrieves the value of a specified field from a given object.
+ *
+ * @param body - The object from which to retrieve the field value.
+ * @param field - The name of the field to retrieve the value from.
+ * @returns The value of the specified field, or undefined if the field does not exist.
+ */
+export const getFieldValue = (
+  body: Record<string, unknown>,
+  field: string
+): unknown => {
+  const fieldValue = body[field];
+  if (fieldValue.hasOwnProperty("@value")) {
+    return fieldValue["@value"];
+  }
+
+  if (fieldValue) {
+    return fieldValue;
+  }
+
+  return undefined;
+};

backend/src/utils/getStandard.ts

@@ -0,0 +1,17 @@
+import { MetadataStandard } from "src/types/validateMetadata";
+/**
+ * Retrieves the metadata standard from the given data.
+ * @param data - The data containing the metadata.
+ * @returns The metadata standard if found, otherwise undefined.
+ */
+export const getStandard = (
+  data: Record<string, unknown>
+): MetadataStandard | undefined => {
+  if (JSON.stringify(data).includes(MetadataStandard.CIP119)) {
+    return MetadataStandard.CIP119;
+  }
+  if (JSON.stringify(data).includes(MetadataStandard.CIP108)) {
+    return MetadataStandard.CIP108;
+  }
+  return undefined;
+};

backend/src/utils/parseMetadata.ts

@@ -0,0 +1,23 @@
+import { getFieldValue } from './getFieldValue';
+
+/**
+ * Parses the metadata from the given body object.
+ *
+ * @param body - The body object containing the metadata.
+ * @returns An object with the parsed metadata.
+ */
+export const parseMetadata = (body: Record<string, unknown>) => {
+  const metadata = {};
+
+  Object.keys(body).forEach((key) => {
+    if (key === 'references') {
+      const parsedReferences = (body[key] as Record<string, unknown>[]).map(
+        (reference) => parseMetadata(reference),
+      );
+      metadata[key] = parsedReferences;
+    } else {
+      metadata[key] = getFieldValue(body, key);
+    }
+  });
+  return metadata;
+};

backend/src/utils/validateCIP108body.ts

@@ -0,0 +1,24 @@
+import { MetadataValidationStatus } from "src/enums/ValidationErrors";
+import { getFieldValue } from "./getFieldValue";
+
+/**
+ * Validates the body of a CIP108 standard.
+ *
+ * @param body - The body of the metadata.
+ * @returns True if the body is valid, otherwise throws an error.
+ * @throws {MetadataValidationStatus} - Throws an error if the body is not in the correct format.
+ */
+export const validateCIP108body = (body: Record<string, unknown>) => {
+  const title = getFieldValue(body, "title");
+  const abstract = getFieldValue(body, "abstract");
+  const motivation = getFieldValue(body, "motivation");
+  const rationale = getFieldValue(body, "rationale");
+  if (!title || !abstract || !motivation || !rationale) {
+    throw MetadataValidationStatus.INCORRECT_FORMAT;
+  }
+  if (String(title).length > 80 || String(abstract).length > 2500) {
+    throw MetadataValidationStatus.INCORRECT_FORMAT;
+  }
+
+  return true;
+};

backend/src/utils/validateMetadataStandard.ts

@@ -0,0 +1,43 @@
+import { Logger } from "@nestjs/common";
+import { LoggerMessage } from "src/enums/LoggerMessage";
+import { MetadataValidationStatus } from "src/enums/ValidationErrors";
+import { MetadataStandard } from "src/types/validateMetadata";
+import { getFieldValue } from "./getFieldValue";
+import { validateCIP108body } from "./validateCIP108body";
+
+/**
+ * Validates the metadata against a specific standard.
+ * @param body - The metadata body to be validated.
+ * @param standard - The metadata standard to validate against.
+ * @throws {MetadataValidationStatus.INCORRECT_FORMAT} - If the metadata does not conform to the specified standard.
+ */
+export const validateMetadataStandard = async (
+  body: Record<string, unknown>,
+  standard: MetadataStandard
+) => {
+  try {
+    switch (standard) {
+      // givenName is the only compulsory field in CIP119
+      case MetadataStandard.CIP119:
+        const givenName = getFieldValue(body, "givenName");
+        if (!givenName) {
+          Logger.error(
+            LoggerMessage.METADATA_VALIDATION_ERROR,
+            MetadataValidationStatus.INCORRECT_FORMAT
+          );
+          throw MetadataValidationStatus.INCORRECT_FORMAT;
+        }
+        return true;
+      case MetadataStandard.CIP108:
+        return validateCIP108body(body);
+      default:
+        return true;
+    }
+  } catch (error) {
+    Logger.error(
+      LoggerMessage.METADATA_VALIDATION_ERROR,
+      MetadataValidationStatus.INCORRECT_FORMAT
+    );
+    throw MetadataValidationStatus.INCORRECT_FORMAT;
+  }
+};