Add GitLab functionalities

.env.example

@@ -1,3 +1,5 @@
 OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…'
 OTA_ENGINE_SMTP_PASSWORD='password'
 OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX
+OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX
+OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX

config/default.json

@@ -59,6 +59,7 @@
     "dataset": {
       "title": "sandbox",
       "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox",
+      "versionsRepositoryURLGitLab": "https://gitlab.com/p2b/contrib-versions",
       "publishingSchedule": "30 8 * * MON"
     }
   }

package.json

@@ -100,7 +100,8 @@
     "swagger-jsdoc": "^6.2.8",
     "swagger-ui-express": "^5.0.0",
     "winston": "^3.3.3",
-    "winston-mail": "^2.0.0"
+    "winston-mail": "^2.0.0",
+    "axios": "^1.7.2"
   },
   "devDependencies": {
     "@commitlint/cli": "^19.0.3",

scripts/dataset/assets/README.templateGitLab.js

@@ -0,0 +1,65 @@
+import config from 'config';
+
+const LOCALE = 'en-EN';
+const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' };
+
+export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) {
+  return `# Open Terms Archive — ${title({ releaseDate })}
+
+${body({ servicesCount, firstVersionDate, lastVersionDate })}`;
+}
+
+export function title({ releaseDate }) {
+  releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
+
+  const title = config.get('@opentermsarchive/engine.dataset.title');
+
+  return `${title} — ${releaseDate} dataset`;
+}
+
+export function body({ servicesCount, firstVersionDate, lastVersionDate }) {
+  firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
+  lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS);
+
+  const versionsRepositoryURLGitLab = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab');
+
+  return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}.
+
+This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURLGitLab}](${versionsRepositoryURLGitLab}).
+
+It has been generated with [Open Terms Archive](https://opentermsarchive.org).
+
+### Dataset format
+
+This dataset represents each version of a document as a separate [Markdown](https://spec.commonmark.org/0.30/) file, nested in a directory with the name of the service provider and in a directory with the name of the terms type. The filesystem layout will look like below.
+
+\`\`\`
+├ README.md
+├┬ Service provider 1 (e.g. Facebook)
+│├┬ Terms type 1 (e.g. Terms of Service)
+││├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-08-01T01-03-12Z.md)
+┆┆┆
+││└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-10-03T08-12-25Z.md)
+┆┆
+│└┬ Terms type X (e.g. Privacy Policy)
+│ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md)
+┆ ┆
+│ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md)
+┆
+└┬ Service provider Y (e.g. Google)
+ ├┬ Terms type 1 (e.g. Developer Terms)
+ │├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2019-03-12T04-18-22Z.md)
+ ┆┆
+ │└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-12-04T22-47-05Z.md)
+ └┬ Terms type Z (e.g. Privacy Policy)
+  ┆
+  ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md)
+  ┆
+  └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md)
+\`\`\`
+
+### License
+
+This dataset is made available under an [Open Database (OdBL) License](https://opendatacommons.org/licenses/odbl/1.0/) by Open Terms Archive Contributors.
+`;
+}

scripts/dataset/index.js

@@ -6,6 +6,7 @@
 import generateRelease from './export/index.js';
 import logger from './logger/index.js';
 import publishRelease from './publish/index.js';
+import publishReleaseGitLab from './publishGitLab/index.js';
 
 export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) {
   const releaseDate = new Date();
@@ -24,13 +25,25 @@
 
   logger.info('Start publishing dataset…');
 
-  const releaseUrl = await publishRelease({
-    archivePath,
-    releaseDate,
-    stats,
-  });
+  if (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined') {
+    const releaseUrl = await publishRelease({
+      archivePath,
+      releaseDate,
+      stats,
+    });
 
-  logger.info(`Dataset published to ${releaseUrl}`);
+    logger.info(`Dataset published to ${releaseUrl}`);
+  }
+
+  if (typeof process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN !== 'undefined') {
+    const releaseUrl = await publishReleaseGitLab({
+      archivePath,
+      releaseDate,
+      stats,
+    });
+
+    logger.info(`Dataset published to ${releaseUrl}`);
+  }
 
   if (!shouldRemoveLocalCopy) {
     return;

scripts/dataset/publishGitLab/index.js

@@ -0,0 +1,102 @@
+import fsApi from 'fs';
+import path from 'path';
+import url from 'url';
+
+import axios from 'axios';
+
+import config from 'config';
+import dotenv from 'dotenv';
+//import { Octokit } from 'octokit';
+
+import FormData from 'form-data';
+
+import * as readme from '../assets/README.templateGitLab.js';
+
+dotenv.config();
+
+const gitlabAPIUrl = "https://gitlab.com/api/v4";
+const gitlabUrl = "https://gitlab.com";
+
+export default async function publishReleaseGitLab({
+  archivePath,
+  releaseDate,
+  stats,
+}) {
+  let projectId = null;
+
+  // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN });
+
+  const [owner, repo] = url
+    .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'))
+    .pathname.split('/')
+    .filter((component) => component);
+  const commonParams = { owner, repo };
+
+  try {
+    const repositoryPath = `${commonParams.owner}/${commonParams.repo}`;
+    const response = await axios.get(
+      `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`,
+      {
+        headers: {
+          Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`,
+        },
+      },
+    );
+    projectId = response.data.id;
+  } catch (error) {
+    //logger.error(`🤖  Error while obtaining projectId: ${error}`);
+    projectId = null;
+  }
+
+  const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag
+
+  try {
+    // First, create the release
+    const releaseResponse = await axios.post(
+      `${gitlabAPIUrl}/projects/${projectId}/releases`,
+      {
+        ref: 'main',
+        tag_name: tagName,
+        name: readme.title({ releaseDate }),
+        description: readme.body(stats),
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`,
+          'Content-Type': 'application/json',
+        },
+      },
+    );
+
+    const releaseId = releaseResponse.data.commit.id;
+
+    // Then, upload the ZIP file as an asset to the release
+    const formData = new FormData();
+    formData.append('name', archivePath);
+    formData.append(
+      'url',
+      `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`,
+    );
+    formData.append('file', fsApi.createReadStream(archivePath), {
+      filename: path.basename(archivePath),
+    });
+
+    const uploadResponse = await axios.post(
+      `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`,
+      formData,
+      {
+        headers: {
+          ...formData.getHeaders(),
+          Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`,
+        },
+      },
+    );
+
+    const releaseUrl = uploadResponse.data.direct_asset_url;
+
+    return releaseUrl;
+  } catch (error) {
+    console.error('Failed to create release or upload ZIP file:', error);
+    throw error;
+  }
+}

src/index.js

@@ -8,6 +8,7 @@ import Archivist from './archivist/index.js';
 import logger from './logger/index.js';
 import Notifier from './notifier/index.js';
 import Reporter from './reporter/index.js';
+import ReporterGitlab from './reporterGitlab/index.js';
 
 const require = createRequire(import.meta.url);
 
@@ -65,13 +66,31 @@ export default async function track({ services, types, extractOnly, schedule })
       } catch (error) {
         logger.error('Cannot instantiate the Reporter module; it will be ignored:', error);
       }
+      archivist.attach(reporter);
     } else {
       logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
     }
   } else {
     logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored');
   }
 
+  if (process.env.OTA_ENGINE_GITLAB_TOKEN) {
+    if (config.has('@opentermsarchive/engine.reporter.gitlabIssues.repositories.declarations')) {
+      try {
+        const reporter = new ReporterGitlab(config.get('@opentermsarchive/engine.reporter'));
+
+        await reporter.initialize();
+        archivist.attach(reporter);
+      } catch (error) {
+        logger.error('Cannot instantiate the ReporterGitlab module; it will be ignored:', error);
+      }
+    } else {
+      logger.warn('Configuration key "reporter.gitlabIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created');
+    }
+  } else {
+    logger.warn('Environment variable "OTA_ENGINE_GITLAB_TOKEN" was not found; the ReporterGitlab module will be ignored');
+  }
+
   if (!schedule) {
     await archivist.track({ services, types });