From fd97234604e2a37091652aaaafcc970022959ded Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Wed, 31 Jul 2024 15:28:36 +0200 Subject: [PATCH 01/11] Add Gtilab functionalities add code to handle Gitlab repository, add axios to project packages, add env variables for Gitlab --- .env.example | 2 + config/default.json | 1 + package-lock.json | 33 +- package.json | 3 +- .../dataset/assets/README.templateGitLab.js | 65 ++++ scripts/dataset/index.js | 25 +- scripts/dataset/publishGitLab/index.js | 102 ++++++ src/index.js | 19 + src/reporterGitlab/gitlab.js | 324 ++++++++++++++++++ src/reporterGitlab/index.js | 147 ++++++++ src/reporterGitlab/labels.json | 77 +++++ src/reporterGitlab/labels.test.js | 30 ++ 12 files changed, 814 insertions(+), 14 deletions(-) create mode 100644 scripts/dataset/assets/README.templateGitLab.js create mode 100644 scripts/dataset/publishGitLab/index.js create mode 100644 src/reporterGitlab/gitlab.js create mode 100644 src/reporterGitlab/index.js create mode 100644 src/reporterGitlab/labels.json create mode 100644 src/reporterGitlab/labels.test.js diff --git a/.env.example b/.env.example index 892ff5b37..08813d698 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,5 @@ OTA_ENGINE_SENDINBLUE_API_KEY='xkeysib-3f51c…' OTA_ENGINE_SMTP_PASSWORD='password' OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX +OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX +OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX diff --git a/config/default.json b/config/default.json index 0d3d82eac..9009d52a6 100644 --- a/config/default.json +++ b/config/default.json @@ -59,6 +59,7 @@ "dataset": { "title": "sandbox", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox", + "versionsRepositoryURLGitLab": "https://gitlab.com/p2b/contrib-versions", "publishingSchedule": "30 8 * * MON" } } diff --git a/package-lock.json b/package-lock.json index 7fb266632..dfed6a315 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,6 +18,7 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", + "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", @@ -171,6 +172,14 @@ "npm": ">=6" } }, + "node_modules/@accordproject/concerto-util/node_modules/axios": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", + "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@accordproject/markdown-cicero": { "version": "0.15.2", "resolved": "https://registry.npmjs.org/@accordproject/markdown-cicero/-/markdown-cicero-0.15.2.tgz", @@ -324,6 +333,14 @@ "npm": ">=6" } }, + "node_modules/@accordproject/markdown-pdf/node_modules/axios": { + "version": "0.23.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", + "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "dependencies": { + "follow-redirects": "^1.14.4" + } + }, "node_modules/@accordproject/markdown-pdf/node_modules/pdfjs-dist": { "version": "2.13.216", "resolved": "https://registry.npmjs.org/pdfjs-dist/-/pdfjs-dist-2.13.216.tgz", @@ -3964,11 +3981,13 @@ } }, "node_modules/axios": { - "version": "0.23.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-0.23.0.tgz", - "integrity": "sha512-NmvAE4i0YAv5cKq8zlDoPd1VLKAqX5oLuZKs8xkJa4qi6RGn0uhCYFjWtHHC9EM/MwOwYWOs53W+V0aqEXq1sg==", + "version": "1.7.2", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", + "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", "dependencies": { - "follow-redirects": "^1.14.4" + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" } }, "node_modules/b4a": { @@ -6478,9 +6497,9 @@ "integrity": "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw==" }, "node_modules/follow-redirects": { - "version": "1.15.3", - "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.3.tgz", - "integrity": "sha512-1VzOtuEM8pC9SFU1E+8KfTjZyMztRsgEfwQl44z8A25uy13jSzTj6dyK2Df52iV0vgHCfBwLhDWevLn95w5v6Q==", + "version": "1.15.6", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", + "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", "funding": [ { "type": "individual", diff --git a/package.json b/package.json index 8fccde58e..8da099a32 100644 --- a/package.json +++ b/package.json @@ -100,7 +100,8 @@ "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "winston": "^3.3.3", - "winston-mail": "^2.0.0" + "winston-mail": "^2.0.0", + "axios": "^1.7.2" }, "devDependencies": { "@commitlint/cli": "^19.0.3", diff --git a/scripts/dataset/assets/README.templateGitLab.js b/scripts/dataset/assets/README.templateGitLab.js new file mode 100644 index 000000000..c9f0bb8d2 --- /dev/null +++ b/scripts/dataset/assets/README.templateGitLab.js @@ -0,0 +1,65 @@ +import config from 'config'; + +const LOCALE = 'en-EN'; +const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; + +export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { + return `# Open Terms Archive — ${title({ releaseDate })} + +${body({ servicesCount, firstVersionDate, lastVersionDate })}`; +} + +export function title({ releaseDate }) { + releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + + const title = config.get('@opentermsarchive/engine.dataset.title'); + + return `${title} — ${releaseDate} dataset`; +} + +export function body({ servicesCount, firstVersionDate, lastVersionDate }) { + firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); + + const versionsRepositoryURLGitLab = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. + +This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURLGitLab}](${versionsRepositoryURLGitLab}). + +It has been generated with [Open Terms Archive](https://opentermsarchive.org). + +### Dataset format + +This dataset represents each version of a document as a separate [Markdown](https://spec.commonmark.org/0.30/) file, nested in a directory with the name of the service provider and in a directory with the name of the terms type. The filesystem layout will look like below. + +\`\`\` +├ README.md +├┬ Service provider 1 (e.g. Facebook) +│├┬ Terms type 1 (e.g. Terms of Service) +││├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-08-01T01-03-12Z.md) +┆┆┆ +││└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-10-03T08-12-25Z.md) +┆┆ +│└┬ Terms type X (e.g. Privacy Policy) +│ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) +┆ ┆ +│ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) +┆ +└┬ Service provider Y (e.g. Google) + ├┬ Terms type 1 (e.g. Developer Terms) + │├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2019-03-12T04-18-22Z.md) + ┆┆ + │└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-12-04T22-47-05Z.md) + └┬ Terms type Z (e.g. Privacy Policy) + ┆ + ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) + ┆ + └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) +\`\`\` + +### License + +This dataset is made available under an [Open Database (OdBL) License](https://opendatacommons.org/licenses/odbl/1.0/) by Open Terms Archive Contributors. +`; +} diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 4c739686d..d6b74a3bf 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -6,6 +6,7 @@ import config from 'config'; import generateRelease from './export/index.js'; import logger from './logger/index.js'; import publishRelease from './publish/index.js'; +import publishReleaseGitLab from './publishGitLab/index.js'; export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName }) { const releaseDate = new Date(); @@ -24,13 +25,25 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start publishing dataset…'); - const releaseUrl = await publishRelease({ - archivePath, - releaseDate, - stats, - }); + if (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined') { + const releaseUrl = await publishRelease({ + archivePath, + releaseDate, + stats, + }); - logger.info(`Dataset published to ${releaseUrl}`); + logger.info(`Dataset published to ${releaseUrl}`); + } + + if (typeof process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN !== 'undefined') { + const releaseUrl = await publishReleaseGitLab({ + archivePath, + releaseDate, + stats, + }); + + logger.info(`Dataset published to ${releaseUrl}`); + } if (!shouldRemoveLocalCopy) { return; diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js new file mode 100644 index 000000000..2a1f9bbc9 --- /dev/null +++ b/scripts/dataset/publishGitLab/index.js @@ -0,0 +1,102 @@ +import fsApi from 'fs'; +import path from 'path'; +import url from 'url'; + +import axios from 'axios'; + +import config from 'config'; +import dotenv from 'dotenv'; +//import { Octokit } from 'octokit'; + +import FormData from 'form-data'; + +import * as readme from '../assets/README.templateGitLab.js'; + +dotenv.config(); + +const gitlabAPIUrl = "https://gitlab.com/api/v4"; +const gitlabUrl = "https://gitlab.com"; + +export default async function publishReleaseGitLab({ + archivePath, + releaseDate, + stats, +}) { + let projectId = null; + + // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); + + const [owner, repo] = url + .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) + .pathname.split('/') + .filter((component) => component); + const commonParams = { owner, repo }; + + try { + const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; + const response = await axios.get( + `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, + }, + }, + ); + projectId = response.data.id; + } catch (error) { + //logger.error(`🤖 Error while obtaining projectId: ${error}`); + projectId = null; + } + + const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag + + try { + // First, create the release + const releaseResponse = await axios.post( + `${gitlabAPIUrl}/projects/${projectId}/releases`, + { + ref: 'main', + tag_name: tagName, + name: readme.title({ releaseDate }), + description: readme.body(stats), + }, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + const releaseId = releaseResponse.data.commit.id; + + // Then, upload the ZIP file as an asset to the release + const formData = new FormData(); + formData.append('name', archivePath); + formData.append( + 'url', + `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`, + ); + formData.append('file', fsApi.createReadStream(archivePath), { + filename: path.basename(archivePath), + }); + + const uploadResponse = await axios.post( + `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, + formData, + { + headers: { + ...formData.getHeaders(), + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, + }, + }, + ); + + const releaseUrl = uploadResponse.data.direct_asset_url; + + return releaseUrl; + } catch (error) { + console.error('Failed to create release or upload ZIP file:', error); + throw error; + } +} diff --git a/src/index.js b/src/index.js index 2083e1802..3f9c48133 100644 --- a/src/index.js +++ b/src/index.js @@ -8,6 +8,7 @@ import Archivist from './archivist/index.js'; import logger from './logger/index.js'; import Notifier from './notifier/index.js'; import Reporter from './reporter/index.js'; +import ReporterGitlab from './reporterGitlab/index.js'; const require = createRequire(import.meta.url); @@ -65,6 +66,7 @@ export default async function track({ services, types, extractOnly, schedule }) } catch (error) { logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); } + archivist.attach(reporter); } else { logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); } @@ -72,6 +74,23 @@ export default async function track({ services, types, extractOnly, schedule }) logger.warn('Environment variable "OTA_ENGINE_GITHUB_TOKEN" was not found; the Reporter module will be ignored'); } + if (process.env.OTA_ENGINE_GITLAB_TOKEN) { + if (config.has('@opentermsarchive/engine.reporter.gitlabIssues.repositories.declarations')) { + try { + const reporter = new ReporterGitlab(config.get('@opentermsarchive/engine.reporter')); + + await reporter.initialize(); + archivist.attach(reporter); + } catch (error) { + logger.error('Cannot instantiate the ReporterGitlab module; it will be ignored:', error); + } + } else { + logger.warn('Configuration key "reporter.gitlabIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); + } + } else { + logger.warn('Environment variable "OTA_ENGINE_GITLAB_TOKEN" was not found; the ReporterGitlab module will be ignored'); + } + if (!schedule) { await archivist.track({ services, types }); diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js new file mode 100644 index 000000000..e688d33e2 --- /dev/null +++ b/src/reporterGitlab/gitlab.js @@ -0,0 +1,324 @@ +import { createRequire } from 'module'; + +import logger from '../logger/index.js'; + +const require = createRequire(import.meta.url); + +export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; + +const gitlabUrl = "https://gitlab.com/api/v4"; + +export default class GitLab { + static ISSUE_STATE_CLOSED = 'closed'; + static ISSUE_STATE_OPEN = 'opened'; + static ISSUE_STATE_ALL = 'all'; + + constructor(repository) { + //const { version } = require('../../package.json'); + + const [owner, repo] = repository.split('/'); + + this.commonParams = { owner, repo }; + } + + async initialize() { + const axios = require('axios'); + + try { + const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; + const response = await axios.get( + `${gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + }, + }, + ); + this.projectId = response.data.id; + } catch (error) { + logger.error(`🤖 Error while obtaining projectId: ${error}`); + this.projectId = null; + } + this.MANAGED_LABELS = require('./labels.json'); + + const existingLabels = await this.getRepositoryLabels(); + const existingLabelsNames = existingLabels.map((label) => label.name); + const missingLabels = this.MANAGED_LABELS.filter( + (label) => !existingLabelsNames.includes(label.name), + ); + + if (missingLabels.length) { + logger.info( + `🤖 Following required labels are not present on the repository: ${missingLabels.map((label) => `"${label.name}"`).join(', ')}. Creating them…`, + ); + + for (const label of missingLabels) { + await this.createLabel({ + /* eslint-disable-line no-await-in-loop */ name: label.name, + color: label.color, + description: `${label.description} ${MANAGED_BY_OTA_MARKER}`, + }); + } + } + } + + async getRepositoryLabels() { + try { + const response = await fetch( + `${gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + { + method: 'GET', + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + }, + }, + ); + if (response.status == 200) { + const labels = response.json(); + return labels; + } else { + logger.error( + `🤖 Failed to get labels: ${response.status_code} - ${response.text}`, + ); + return null; + } + } catch (error) { + logger.error(`🤖 Could get labels: ${error}`); + } + } + + async createLabel({ name, color, description }) { + const axios = require('axios'); + + try { + const label = { + name: name, + color: color, + description: description, + }; + const response = await axios.post( + `${gitlabUrl}/projects/${this.projectId}/labels`, + label, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + logger.info(`🤖 New label created: ${response.data.name}`); + } catch (error) { + logger.error(`🤖 Failed to create label: ${error}`); + } + } + + async createIssue({ title, description, labels }) { + const axios = require('axios'); + + try { + const issue = { + title: title, + labels: labels, + description: description, + }; + const response = await axios.post( + `${gitlabUrl}/projects/${this.projectId}/issues`, + issue, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + logger.info( + `🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`, + ); + + return response; + } catch (error) { + logger.error(`🤖 Could not create GitLab issue "${title}": ${error}`); + } + } + + async setIssueLabels({ issue, labels }) { + const axios = require('axios'); + + try { + const newLabels = { + labels: labels, + }; + const response = await axios.put( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + newLabels, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + } catch (error) { + logger.error( + `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async openIssue(issue) { + const axios = require('axios'); + + try { + const updateIssue = { + state_event: 'reopen', + }; + const response = await axios.put( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + updateIssue, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + logger.info(`🤖 Opened GitLab issue #${issue.iid}`); + } catch (error) { + logger.error( + `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async closeIssue(issue) { + const axios = require('axios'); + + try { + const updateIssue = { + state_event: 'close', + }; + const response = await axios.put( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + updateIssue, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + + logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + } catch (error) { + logger.error( + `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async getIssue({ title, ...searchParams }) { + const axios = require('axios'); + + try { + let apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + if (searchParams.state == 'all') + apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; + const response = await axios.get(apiUrl, { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + }, + }); + const issues = response.data; + + const [issue] = issues.filter((item) => item.title === title); // since only one is expected, use the first one + + setTimeout(() => { + console.log(title + ' - ' + apiUrl); + }, 5000); + + return issue; + } catch (error) { + logger.error(`🤖 Could not find GitLab issue "${title}": ${error}`); + } + } + + async addCommentToIssue({ issue, comment }) { + const axios = require('axios'); + const body = { + body: comment, + }; + + try { + const response = await axios.post( + `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, + body, + { + headers: { + Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, + 'Content-Type': 'application/json', + }, + }, + ); + logger.info( + `🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`, + ); + + return response.data.body; + } catch (error) { + logger.error( + `🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`, + ); + } + } + + async closeIssueWithCommentIfExists({ title, comment }) { + const openedIssue = await this.getIssue({ + title, + state: GitLab.ISSUE_STATE_OPEN, + }); + + if (!openedIssue) { + return; + } + + await this.addCommentToIssue({ issue: openedIssue, comment }); + + return this.closeIssue(openedIssue); + } + + async createOrUpdateIssue({ title, description, label }) { + const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_ALL }); + + if (!issue) { + return this.createIssue({ title, description, labels: [label] }); + } + + if (issue.state == GitLab.ISSUE_STATE_CLOSED) { + await this.openIssue(issue); + } + + const managedLabelsNames = this.MANAGED_LABELS.map((label) => label.name); + const [managedLabel] = issue.labels.filter((label) => + managedLabelsNames.includes(label.name), + ); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive + + if (managedLabel?.name == label) { + // if the label is already assigned to the issue, the error is redundant with the one already reported and no further action is necessary + return; + } + + const labelsNotManagedToKeep = issue.labels + .map((label) => label.name) + .filter((label) => !managedLabelsNames.includes(label)); + + await this.setIssueLabels({ + issue, + labels: [label, ...labelsNotManagedToKeep], + }); + await this.addCommentToIssue({ issue, comment: description }); + } +} diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js new file mode 100644 index 000000000..d0917607c --- /dev/null +++ b/src/reporterGitlab/index.js @@ -0,0 +1,147 @@ +import mime from 'mime'; + +import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; + +import GitLab from './gitlab.js'; + +const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/'; +const DOC_URL = 'https://docs.opentermsarchive.org'; +const REPO_URL = 'https://gitlab.com/'; + +const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = { + 'has no match': 'selectors', + 'HTTP code 404': 'location', + 'HTTP code 403': '403', + 'HTTP code 429': '429', + 'HTTP code 500': '500', + 'HTTP code 502': '502', + 'HTTP code 503': '503', + 'Timed out after': 'timeout', + 'getaddrinfo EAI_AGAIN': 'EAI_AGAIN', + 'getaddrinfo ENOTFOUND': 'ENOTFOUND', + 'Response is empty': 'empty response', + 'unable to verify the first certificate': 'first certificate', + 'certificate has expired': 'certificate expired', + 'maximum redirect reached': 'redirects', +}; + +function getLabelNameFromError(error) { + return ERROR_MESSAGE_TO_ISSUE_LABEL_MAP[Object.keys(ERROR_MESSAGE_TO_ISSUE_LABEL_MAP).find(substring => error.toString().includes(substring))] || 'to clarify'; +} + +// In the following class, it is assumed that each issue is managed using its title as a unique identifier +export default class Reporter { + constructor(config) { + const { repositories } = config.gitlabIssues; + + for (const repositoryType of Object.keys(repositories)) { + if (!repositories[repositoryType].includes('/') || repositories[repositoryType].includes('https://')) { + throw new Error(`Configuration entry "reporter.gitlabIssues.repositories.${repositoryType}" is expected to be a string in the format /, but received: "${repositories[repositoryType]}"`); + } + } + + this.gitlab = new GitLab(repositories.declarations); + this.repositories = repositories; + } + + initialize() { + return this.gitlab.initialize(); + } + + async onVersionRecorded(version) { + await this.gitlab.closeIssueWithCommentIfExists({ + title: Reporter.generateTitleID(version.serviceId, version.termsType), + comment: `### Tracking resumed + +A new version has been recorded.`, + }); + } + + async onVersionNotChanged(version) { + await this.gitlab.closeIssueWithCommentIfExists({ + title: Reporter.generateTitleID(version.serviceId, version.termsType), + comment: `### Tracking resumed + +No changes were found in the last run, so no new version has been recorded.`, + }); + } + + async onFirstVersionRecorded(version) { + return this.onVersionRecorded(version); + } + + async onInaccessibleContent(error, terms) { + await this.gitlab.createOrUpdateIssue({ + title: Reporter.generateTitleID(terms.service.id, terms.type), + description: this.generateDescription({ error, terms }), + label: getLabelNameFromError(error), + }); + } + + generateDescription({ error, terms }) { + const date = new Date(); + const currentFormattedDate = date.toLocaleDateString('en-GB', { year: 'numeric', month: 'long', day: 'numeric', hour: 'numeric', minute: 'numeric', second: 'numeric', timeZoneName: 'short', timeZone: 'UTC' }); + const validUntil = toISODateWithoutMilliseconds(date); + + const hasSnapshots = terms.sourceDocuments.every(sourceDocument => sourceDocument.snapshotId); + + const contributionToolParams = new URLSearchParams({ + json: JSON.stringify(terms.toPersistence()), + destination: this.repositories.declarations, + step: '2', + }); + const contributionToolUrl = `${CONTRIBUTION_TOOL_URL}?${contributionToolParams}`; + + const latestDeclarationLink = `[Latest declaration](${REPO_URL}/${this.repositories.declarations}/-/blob/main/declarations/${encodeURIComponent(terms.service.name)}.json)`; + const latestVersionLink = `[Latest version](${REPO_URL}/${this.repositories.versions}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}.md)`; + const snapshotsBaseUrl = `${REPO_URL}/${this.repositories.snapshots}/-/blob/main/${encodeURIComponent(terms.service.name)}/${encodeURIComponent(terms.type)}`; + const latestSnapshotsLink = terms.hasMultipleSourceDocuments + ? `Latest snapshots:\n - ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.id}](${snapshotsBaseUrl}.%20#${sourceDocument.id}.${mime.getExtension(sourceDocument.mimeType)})`).join('\n - ')}` + : `[Latest snapshot](${snapshotsBaseUrl}.${mime.getExtension(terms.sourceDocuments[0].mimeType)})`; + + /* eslint-disable no-irregular-whitespace */ + return ` +### No version of the \`${terms.type}\` of service \`${terms.service.name}\` is recorded anymore since ${currentFormattedDate} + +The source document${terms.hasMultipleSourceDocuments ? 's have' : ' has'}${hasSnapshots ? ' ' : ' not '}been recorded in ${terms.hasMultipleSourceDocuments ? 'snapshots' : 'a snapshot'}, ${hasSnapshots ? 'but ' : 'thus '} no version can be [extracted](${DOC_URL}/#tracking-terms). +${hasSnapshots ? 'After correction, it might still be possible to recover the missed versions.' : ''} + +### What went wrong + +- ${error.reasons.join('\n- ')} + +### How to resume tracking + +First of all, check if the source documents are accessible through a web browser: + +- [ ] ${terms.sourceDocuments.map(sourceDocument => `[${sourceDocument.location}](${sourceDocument.location})`).join('\n- [ ] ')} + +#### If the source documents are accessible through a web browser + +[Edit the declaration](${contributionToolUrl}): +- Try updating the selectors. +- Try switching client scripts on with expert mode. + +#### If the source documents are not accessible anymore + +- If the source documents have moved, find their new location and [update it](${contributionToolUrl}). +- If these terms have been removed, move them from the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. +- If the service has closed, move the entire contents of the declaration to its [history file](${DOC_URL}/contributing-terms/#service-history), using \`${validUntil}\` as the \`validUntil\` value. + +#### If none of the above works + +If the source documents are accessible in a browser but fetching them always fails from the Open Terms Archive server, this is most likely because the service provider has blocked the Open Terms Archive robots from accessing its content. In this case, updating the declaration will not enable resuming tracking. Only an agreement with the service provider, an engine upgrade, or some technical workarounds provided by the administrator of this collection’s server might resume tracking. + +### References + +- ${latestDeclarationLink} +${this.repositories.versions ? `- ${latestVersionLink}` : ''} +${this.repositories.snapshots ? `- ${latestSnapshotsLink}` : ''} +`; + /* eslint-enable no-irregular-whitespace */ + } + + static generateTitleID(serviceId, type) { + return `\`${serviceId}\` ‧ \`${type}\` ‧ not tracked anymore`; + } +} diff --git a/src/reporterGitlab/labels.json b/src/reporterGitlab/labels.json new file mode 100644 index 000000000..8d73f4e8e --- /dev/null +++ b/src/reporterGitlab/labels.json @@ -0,0 +1,77 @@ +[ + { + "name": "403", + "color": "#0b08a0", + "description": "Fetching fails with a 403 (forbidden) HTTP code" + }, + { + "name": "429", + "color": "#0b08a0", + "description": "Fetching fails with a 429 (too many requests) HTTP code" + }, + { + "name": "500", + "color": "#0b08a0", + "description": "Fetching fails with a 500 (internal server error) HTTP code" + }, + { + "name": "502", + "color": "#0b08a0", + "description": "Fetching fails with a 502 (bad gateway) HTTP code" + }, + { + "name": "503", + "color": "#0b08a0", + "description": "Fetching fails with a 503 (service unavailable) HTTP code" + }, + { + "name": "certificate expired", + "color": "#0b08a0", + "description": "Fetching fails because the domain SSL certificate has expired" + }, + { + "name": "EAI_AGAIN", + "color": "#0b08a0", + "description": "Fetching fails because the domain fails to resolve on DNS" + }, + { + "name": "ENOTFOUND", + "color": "#0b08a0", + "description": "Fetching fails because the domain fails to resolve on DNS" + }, + { + "name": "empty response", + "color": "#0b08a0", + "description": "Fetching fails with a “response is empty” error" + }, + { + "name": "first certificate", + "color": "#0b08a0", + "description": "Fetching fails with an “unable to verify the first certificate” error" + }, + { + "name": "redirects", + "color": "#0b08a0", + "description": "Fetching fails with a “too many redirects” error" + }, + { + "name": "timeout", + "color": "#0b08a0", + "description": "Fetching fails with a timeout error" + }, + { + "name": "to clarify", + "color": "#0496ff", + "description": "Default failure label" + }, + { + "name": "selectors", + "color": "#FBCA04", + "description": "Extraction selectors are outdated" + }, + { + "name": "location", + "color": "#FBCA04", + "description": "Fetch location is outdated" + } +] diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js new file mode 100644 index 000000000..0b4e9f75b --- /dev/null +++ b/src/reporterGitlab/labels.test.js @@ -0,0 +1,30 @@ +import { createRequire } from 'module'; + +import chai from 'chai'; + +import { MANAGED_BY_OTA_MARKER } from './gitlab.js'; + +const require = createRequire(import.meta.url); + +const { expect } = chai; +const labels = require('./labels.json'); + +const GITLAB_LABEL_DESCRIPTION_MAX_LENGTH = 255; + +describe('Reporter GitLab labels', () => { + labels.forEach(label => { + describe(`"${label.name}"`, () => { + it('complies with the GitLab character limit for descriptions', () => { + const descriptionLength = label.description.length + MANAGED_BY_OTA_MARKER.length; + + expect(descriptionLength).to.be.lessThan(GITLAB_LABEL_DESCRIPTION_MAX_LENGTH); + }); + + it('complies with the GitHub constraints for color', () => { + const validHexColorRegex = /^\#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` + + expect(validHexColorRegex.test(label.color)).to.be.true; + }); + }); + }); +}); From 7a94ea55694f67eac568374e12d8039f726ebeef Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Mon, 12 Aug 2024 11:14:32 +0200 Subject: [PATCH 02/11] Fix new gitlab files using eslint --- package.json | 4 +- scripts/dataset/index.js | 2 +- scripts/dataset/publishGitLab/index.js | 31 +++--- src/index.js | 1 - src/reporterGitlab/gitlab.js | 131 ++++++++++--------------- src/reporterGitlab/labels.test.js | 2 +- 6 files changed, 70 insertions(+), 101 deletions(-) diff --git a/package.json b/package.json index 8da099a32..85e90b222 100644 --- a/package.json +++ b/package.json @@ -57,6 +57,7 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", + "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", @@ -100,8 +101,7 @@ "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "winston": "^3.3.3", - "winston-mail": "^2.0.0", - "axios": "^1.7.2" + "winston-mail": "^2.0.0" }, "devDependencies": { "@commitlint/cli": "^19.0.3", diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index d6b74a3bf..735ad2d3c 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -41,7 +41,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } releaseDate, stats, }); - + logger.info(`Dataset published to ${releaseUrl}`); } diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 2a1f9bbc9..0e4d543c4 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -3,19 +3,17 @@ import path from 'path'; import url from 'url'; import axios from 'axios'; - import config from 'config'; import dotenv from 'dotenv'; -//import { Octokit } from 'octokit'; - import FormData from 'form-data'; import * as readme from '../assets/README.templateGitLab.js'; +import logger from '../logger/index.js'; dotenv.config(); -const gitlabAPIUrl = "https://gitlab.com/api/v4"; -const gitlabUrl = "https://gitlab.com"; +const gitlabAPIUrl = 'https://gitlab.com/api/v4'; +const gitlabUrl = 'https://gitlab.com'; export default async function publishReleaseGitLab({ archivePath, @@ -26,25 +24,22 @@ export default async function publishReleaseGitLab({ // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); - const [owner, repo] = url + const [ owner, repo ] = url .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) .pathname.split('/') - .filter((component) => component); + .filter(component => component); const commonParams = { owner, repo }; try { const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; const response = await axios.get( `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, - }, - }, + { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}` } }, ); + projectId = response.data.id; } catch (error) { - //logger.error(`🤖 Error while obtaining projectId: ${error}`); + // logger.error(`🤖 Error while obtaining projectId: ${error}`); projectId = null; } @@ -67,19 +62,19 @@ export default async function publishReleaseGitLab({ }, }, ); - const releaseId = releaseResponse.data.commit.id; + logger.info(`Created release with releaseId: ${releaseId}`); + // Then, upload the ZIP file as an asset to the release const formData = new FormData(); + formData.append('name', archivePath); formData.append( 'url', `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`, ); - formData.append('file', fsApi.createReadStream(archivePath), { - filename: path.basename(archivePath), - }); + formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); const uploadResponse = await axios.post( `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, @@ -96,7 +91,7 @@ export default async function publishReleaseGitLab({ return releaseUrl; } catch (error) { - console.error('Failed to create release or upload ZIP file:', error); + logger.error('Failed to create release or upload ZIP file:', error); throw error; } } diff --git a/src/index.js b/src/index.js index 3f9c48133..517512e07 100644 --- a/src/index.js +++ b/src/index.js @@ -66,7 +66,6 @@ export default async function track({ services, types, extractOnly, schedule }) } catch (error) { logger.error('Cannot instantiate the Reporter module; it will be ignored:', error); } - archivist.attach(reporter); } else { logger.warn('Configuration key "reporter.githubIssues.repositories.declarations" was not found; issues on the declarations repository cannot be created'); } diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index e688d33e2..bf7b8fa57 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -6,7 +6,7 @@ const require = createRequire(import.meta.url); export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; -const gitlabUrl = "https://gitlab.com/api/v4"; +const gitlabUrl = 'https://gitlab.com/api/v4'; export default class GitLab { static ISSUE_STATE_CLOSED = 'closed'; @@ -14,9 +14,9 @@ export default class GitLab { static ISSUE_STATE_ALL = 'all'; constructor(repository) { - //const { version } = require('../../package.json'); + // const { version } = require('../../package.json'); - const [owner, repo] = repository.split('/'); + const [ owner, repo ] = repository.split('/'); this.commonParams = { owner, repo }; } @@ -28,12 +28,9 @@ export default class GitLab { const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; const response = await axios.get( `${gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - }, - }, + { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }, ); + this.projectId = response.data.id; } catch (error) { logger.error(`🤖 Error while obtaining projectId: ${error}`); @@ -42,19 +39,15 @@ export default class GitLab { this.MANAGED_LABELS = require('./labels.json'); const existingLabels = await this.getRepositoryLabels(); - const existingLabelsNames = existingLabels.map((label) => label.name); - const missingLabels = this.MANAGED_LABELS.filter( - (label) => !existingLabelsNames.includes(label.name), - ); + const existingLabelsNames = existingLabels.map(label => label.name); + const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name)); if (missingLabels.length) { - logger.info( - `🤖 Following required labels are not present on the repository: ${missingLabels.map((label) => `"${label.name}"`).join(', ')}. Creating them…`, - ); + logger.info(`🤖 Following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); for (const label of missingLabels) { - await this.createLabel({ - /* eslint-disable-line no-await-in-loop */ name: label.name, + await this.createLabel({ /* eslint-disable-line no-await-in-loop */ + name: label.name, color: label.color, description: `${label.description} ${MANAGED_BY_OTA_MARKER}`, }); @@ -65,23 +58,21 @@ export default class GitLab { async getRepositoryLabels() { try { const response = await fetch( - `${gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + `https://gitlab.com/api/v4/projects/4/labels?with_counts=true`, { method: 'GET', - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - }, + headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` }, }, ); + if (response.status == 200) { const labels = response.json(); + return labels; - } else { - logger.error( - `🤖 Failed to get labels: ${response.status_code} - ${response.text}`, - ); - return null; } + logger.error(`🤖 Failed to get labels: ${response.status_code} - ${response.text}`); + + return null; } catch (error) { logger.error(`🤖 Could get labels: ${error}`); } @@ -92,9 +83,9 @@ export default class GitLab { try { const label = { - name: name, - color: color, - description: description, + name, + color, + description, }; const response = await axios.post( `${gitlabUrl}/projects/${this.projectId}/labels`, @@ -106,6 +97,7 @@ export default class GitLab { }, }, ); + logger.info(`🤖 New label created: ${response.data.name}`); } catch (error) { logger.error(`🤖 Failed to create label: ${error}`); @@ -117,9 +109,9 @@ export default class GitLab { try { const issue = { - title: title, - labels: labels, - description: description, + title, + labels, + description, }; const response = await axios.post( `${gitlabUrl}/projects/${this.projectId}/issues`, @@ -131,9 +123,8 @@ export default class GitLab { }, }, ); - logger.info( - `🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`, - ); + + logger.info(`🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`); return response; } catch (error) { @@ -145,9 +136,7 @@ export default class GitLab { const axios = require('axios'); try { - const newLabels = { - labels: labels, - }; + const newLabels = { labels }; const response = await axios.put( `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, newLabels, @@ -159,11 +148,11 @@ export default class GitLab { }, ); + logger.debug(`response data: ${response.data}`); + logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); } catch (error) { - logger.error( - `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -171,9 +160,7 @@ export default class GitLab { const axios = require('axios'); try { - const updateIssue = { - state_event: 'reopen', - }; + const updateIssue = { state_event: 'reopen' }; const response = await axios.put( `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, updateIssue, @@ -185,11 +172,11 @@ export default class GitLab { }, ); + logger.debug(`response data: ${response.data}`); + logger.info(`🤖 Opened GitLab issue #${issue.iid}`); } catch (error) { - logger.error( - `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -197,9 +184,7 @@ export default class GitLab { const axios = require('axios'); try { - const updateIssue = { - state_event: 'close', - }; + const updateIssue = { state_event: 'close' }; const response = await axios.put( `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, updateIssue, @@ -211,11 +196,11 @@ export default class GitLab { }, ); + logger.debug(`response data: ${response.data}`); + logger.info(`🤖 Closed GitLab issue #${issue.iid}`); } catch (error) { - logger.error( - `🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -224,20 +209,16 @@ export default class GitLab { try { let apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; - if (searchParams.state == 'all') - apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + + if (searchParams.state == 'all') apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; - const response = await axios.get(apiUrl, { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - }, - }); + const response = await axios.get(apiUrl, { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }); const issues = response.data; - const [issue] = issues.filter((item) => item.title === title); // since only one is expected, use the first one + const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one setTimeout(() => { - console.log(title + ' - ' + apiUrl); + console.log(`${title} - ${apiUrl}`); }, 5000); return issue; @@ -248,9 +229,7 @@ export default class GitLab { async addCommentToIssue({ issue, comment }) { const axios = require('axios'); - const body = { - body: comment, - }; + const body = { body: comment }; try { const response = await axios.post( @@ -263,15 +242,12 @@ export default class GitLab { }, }, ); - logger.info( - `🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`, - ); + + logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`); return response.data.body; } catch (error) { - logger.error( - `🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`, - ); + logger.error(`🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -301,10 +277,9 @@ export default class GitLab { await this.openIssue(issue); } - const managedLabelsNames = this.MANAGED_LABELS.map((label) => label.name); - const [managedLabel] = issue.labels.filter((label) => - managedLabelsNames.includes(label.name), - ); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive + const managedLabelsNames = this.MANAGED_LABELS.map(label => label.name); + const [managedLabel] = issue.labels.filter(label => + managedLabelsNames.includes(label.name)); // it is assumed that only one specific reason for failure is possible at a time, making managed labels mutually exclusive if (managedLabel?.name == label) { // if the label is already assigned to the issue, the error is redundant with the one already reported and no further action is necessary @@ -312,12 +287,12 @@ export default class GitLab { } const labelsNotManagedToKeep = issue.labels - .map((label) => label.name) - .filter((label) => !managedLabelsNames.includes(label)); + .map(label => label.name) + .filter(label => !managedLabelsNames.includes(label)); await this.setIssueLabels({ issue, - labels: [label, ...labelsNotManagedToKeep], + labels: [ label, ...labelsNotManagedToKeep ], }); await this.addCommentToIssue({ issue, comment: description }); } diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index 0b4e9f75b..3a0c78763 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -21,7 +21,7 @@ describe('Reporter GitLab labels', () => { }); it('complies with the GitHub constraints for color', () => { - const validHexColorRegex = /^\#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` + const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` expect(validHexColorRegex.test(label.color)).to.be.true; }); From e540a1b0efdd486e0ad6c2b310f69d4788b5e2f1 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Fri, 6 Sep 2024 17:57:35 +0200 Subject: [PATCH 03/11] Add tests for GitLab class, remove axios and format fixes add test file for GitLab, fix formatting in files, refactor GitLab code to remove axios library --- .env.example | 2 + package-lock.json | 11 - package.json | 1 - scripts/dataset/publishGitLab/index.js | 84 ++-- src/reporterGitlab/gitlab.js | 295 ++++++++------ src/reporterGitlab/gitlab.test.js | 532 +++++++++++++++++++++++++ src/reporterGitlab/index.js | 2 +- src/reporterGitlab/labels.test.js | 2 +- 8 files changed, 768 insertions(+), 161 deletions(-) create mode 100644 src/reporterGitlab/gitlab.test.js diff --git a/.env.example b/.env.example index 08813d698..bae1061cd 100644 --- a/.env.example +++ b/.env.example @@ -3,3 +3,5 @@ OTA_ENGINE_SMTP_PASSWORD='password' OTA_ENGINE_GITHUB_TOKEN=ghp_XXXXXXXXX OTA_ENGINE_GITLAB_TOKEN=XXXXXXXXXX OTA_ENGINE_GITLAB_RELEASES_TOKEN=XXXXXXXXXX +OTA_ENGINE_GITLAB_API_BASE_URL=https://gitlab.com/api/v4 +OTA_ENGINE_GITLAB_BASE_URL=https://gitlab.com \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index dfed6a315..55828f585 100644 --- a/package-lock.json +++ b/package-lock.json @@ -18,7 +18,6 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", - "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", @@ -3980,16 +3979,6 @@ "uuid": "dist/bin/uuid" } }, - "node_modules/axios": { - "version": "1.7.2", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.2.tgz", - "integrity": "sha512-2A8QhOMrbomlDuiLeK9XibIBzuHeRcqqNOHp0Cyp5EoJ1IFDh+XZH3A6BkXtv0K4gFGCI0Y4BM7B1wOEi0Rmgw==", - "dependencies": { - "follow-redirects": "^1.15.6", - "form-data": "^4.0.0", - "proxy-from-env": "^1.1.0" - } - }, "node_modules/b4a": { "version": "1.6.6", "resolved": "https://registry.npmjs.org/b4a/-/b4a-1.6.6.tgz", diff --git a/package.json b/package.json index 85e90b222..8fccde58e 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,6 @@ "ajv": "^6.12.6", "archiver": "^5.3.0", "async": "^3.2.2", - "axios": "^1.7.2", "chai": "^4.3.4", "chai-as-promised": "^7.1.1", "chai-exclude": "^2.1.0", diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 0e4d543c4..c752c1abe 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -2,18 +2,19 @@ import fsApi from 'fs'; import path from 'path'; import url from 'url'; -import axios from 'axios'; import config from 'config'; import dotenv from 'dotenv'; import FormData from 'form-data'; +import nodeFetch from 'node-fetch'; +import GitLab from '../../../src/reporterGitlab/gitlab.js'; import * as readme from '../assets/README.templateGitLab.js'; import logger from '../logger/index.js'; dotenv.config(); -const gitlabAPIUrl = 'https://gitlab.com/api/v4'; -const gitlabUrl = 'https://gitlab.com'; +const gitlabAPIUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; +const gitlabUrl = process.env.OTA_ENGINE_GITLAB_BASE_URL; export default async function publishReleaseGitLab({ archivePath, @@ -22,8 +23,6 @@ export default async function publishReleaseGitLab({ }) { let projectId = null; - // const octokit = new Octokit({ auth: process.env.GITHUB_TOKEN }); - const [ owner, repo ] = url .parse(config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')) .pathname.split('/') @@ -32,37 +31,53 @@ export default async function publishReleaseGitLab({ try { const repositoryPath = `${commonParams.owner}/${commonParams.repo}`; - const response = await axios.get( + + const options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + + options.method = 'GET'; + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( `${gitlabAPIUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}` } }, + options, ); + const res = await response.json(); - projectId = response.data.id; + projectId = res.id; } catch (error) { - // logger.error(`🤖 Error while obtaining projectId: ${error}`); + logger.error(`🤖 Error while obtaining projectId: ${error}`); projectId = null; } const tagName = `${path.basename(archivePath, path.extname(archivePath))}`; // use archive filename as Git tag try { - // First, create the release - const releaseResponse = await axios.post( + let options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + + options.method = 'POST'; + options.body = { + ref: 'main', + tag_name: tagName, + name: readme.title({ releaseDate }), + description: readme.body(stats), + }; + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + options.body = JSON.stringify(options.body); + + const releaseResponse = await nodeFetch( `${gitlabAPIUrl}/projects/${projectId}/releases`, - { - ref: 'main', - tag_name: tagName, - name: readme.title({ releaseDate }), - description: readme.body(stats), - }, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + options, ); - const releaseId = releaseResponse.data.commit.id; + const releaseRes = await releaseResponse.json(); + + const releaseId = releaseRes.commit.id; logger.info(`Created release with releaseId: ${releaseId}`); @@ -76,18 +91,21 @@ export default async function publishReleaseGitLab({ ); formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); - const uploadResponse = await axios.post( + options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + options.method = 'POST'; + options.headers = { + ...formData.getHeaders(), + ...options.headers, + }; + options.body = formData; + + const uploadResponse = await nodeFetch( `${gitlabAPIUrl}/projects/${projectId}/releases/${tagName}/assets/links`, - formData, - { - headers: { - ...formData.getHeaders(), - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN}`, - }, - }, + options, ); + const uploadRes = await uploadResponse.json(); - const releaseUrl = uploadResponse.data.direct_asset_url; + const releaseUrl = uploadRes.direct_asset_url; return releaseUrl; } catch (error) { diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index bf7b8fa57..392b5b481 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -1,37 +1,48 @@ import { createRequire } from 'module'; +import HttpProxyAgent from 'http-proxy-agent'; +import HttpsProxyAgent from 'https-proxy-agent'; +import nodeFetch from 'node-fetch'; + import logger from '../logger/index.js'; const require = createRequire(import.meta.url); export const MANAGED_BY_OTA_MARKER = '[managed by OTA]'; -const gitlabUrl = 'https://gitlab.com/api/v4'; - export default class GitLab { static ISSUE_STATE_CLOSED = 'closed'; static ISSUE_STATE_OPEN = 'opened'; static ISSUE_STATE_ALL = 'all'; constructor(repository) { - // const { version } = require('../../package.json'); - const [ owner, repo ] = repository.split('/'); this.commonParams = { owner, repo }; + this.projectId = null; + const gitlabUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; + + this.gitlabUrl = gitlabUrl; } async initialize() { - const axios = require('axios'); + const options = GitLab.baseOptionsHttpReq(); try { const repositoryPath = `${this.commonParams.owner}/${this.commonParams.repo}`; - const response = await axios.get( - `${gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, - { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${encodeURIComponent(repositoryPath)}`, + options, ); - this.projectId = response.data.id; + const res = await response.json(); + + if (response.ok) { + this.projectId = res.id; + } else { + logger.error(`🤖 Error while obtaining projectId: ${JSON.strinfigy(res)}`); + this.projectId = null; + } } catch (error) { logger.error(`🤖 Error while obtaining projectId: ${error}`); this.projectId = null; @@ -57,20 +68,18 @@ export default class GitLab { async getRepositoryLabels() { try { - const response = await fetch( - `https://gitlab.com/api/v4/projects/4/labels?with_counts=true`, - { - method: 'GET', - headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` }, - }, + const options = GitLab.baseOptionsHttpReq(); + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/labels?with_counts=true`, + options, ); + const res = await response.json(); - if (response.status == 200) { - const labels = response.json(); - - return labels; + if (response.ok) { + return res; } - logger.error(`🤖 Failed to get labels: ${response.status_code} - ${response.text}`); + + logger.error(`🤖 Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); return null; } catch (error) { @@ -79,191 +88,235 @@ export default class GitLab { } async createLabel({ name, color, description }) { - const axios = require('axios'); - try { const label = { name, color, description, }; - const response = await axios.post( - `${gitlabUrl}/projects/${this.projectId}/labels`, - label, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(label); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/labels`, + options, ); - logger.info(`🤖 New label created: ${response.data.name}`); + const res = await response.json(); + + if (response.ok) { + logger.info(`🤖 New label created: ${res.name} , color: ${res.color}`); + } else { + logger.error(`createLabel response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Failed to create label: ${error}`); } } async createIssue({ title, description, labels }) { - const axios = require('axios'); - try { const issue = { title, labels, description, }; - const response = await axios.post( - `${gitlabUrl}/projects/${this.projectId}/issues`, - issue, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(issue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues`, + options, ); - logger.info(`🤖 Created GitLab issue #${response.data.iid} "${title}": ${response.data.web_url}`); + const res = await response.json(); + + if (response.ok) { + logger.info(`🤖 Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); + + return res; + } - return response; + logger.error(`createIssue response: ${JSON.stringify(res)}`); } catch (error) { logger.error(`🤖 Could not create GitLab issue "${title}": ${error}`); } } async setIssueLabels({ issue, labels }) { - const axios = require('axios'); + const newLabels = { labels }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(newLabels); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; try { - const newLabels = { labels }; - const response = await axios.put( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, - newLabels, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, ); - logger.debug(`response data: ${response.data}`); + const res = await response.json(); - logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + if (response.ok) { + logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + } else { + logger.error(`setIssueLabels response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async openIssue(issue) { - const axios = require('axios'); + const updateIssue = { state_event: 'reopen' }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(updateIssue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; try { - const updateIssue = { state_event: 'reopen' }; - const response = await axios.put( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, - updateIssue, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, ); + const res = await response.json(); - logger.debug(`response data: ${response.data}`); - - logger.info(`🤖 Opened GitLab issue #${issue.iid}`); + if (response.ok) { + logger.info(`🤖 Opened GitLab issue #${res.iid}`); + } else { + logger.error(`openIssue response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async closeIssue(issue) { - const axios = require('axios'); + const updateIssue = { state_event: 'close' }; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'PUT'; + options.body = JSON.stringify(updateIssue); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; try { - const updateIssue = { state_event: 'close' }; - const response = await axios.put( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, - updateIssue, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}`, + options, ); + const res = await response.json(); - logger.debug(`response data: ${response.data}`); - - logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + if (response.ok) { + logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + } else { + logger.error(`closeIssue response: ${JSON.stringify(res)}`); + } } catch (error) { logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async getIssue({ title, ...searchParams }) { - const axios = require('axios'); - try { - let apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + let apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?state=${searchParams.state}&per_page=100`; + + if (searchParams.state == 'all') apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; + apiUrl = `${this.gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; + + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'GET'; + + const response = await nodeFetch(apiUrl, options); + const res = await response.json(); - if (searchParams.state == 'all') apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?per_page=100`; - apiUrl = `${gitlabUrl}/projects/${this.projectId}/issues?search=${encodeURIComponent(title)}&per_page=100`; - const response = await axios.get(apiUrl, { headers: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }); - const issues = response.data; + if (response.ok) { + logger.debug(`response data: ${JSON.stringify(res)}`); + const issues = res; - const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one + const [issue] = issues.filter(item => item.title === title); // since only one is expected, use the first one - setTimeout(() => { - console.log(`${title} - ${apiUrl}`); - }, 5000); + return issue; + } - return issue; + logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { logger.error(`🤖 Could not find GitLab issue "${title}": ${error}`); } } async addCommentToIssue({ issue, comment }) { - const axios = require('axios'); const body = { body: comment }; + const options = GitLab.baseOptionsHttpReq(); + + options.method = 'POST'; + options.body = JSON.stringify(body); + options.headers = { + 'Content-Type': 'application/json', + ...options.headers, + }; + try { - const response = await axios.post( - `${gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, - body, - { - headers: { - Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}`, - 'Content-Type': 'application/json', - }, - }, + const response = await nodeFetch( + `${this.gitlabUrl}/projects/${this.projectId}/issues/${issue.iid}/notes`, + options, ); + const res = await response.json(); - logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${response.data.id}`); + if (response.ok) { + logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); - return response.data.body; + return res.body; + } + + logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { logger.error(`🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } async closeIssueWithCommentIfExists({ title, comment }) { - const openedIssue = await this.getIssue({ + const issue = await this.getIssue({ title, state: GitLab.ISSUE_STATE_OPEN, }); - if (!openedIssue) { + // if issue does not exist in the "opened" state + if (!issue) { return; } - await this.addCommentToIssue({ issue: openedIssue, comment }); + await this.addCommentToIssue({ issue, comment }); - return this.closeIssue(openedIssue); + return this.closeIssue(issue); } async createOrUpdateIssue({ title, description, label }) { @@ -296,4 +349,18 @@ export default class GitLab { }); await this.addCommentToIssue({ issue, comment: description }); } + + static baseOptionsHttpReq(token = process.env.OTA_ENGINE_GITLAB_TOKEN) { + const options = {}; + + if (process.env.HTTPS_PROXY) { + options.agent = new HttpsProxyAgent(process.env.HTTPS_PROXY); + } else if (process.env.HTTP_PROXY) { + options.agent = new HttpProxyAgent(process.env.HTTP_PROXY); + } + + options.headers = { Authorization: `Bearer ${token}` }; + + return options; + } } diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js new file mode 100644 index 000000000..ede97cd18 --- /dev/null +++ b/src/reporterGitlab/gitlab.test.js @@ -0,0 +1,532 @@ +import { createRequire } from 'module'; + +import { expect } from 'chai'; +import nock from 'nock'; + +import GitLab from './gitlab.js'; + +const require = createRequire(import.meta.url); + +describe('GitLab', function () { + this.timeout(5000); + + let MANAGED_LABELS; + let gitlab; + let gitlabApiUrl = ''; + let reqHeaders; + const projectId = '4'; + + before(() => { + MANAGED_LABELS = require('./labels.json'); + gitlab = new GitLab('owner/repo'); + gitlab.projectId = projectId; + gitlabApiUrl = gitlab.gitlabUrl; + reqHeaders = { reqheaders: { Authorization: `Bearer ${process.env.OTA_ENGINE_GITLAB_TOKEN}` } }; + }); + + describe('#Gitlab_initialize', () => { + const scopes = []; + + before(async () => { + const existingLabels = MANAGED_LABELS.slice(0, -2); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${encodeURIComponent('owner/repo')}`) + .reply(200, { id: 4 }); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, existingLabels); + + const missingLabels = MANAGED_LABELS.slice(-2); + + for (const label of missingLabels) { + scopes.push(nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/labels`) + .reply(200, { name: label.name })); + } + + await gitlab.initialize(); + }); + + after(nock.cleanAll); + + it('should create missing labels', () => { + scopes.forEach(scope => expect(scope.isDone()).to.be.true); + }); + }); + + describe('#Gitlab_getRepositoryLabels', () => { + let scope; + let result; + const LABELS = [{ name: 'bug' }, { name: 'enhancement' }]; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, LABELS); + + result = await gitlab.getRepositoryLabels(); + }); + + after(nock.cleanAll); + + it('fetches repository labels', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the repository labels', () => { + expect(result).to.deep.equal(LABELS); + }); + }); + + describe('#Gitlab_createLabel', () => { + let scope; + const LABEL = { name: 'new_label', color: 'ffffff' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/labels`, body => body.name === LABEL.name) + .reply(200, LABEL); + + await gitlab.createLabel(LABEL); + }); + + after(nock.cleanAll); + + it('creates the new label', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_createIssue', () => { + let scope; + let result; + + const ISSUE = { + title: 'New Issue', + description: 'Description of the new issue', + labels: ['bug'], + }; + const CREATED_ISSUE = { + title: 'New Issue', + description: 'Description of the new issue', + labels: ['bug'], + iid: 555, + web_url: 'https://example.com/test/test', + }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues`) + .reply(200, CREATED_ISSUE); + + result = await gitlab.createIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('creates the new issue', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the created issue', () => { + expect(result).to.deep.equal(CREATED_ISSUE); + }); + }); + + describe('#Gitlab_setIssueLabels', () => { + let scope; + const issue = { + iid: 123, + title: 'test issue', + }; + const labels = [ 'bug', 'enhancement' ]; + + const response = { + iid: 123, + labels, + }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${issue.iid}`, { labels }) + .reply(200, response); + + await gitlab.setIssueLabels({ issue, labels }); + }); + + after(nock.cleanAll); + + it('sets labels on the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_openIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'issue reopened' }; + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const response = { iid: 123 }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + .reply(200, response); + + await gitlab.openIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('opens the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_closeIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'close issue' }; + const EXPECTED_REQUEST_BODY = { state_event: 'close' }; + const response = { iid: 123 }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, EXPECTED_REQUEST_BODY) + .reply(200, response); + + await gitlab.closeIssue(ISSUE); + }); + + after(nock.cleanAll); + + it('closes the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_getIssue', () => { + let scope; + let result; + + const ISSUE = { number: 123, title: 'Test Issue' }; + const ANOTHER_ISSUE = { number: 124, title: 'Test Issue 2' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [ ISSUE, ANOTHER_ISSUE ]); + + result = await gitlab.getIssue({ title: ISSUE.title }); + }); + + after(nock.cleanAll); + + it('searches for the issue', () => { + expect(scope.isDone()).to.be.true; + }); + + it('returns the expected issue', () => { + expect(result).to.deep.equal(ISSUE); + }); + }); + + describe('#Gitlab_addCommentToIssue', () => { + let scope; + const ISSUE = { iid: 123, title: 'Test Issue' }; + const COMMENT = 'Test comment'; + const response = { iid: 123, id: 23, body: 'Test comment' }; + + before(async () => { + scope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, response); + + await gitlab.addCommentToIssue({ issue: ISSUE, comment: COMMENT }); + }); + + after(nock.cleanAll); + + it('adds the comment to the issue', () => { + expect(scope.isDone()).to.be.true; + }); + }); + + describe('#Gitlab_closeIssueWithCommentIfExists', () => { + after(nock.cleanAll); + + context('when the issue exists and is open', () => { + const ISSUE = { + iid: 123, + title: 'Open Issue', + state: GitLab.ISSUE_STATE_OPEN, + }; + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [ISSUE]); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + + it('closes the issue', () => { + expect(closeIssueScope.isDone()).to.be.true; + }); + }); + + context('when the issue exists and is closed', () => { + const ISSUE = { + number: 123, + title: 'Closed Issue', + state: GitLab.ISSUE_STATE_CLOSED, + }; + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, []); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${ISSUE.iid}/notes`, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${ISSUE.iid}`, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: ISSUE.title, comment: COMMENT }); + }); + + it('does not add comment', () => { + expect(addCommentScope.isDone()).to.be.false; + }); + + it('does not attempt to close the issue', () => { + expect(closeIssueScope.isDone()).to.be.false; + }); + }); + + context('when the issue does not exist', () => { + let addCommentScope; + let closeIssueScope; + const COMMENT = 'Closing comment'; + const TITLE = 'Non-existent Issue'; + const responseAddcomment = { iid: 123, id: 23, body: COMMENT }; + const closeissueBody = { state_event: 'close' }; + const responseCloseissue = { iid: 123 }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(TITLE)}&per_page=100`) + .reply(200, []); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(/\/projects\/\d+\/issues\/\d+\/notes/, { body: COMMENT }) + .reply(200, responseAddcomment); + + closeIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(/\/projects\/\d+\/issues\/\d+/, closeissueBody) + .reply(200, responseCloseissue); + + await gitlab.closeIssueWithCommentIfExists({ title: TITLE, comment: COMMENT }); + }); + + it('does not attempt to add comment', () => { + expect(addCommentScope.isDone()).to.be.false; + }); + + it('does not attempt to close the issue', () => { + expect(closeIssueScope.isDone()).to.be.false; + }); + }); + }); + + describe('#Gitlab_createOrUpdateIssue', () => { + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${encodeURIComponent('owner/repo')}`) + .reply(200, { id: 4 }); + + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/labels?with_counts=true`) + .reply(200, MANAGED_LABELS); + + await gitlab.initialize(); + }); + + context('when the issue does not exist', () => { + let createIssueScope; + const ISSUE_TO_CREATE = { + title: 'New Issue', + description: 'Description of the new issue', + label: 'bug', + }; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE_TO_CREATE.title)}&per_page=100`) + .reply(200, []); // Simulate that there is no issues on the repository + + createIssueScope = nock(gitlabApiUrl, reqHeaders) + .post( + `/projects/${projectId}/issues`, + { + title: ISSUE_TO_CREATE.title, + description: ISSUE_TO_CREATE.description, + labels: [ISSUE_TO_CREATE.label], + }, + ) + .reply(200, { iid: 123, web_url: 'https://example.com/test/test' }); + + await gitlab.createOrUpdateIssue(ISSUE_TO_CREATE); + }); + + it('creates the issue', () => { + expect(createIssueScope.isDone()).to.be.true; + }); + }); + + context('when the issue already exists', () => { + const ISSUE = { + title: 'Existing Issue', + description: 'New comment', + label: 'location', + }; + + context('when issue is closed', () => { + let setIssueLabelsScope; + let addCommentScope; + let openIssueScope; + + const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = { + iid: 123, + title: ISSUE.title, + description: ISSUE.description, + labels: [{ name: 'selectors' }], + state: GitLab.ISSUE_STATE_CLOSED, + }; + + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const responseIssuereopened = { iid: 123 }; + const responseSetLabels = { + iid: 123, + labels: ['location'], + }; + const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description }; + const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); + + openIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + .reply(200, responseIssuereopened); + + setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + .reply(200, responseSetLabels); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + .reply(200, responseAddcomment); + + await gitlab.createOrUpdateIssue(ISSUE); + }); + + it('reopens the issue', () => { + expect(openIssueScope.isDone()).to.be.true; + }); + + it("updates the issue's label", () => { + expect(setIssueLabelsScope.isDone()).to.be.true; + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + }); + + context('when issue is already opened', () => { + let setIssueLabelsScope; + let addCommentScope; + let openIssueScope; + + const GITLAB_RESPONSE_FOR_EXISTING_ISSUE = { + number: 123, + title: ISSUE.title, + description: ISSUE.description, + labels: [{ name: 'selectors' }], + state: GitLab.ISSUE_STATE_OPEN, + }; + + const EXPECTED_REQUEST_BODY = { state_event: 'reopen' }; + const responseIssuereopened = { iid: 123 }; + const responseSetLabels = { + iid: 123, + labels: ['location'], + }; + const responseAddcomment = { iid: 123, id: 23, body: ISSUE.description }; + const { iid } = GITLAB_RESPONSE_FOR_EXISTING_ISSUE; + + before(async () => { + nock(gitlabApiUrl, reqHeaders) + .get(`/projects/${projectId}/issues?search=${encodeURIComponent(ISSUE.title)}&per_page=100`) + .reply(200, [GITLAB_RESPONSE_FOR_EXISTING_ISSUE]); + + openIssueScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, EXPECTED_REQUEST_BODY) + .reply(200, responseIssuereopened); + + setIssueLabelsScope = nock(gitlabApiUrl, reqHeaders) + .put(`/projects/${projectId}/issues/${iid}`, { labels: ['location'] }) + .reply(200, responseSetLabels); + + addCommentScope = nock(gitlabApiUrl, reqHeaders) + .post(`/projects/${projectId}/issues/${iid}/notes`, { body: ISSUE.description }) + .reply(200, responseAddcomment); + + await gitlab.createOrUpdateIssue(ISSUE); + }); + + it('does not change the issue state', () => { + expect(openIssueScope.isDone()).to.be.false; + }); + + it("updates the issue's label", () => { + expect(setIssueLabelsScope.isDone()).to.be.true; + }); + + it('adds comment to the issue', () => { + expect(addCommentScope.isDone()).to.be.true; + }); + }); + }); + }); +}); diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js index d0917607c..be5161188 100644 --- a/src/reporterGitlab/index.js +++ b/src/reporterGitlab/index.js @@ -6,7 +6,7 @@ import GitLab from './gitlab.js'; const CONTRIBUTION_TOOL_URL = 'https://contribute.opentermsarchive.org/'; const DOC_URL = 'https://docs.opentermsarchive.org'; -const REPO_URL = 'https://gitlab.com/'; +const REPO_URL = process.env.OTA_ENGINE_GITLAB_BASE_URL; const ERROR_MESSAGE_TO_ISSUE_LABEL_MAP = { 'has no match': 'selectors', diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index 3a0c78763..a1e61f2d8 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -20,7 +20,7 @@ describe('Reporter GitLab labels', () => { expect(descriptionLength).to.be.lessThan(GITLAB_LABEL_DESCRIPTION_MAX_LENGTH); }); - it('complies with the GitHub constraints for color', () => { + it('complies with the GitLab constraints for color', () => { const validHexColorRegex = /^#[0-9a-fA-F]{6}$/; // Regex for a valid 6-digit hexadecimal color code with the `#` expect(validHexColorRegex.test(label.color)).to.be.true; From ec13bb0a94f60baf655ab065ee818d37676bd90f Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Thu, 12 Sep 2024 11:15:13 +0200 Subject: [PATCH 04/11] remove existing GitLab repository in the config remove the GitLab releases repository with a new one used as an example --- config/default.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/default.json b/config/default.json index 9009d52a6..8102f2221 100644 --- a/config/default.json +++ b/config/default.json @@ -59,7 +59,7 @@ "dataset": { "title": "sandbox", "versionsRepositoryURL": "https://github.com/OpenTermsArchive/sandbox", - "versionsRepositoryURLGitLab": "https://gitlab.com/p2b/contrib-versions", + "versionsRepositoryURLGitLab": "https://gitlab.com/ota-sandbox-example/sandbox", "publishingSchedule": "30 8 * * MON" } } From 12005fb321ed8ad017f69e85b9cf7ebc84f06062 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 8 Oct 2024 14:33:44 +0200 Subject: [PATCH 05/11] add EUPL-1.2 copyright --- scripts/dataset/assets/README.templateGitLab.js | 16 ++++++++++++++++ scripts/dataset/publishGitLab/index.js | 16 ++++++++++++++++ src/reporterGitlab/gitlab.js | 16 ++++++++++++++++ src/reporterGitlab/gitlab.test.js | 16 ++++++++++++++++ src/reporterGitlab/index.js | 16 ++++++++++++++++ src/reporterGitlab/labels.test.js | 16 ++++++++++++++++ 6 files changed, 96 insertions(+) diff --git a/scripts/dataset/assets/README.templateGitLab.js b/scripts/dataset/assets/README.templateGitLab.js index c9f0bb8d2..ef8361e67 100644 --- a/scripts/dataset/assets/README.templateGitLab.js +++ b/scripts/dataset/assets/README.templateGitLab.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import config from 'config'; const LOCALE = 'en-EN'; diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index c752c1abe..2f2f7f9b7 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import fsApi from 'fs'; import path from 'path'; import url from 'url'; diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index 392b5b481..68a638100 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import { createRequire } from 'module'; import HttpProxyAgent from 'http-proxy-agent'; diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js index ede97cd18..093384ce0 100644 --- a/src/reporterGitlab/gitlab.test.js +++ b/src/reporterGitlab/gitlab.test.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import { createRequire } from 'module'; import { expect } from 'chai'; diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js index be5161188..923d0011c 100644 --- a/src/reporterGitlab/index.js +++ b/src/reporterGitlab/index.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import mime from 'mime'; import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index a1e61f2d8..dd6e8fe59 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -1,3 +1,19 @@ +// Copyright (c) 2024 European Union +// * +// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the +// European Commission – subsequent versions of the EUPL (the “Licence”); +// You may not use this work except in compliance with the Licence. +// You may obtain a copy of the Licence at: +// * +// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 +// * +// Unless required by applicable law or agreed to in writing, software distributed under +// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS +// OF ANY KIND, either express or implied. See the Licence for the specific language +// governing permissions and limitations under the Licence. +// +// EUPL text (EUPL-1.2) + import { createRequire } from 'module'; import chai from 'chai'; From 5fcdc379ae9cb02efb25870b2291730bbd191e48 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Wed, 9 Oct 2024 17:22:41 +0200 Subject: [PATCH 06/11] use only a single template for README remove duplicate readme for gitlab and handle the new input parameter --- scripts/dataset/assets/README.template.js | 8 +- .../dataset/assets/README.templateGitLab.js | 81 ------------------- scripts/dataset/export/index.js | 3 +- scripts/dataset/index.js | 16 +++- scripts/dataset/publishGitLab/index.js | 2 +- 5 files changed, 19 insertions(+), 91 deletions(-) delete mode 100644 scripts/dataset/assets/README.templateGitLab.js diff --git a/scripts/dataset/assets/README.template.js b/scripts/dataset/assets/README.template.js index 1c63d117c..1cda34281 100644 --- a/scripts/dataset/assets/README.template.js +++ b/scripts/dataset/assets/README.template.js @@ -3,10 +3,10 @@ import config from 'config'; const LOCALE = 'en-EN'; const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; -export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { +export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { return `# Open Terms Archive — ${title({ releaseDate })} -${body({ servicesCount, firstVersionDate, lastVersionDate })}`; +${body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL })}`; } export function title({ releaseDate }) { @@ -17,12 +17,10 @@ export function title({ releaseDate }) { return `${title} — ${releaseDate} dataset`; } -export function body({ servicesCount, firstVersionDate, lastVersionDate }) { +export function body({ servicesCount, firstVersionDate, lastVersionDate, versionsRepositoryURL }) { firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - const versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); - return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURL}](${versionsRepositoryURL}). diff --git a/scripts/dataset/assets/README.templateGitLab.js b/scripts/dataset/assets/README.templateGitLab.js deleted file mode 100644 index ef8361e67..000000000 --- a/scripts/dataset/assets/README.templateGitLab.js +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - -import config from 'config'; - -const LOCALE = 'en-EN'; -const DATE_OPTIONS = { year: 'numeric', month: 'long', day: 'numeric' }; - -export default function readme({ releaseDate, servicesCount, firstVersionDate, lastVersionDate }) { - return `# Open Terms Archive — ${title({ releaseDate })} - -${body({ servicesCount, firstVersionDate, lastVersionDate })}`; -} - -export function title({ releaseDate }) { - releaseDate = releaseDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - - const title = config.get('@opentermsarchive/engine.dataset.title'); - - return `${title} — ${releaseDate} dataset`; -} - -export function body({ servicesCount, firstVersionDate, lastVersionDate }) { - firstVersionDate = firstVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - lastVersionDate = lastVersionDate.toLocaleDateString(LOCALE, DATE_OPTIONS); - - const versionsRepositoryURLGitLab = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); - - return `This dataset consolidates the contractual documents of ${servicesCount} service providers, in all their versions that were accessible online between ${firstVersionDate} and ${lastVersionDate}. - -This dataset is tailored for datascientists and other analysts. You can also explore all these versions interactively on [${versionsRepositoryURLGitLab}](${versionsRepositoryURLGitLab}). - -It has been generated with [Open Terms Archive](https://opentermsarchive.org). - -### Dataset format - -This dataset represents each version of a document as a separate [Markdown](https://spec.commonmark.org/0.30/) file, nested in a directory with the name of the service provider and in a directory with the name of the terms type. The filesystem layout will look like below. - -\`\`\` -├ README.md -├┬ Service provider 1 (e.g. Facebook) -│├┬ Terms type 1 (e.g. Terms of Service) -││├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-08-01T01-03-12Z.md) -┆┆┆ -││└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-10-03T08-12-25Z.md) -┆┆ -│└┬ Terms type X (e.g. Privacy Policy) -│ ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) -┆ ┆ -│ └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) -┆ -└┬ Service provider Y (e.g. Google) - ├┬ Terms type 1 (e.g. Developer Terms) - │├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2019-03-12T04-18-22Z.md) - ┆┆ - │└ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-12-04T22-47-05Z.md) - └┬ Terms type Z (e.g. Privacy Policy) - ┆ - ├ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-05-02T03-02-15Z.md) - ┆ - └ YYYY-DD-MMTHH-MM-SSZ.md (e.g. 2021-11-14T12-36-45Z.md) -\`\`\` - -### License - -This dataset is made available under an [Open Database (OdBL) License](https://opendatacommons.org/licenses/odbl/1.0/) by Open Terms Archive Contributors. -`; -} diff --git a/scripts/dataset/export/index.js b/scripts/dataset/export/index.js index bc0a028ab..3215768c1 100644 --- a/scripts/dataset/export/index.js +++ b/scripts/dataset/export/index.js @@ -16,7 +16,7 @@ const fs = fsApi.promises; const ARCHIVE_FORMAT = 'zip'; // for supported formats, see https://www.archiverjs.com/docs/archive-formats -export default async function generate({ archivePath, releaseDate }) { +export default async function generate({ archivePath, releaseDate, versionsRepositoryURL }) { const versionsRepository = await RepositoryFactory.create(config.get('@opentermsarchive/engine.recorder.versions.storage')).initialize(); const archive = await initializeArchive(archivePath); @@ -61,6 +61,7 @@ export default async function generate({ archivePath, releaseDate }) { releaseDate, firstVersionDate, lastVersionDate, + versionsRepositoryURL, }), { name: `${archive.basename}/README.md` }, ); diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 735ad2d3c..8041813c9 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -15,7 +15,17 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start exporting dataset…'); - const stats = await generateRelease({ archivePath, releaseDate }); + const usesGitHub = (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined'); + const usesGitLab = (typeof process.env.OTA_ENGINE_GITLAB_TOKEN !== 'undefined'); + + let versionsRepositoryURL = ''; + + if (usesGitHub) + versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); + if (usesGitLab) + versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + const stats = await generateRelease({ archivePath, releaseDate, versionsRepositoryURL }); logger.info(`Dataset exported in ${archivePath}`); @@ -25,7 +35,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info('Start publishing dataset…'); - if (typeof process.env.OTA_ENGINE_GITHUB_TOKEN !== 'undefined') { + if (usesGitHub) { const releaseUrl = await publishRelease({ archivePath, releaseDate, @@ -35,7 +45,7 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } logger.info(`Dataset published to ${releaseUrl}`); } - if (typeof process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN !== 'undefined') { + if (usesGitLab) { const releaseUrl = await publishReleaseGitLab({ archivePath, releaseDate, diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 2f2f7f9b7..e2a46b653 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -24,7 +24,7 @@ import FormData from 'form-data'; import nodeFetch from 'node-fetch'; import GitLab from '../../../src/reporterGitlab/gitlab.js'; -import * as readme from '../assets/README.templateGitLab.js'; +import * as readme from '../assets/README.template.js'; import logger from '../logger/index.js'; dotenv.config(); From aa9ea5be3c4546b2668af76f95cdb54bcf814989 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Mon, 21 Oct 2024 09:35:12 +0200 Subject: [PATCH 07/11] Fix release zip upload not working Fix the release zip workflow using GitLab packages to upload the file --- scripts/dataset/index.js | 8 +++--- scripts/dataset/publishGitLab/index.js | 34 ++++++++++++++++++++------ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/scripts/dataset/index.js b/scripts/dataset/index.js index 8041813c9..66ea0a1f0 100644 --- a/scripts/dataset/index.js +++ b/scripts/dataset/index.js @@ -19,11 +19,9 @@ export async function release({ shouldPublish, shouldRemoveLocalCopy, fileName } const usesGitLab = (typeof process.env.OTA_ENGINE_GITLAB_TOKEN !== 'undefined'); let versionsRepositoryURL = ''; - - if (usesGitHub) - versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); - if (usesGitLab) - versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); + + if (usesGitHub) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURL'); + if (usesGitLab) versionsRepositoryURL = config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab'); const stats = await generateRelease({ archivePath, releaseDate, versionsRepositoryURL }); diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index e2a46b653..b8614b07b 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -30,7 +30,6 @@ import logger from '../logger/index.js'; dotenv.config(); const gitlabAPIUrl = process.env.OTA_ENGINE_GITLAB_API_BASE_URL; -const gitlabUrl = process.env.OTA_ENGINE_GITLAB_BASE_URL; export default async function publishReleaseGitLab({ archivePath, @@ -97,14 +96,36 @@ export default async function publishReleaseGitLab({ logger.info(`Created release with releaseId: ${releaseId}`); - // Then, upload the ZIP file as an asset to the release + // Upload the package + options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); + options.method = 'PUT'; + options.body = fsApi.createReadStream(archivePath); + + // restrict characters to the ones allowed by GitLab APIs + const packageName = config.get('@opentermsarchive/engine.dataset.title').replace(/[^a-zA-Z0-9.\-_]/g, '-'); + const packageVersion = tagName.replace(/[^a-zA-Z0-9.\-_]/g, '-'); + const packageFileName = archivePath.replace(/[^a-zA-Z0-9.\-_/]/g, '-'); + + logger.debug(`packageName: ${packageName}, packageVersion: ${packageVersion} packageFileName: ${packageFileName}`); + + const packageResponse = await nodeFetch( + `${gitlabAPIUrl}/projects/${projectId}/packages/generic/${packageName}/${packageVersion}/${packageFileName}?status=default&select=package_file`, + options, + ); + const packageRes = await packageResponse.json(); + + const packageFilesId = packageRes.id; + + logger.debug(`package file id: ${packageFilesId}`); + + // use the package id to build the download url for the release + const publishedPackageUrl = `${config.get('@opentermsarchive/engine.dataset.versionsRepositoryURLGitLab')}/-/package_files/${packageFilesId}/download`; + + // Create the release and link the package const formData = new FormData(); formData.append('name', archivePath); - formData.append( - 'url', - `${gitlabUrl}/${commonParams.owner}/${commonParams.repo}/-/archive/${tagName}/${archivePath}`, - ); + formData.append('url', publishedPackageUrl); formData.append('file', fsApi.createReadStream(archivePath), { filename: path.basename(archivePath) }); options = GitLab.baseOptionsHttpReq(process.env.OTA_ENGINE_GITLAB_RELEASES_TOKEN); @@ -120,7 +141,6 @@ export default async function publishReleaseGitLab({ options, ); const uploadRes = await uploadResponse.json(); - const releaseUrl = uploadRes.direct_asset_url; return releaseUrl; From 1d8a3bd0d6cdc8f8ec0fbf234acbeb8774ec93e8 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Mon, 21 Oct 2024 10:07:23 +0200 Subject: [PATCH 08/11] Update changelog --- CHANGELOG.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 242c94665..919be8c90 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All changes that impact users of this module are documented in this file, in the [Common Changelog](https://common-changelog.org) format with some additional specifications defined in the CONTRIBUTING file. This codebase adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## Unreleased [minor] + +### Added + +- Add GtiLab functionalities + + ## 2.2.1 - 2024-06-07 _Full changeset and discussions: [#1088](https://github.com/OpenTermsArchive/engine/pull/1088)._ From c7956afdf73bcb723b20185f38308ab0fd4de960 Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 22 Oct 2024 09:33:51 +0200 Subject: [PATCH 09/11] Fix typo on changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e39d24b90..99740d259 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ All changes that impact users of this module are documented in this file, in the ### Added -- Add GtiLab functionalities +- Add GitLab functionalities ## 2.3.0 - 2024-10-21 From dcd4a0378fd3e4a1969f29cc08a4bd050354d13e Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 29 Oct 2024 11:12:43 +0100 Subject: [PATCH 10/11] Remove explicit EUPL-1.2 copyright in files --- scripts/dataset/publishGitLab/index.js | 16 ---------------- src/reporterGitlab/gitlab.js | 16 ---------------- src/reporterGitlab/gitlab.test.js | 16 ---------------- src/reporterGitlab/index.js | 16 ---------------- src/reporterGitlab/labels.test.js | 16 ---------------- 5 files changed, 80 deletions(-) diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index b8614b07b..16f08bfc4 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import fsApi from 'fs'; import path from 'path'; import url from 'url'; diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index 68a638100..392b5b481 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import { createRequire } from 'module'; import HttpProxyAgent from 'http-proxy-agent'; diff --git a/src/reporterGitlab/gitlab.test.js b/src/reporterGitlab/gitlab.test.js index 093384ce0..ede97cd18 100644 --- a/src/reporterGitlab/gitlab.test.js +++ b/src/reporterGitlab/gitlab.test.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import { createRequire } from 'module'; import { expect } from 'chai'; diff --git a/src/reporterGitlab/index.js b/src/reporterGitlab/index.js index 923d0011c..be5161188 100644 --- a/src/reporterGitlab/index.js +++ b/src/reporterGitlab/index.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import mime from 'mime'; import { toISODateWithoutMilliseconds } from '../archivist/utils/date.js'; diff --git a/src/reporterGitlab/labels.test.js b/src/reporterGitlab/labels.test.js index dd6e8fe59..a1e61f2d8 100644 --- a/src/reporterGitlab/labels.test.js +++ b/src/reporterGitlab/labels.test.js @@ -1,19 +1,3 @@ -// Copyright (c) 2024 European Union -// * -// Licensed under the EUPL, Version 1.2 or – as soon they will be approved by the -// European Commission – subsequent versions of the EUPL (the “Licence”); -// You may not use this work except in compliance with the Licence. -// You may obtain a copy of the Licence at: -// * -// https://joinup.ec.europa.eu/collection/eupl/eupl-text-eupl-12 -// * -// Unless required by applicable law or agreed to in writing, software distributed under -// the Licence is distributed on an “AS IS” basis, WITHOUT WARRANTIES OR CONDITIONS -// OF ANY KIND, either express or implied. See the Licence for the specific language -// governing permissions and limitations under the Licence. -// -// EUPL text (EUPL-1.2) - import { createRequire } from 'module'; import chai from 'chai'; From 5045281f9f4b9352add2d25189d07b7a091c1ffe Mon Sep 17 00:00:00 2001 From: ZAGO Alessandro Date: Tue, 29 Oct 2024 11:34:12 +0100 Subject: [PATCH 11/11] Align logging messages format with latest releases aligned format and minor wording fixes --- scripts/dataset/publishGitLab/index.js | 2 +- src/reporterGitlab/gitlab.js | 36 +++++++++++++------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/scripts/dataset/publishGitLab/index.js b/scripts/dataset/publishGitLab/index.js index 16f08bfc4..e41c3e1c8 100644 --- a/scripts/dataset/publishGitLab/index.js +++ b/scripts/dataset/publishGitLab/index.js @@ -47,7 +47,7 @@ export default async function publishReleaseGitLab({ projectId = res.id; } catch (error) { - logger.error(`🤖 Error while obtaining projectId: ${error}`); + logger.error(`Error while obtaining projectId: ${error}`); projectId = null; } diff --git a/src/reporterGitlab/gitlab.js b/src/reporterGitlab/gitlab.js index 392b5b481..cf4133a95 100644 --- a/src/reporterGitlab/gitlab.js +++ b/src/reporterGitlab/gitlab.js @@ -40,11 +40,11 @@ export default class GitLab { if (response.ok) { this.projectId = res.id; } else { - logger.error(`🤖 Error while obtaining projectId: ${JSON.strinfigy(res)}`); + logger.error(`Error while obtaining projectId: ${JSON.strinfigy(res)}`); this.projectId = null; } } catch (error) { - logger.error(`🤖 Error while obtaining projectId: ${error}`); + logger.error(`Error while obtaining projectId: ${error}`); this.projectId = null; } this.MANAGED_LABELS = require('./labels.json'); @@ -54,7 +54,7 @@ export default class GitLab { const missingLabels = this.MANAGED_LABELS.filter(label => !existingLabelsNames.includes(label.name)); if (missingLabels.length) { - logger.info(`🤖 Following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); + logger.info(`The following required labels are not present on the repository: ${missingLabels.map(label => `"${label.name}"`).join(', ')}. Creating them…`); for (const label of missingLabels) { await this.createLabel({ /* eslint-disable-line no-await-in-loop */ @@ -79,11 +79,11 @@ export default class GitLab { return res; } - logger.error(`🤖 Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); + logger.error(`Failed to get labels: ${response.status} - ${JSON.stringify(res)}`); return null; } catch (error) { - logger.error(`🤖 Could get labels: ${error}`); + logger.error(`Could not get labels: ${error}`); } } @@ -112,12 +112,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 New label created: ${res.name} , color: ${res.color}`); + logger.info(`New label created: ${res.name} , color: ${res.color}`); } else { logger.error(`createLabel response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Failed to create label: ${error}`); + logger.error(`Failed to create label: ${error}`); } } @@ -146,14 +146,14 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); + logger.info(`Created GitLab issue #${res.iid} "${title}": ${res.web_url}`); return res; } logger.error(`createIssue response: ${JSON.stringify(res)}`); } catch (error) { - logger.error(`🤖 Could not create GitLab issue "${title}": ${error}`); + logger.error(`Could not create GitLab issue "${title}": ${error}`); } } @@ -177,12 +177,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Updated labels to GitLab issue #${issue.iid}`); + logger.info(`Updated labels to GitLab issue #${issue.iid}`); } else { logger.error(`setIssueLabels response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -205,12 +205,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Opened GitLab issue #${res.iid}`); + logger.info(`Opened GitLab issue #${res.iid}`); } else { logger.error(`openIssue response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -234,12 +234,12 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Closed GitLab issue #${issue.iid}`); + logger.info(`Closed GitLab issue #${issue.iid}`); } else { logger.error(`closeIssue response: ${JSON.stringify(res)}`); } } catch (error) { - logger.error(`🤖 Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not update GitLab issue #${issue.iid} "${issue.title}": ${error}`); } } @@ -268,7 +268,7 @@ export default class GitLab { logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { - logger.error(`🤖 Could not find GitLab issue "${title}": ${error}`); + logger.error(`Could not find GitLab issue "${title}": ${error}`); } } @@ -292,14 +292,14 @@ export default class GitLab { const res = await response.json(); if (response.ok) { - logger.info(`🤖 Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); + logger.info(`Added comment to GitLab issue #${issue.iid} ${issue.title}: ${res.id}`); return res.body; } logger.error(`openIssue response: ${JSON.stringify(res)}`); } catch (error) { - logger.error(`🤖 Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); + logger.error(`Could not add comment to GitLab issue #${issue.iid} "${issue.title}": ${error}`); } }