From 5ece08f6d144a17f7a3a27da4d266e6f7d3ba7ff Mon Sep 17 00:00:00 2001 From: mozafar Date: Fri, 17 Apr 2020 19:26:57 +0100 Subject: [PATCH] Add validate-new-release command to check the diff of a plugins.xml file checks the diff of plugins.xml (in a PR for example), extracts the data of the new plugins, downloading their package file and validating the checksum --- README.md | 6 ++- index.js | 14 +++--- package-lock.json | 21 ++++++++ package.json | 1 + src/help.js | 6 +++ src/validate-new-release/checkSumFile.js | 14 ++++++ src/validate-new-release/download-package.js | 19 +++++++ .../extract-release-data.js | 20 ++++++++ src/validate-new-release/index.js | 50 +++++++++++++++++++ 9 files changed, 142 insertions(+), 9 deletions(-) create mode 100644 src/validate-new-release/checkSumFile.js create mode 100644 src/validate-new-release/download-package.js create mode 100644 src/validate-new-release/extract-release-data.js create mode 100644 src/validate-new-release/index.js diff --git a/README.md b/README.md index a10d85c..4c2a7b7 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,11 @@ A set of tools by the [Public Knowledge Project](https://docs.pkp.sfu.ca/) to he ### pkp-plugin validate-releases --input ./plugins.xml -(internal tool) used by the CI job for plugins registry to extract the releases from `plugins.xml` and validate their MD5 hashes +(used internally by CI) extracts all the releases from `plugins.xml` and validate their MD5 hashes + +### pkp-plugin validate-new-release + +(used internally by CI) validates the new releases added to a plugin.xml file (used internally by the CI tool) ### pkp-plugin version diff --git a/index.js b/index.js index 93d097b..340ecb1 100644 --- a/index.js +++ b/index.js @@ -3,6 +3,7 @@ const minimist = require('minimist') const help = require('./src/help') const version = require('./src/version') const validateAllReleases = require('./src/validate-all-releases') +const validateNewRelease = require('./src/validate-new-release') const start = async args => { const params = minimist(args) @@ -17,17 +18,14 @@ const start = async args => { } switch (cmd) { + case 'validate-new-release': + return validateNewRelease(params) case 'validate-releases': - case 'validatereleases': - case 'validateReleases': - validateAllReleases(params) - break + return validateAllReleases(params) case 'version': - version(params) - break + return version(params) case 'help': - help(params) - break + return help(params) default: console.error(`"${cmd}" is not a valid command!`) break diff --git a/package-lock.json b/package-lock.json index 6d42c4c..679e84c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -59,6 +59,22 @@ "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=" }, + "debug": { + "version": "3.2.6", + "resolved": "https://registry.npmjs.org/debug/-/debug-3.2.6.tgz", + "integrity": "sha512-mel+jf7nrtEl5Pn1Qx46zARXKDpBbvzezse7p7LqINmdoIk8PYP5SySaxEmYv6TZ0JyEKA1hsCId6DIhgITtWQ==", + "requires": { + "ms": "^2.1.1" + } + }, + "follow-redirects": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.11.0.tgz", + "integrity": "sha512-KZm0V+ll8PfBrKwMzdo5D13b1bur9Iq9Zd/RMmAoQQcl2PxxFml8cxXPaaPYVbV0RjNjq1CU7zIzAOqtUPudmA==", + "requires": { + "debug": "^3.0.0" + } + }, "fs.realpath": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", @@ -114,6 +130,11 @@ "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" }, + "ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, "once": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", diff --git a/package.json b/package.json index 4faab32..6c8fe10 100644 --- a/package.json +++ b/package.json @@ -14,6 +14,7 @@ "license": "ISC", "dependencies": { "chalk": "^4.0.0", + "follow-redirects": "^1.11.0", "minimist": "^1.2.5", "shelljs": "^0.8.3", "xml2js": "^0.4.23" diff --git a/src/help.js b/src/help.js index 4a36ba4..b72ca14 100644 --- a/src/help.js +++ b/src/help.js @@ -9,4 +9,10 @@ module.exports = args => { ' extracts all releases for all plugins from a plugins.xml file and validates their MD5 checksum (used internally by the CI tool)' ) ) + console.log( + chalk.white( + chalk.green('validate-new-release'), + ' validates the new releases added to a plugin.xml file (used internally by the CI tool)' + ) + ) } diff --git a/src/validate-new-release/checkSumFile.js b/src/validate-new-release/checkSumFile.js new file mode 100644 index 0000000..940a4a6 --- /dev/null +++ b/src/validate-new-release/checkSumFile.js @@ -0,0 +1,14 @@ +const crypto = require('crypto') +const fs = require('fs') +/** + * Calculates the md5 check sum of a file + */ +module.exports = function checksumFile (path, hashName = 'md5') { + return new Promise((resolve, reject) => { + const hash = crypto.createHash(hashName) + const stream = fs.createReadStream(path) + stream.on('error', err => reject(err)) + stream.on('data', chunk => hash.update(chunk)) + stream.on('end', () => resolve(hash.digest('hex'))) + }) +} diff --git a/src/validate-new-release/download-package.js b/src/validate-new-release/download-package.js new file mode 100644 index 0000000..e82ed16 --- /dev/null +++ b/src/validate-new-release/download-package.js @@ -0,0 +1,19 @@ +const https = require('follow-redirects').https +const fs = require('fs') + +/** + * Downloads a file from the internet following the redirect (which is needed for Github links) + */ +module.exports = (package, fileName) => { + return new Promise((resolve, reject) => { + const file = fs.createWriteStream(fileName) + const req = https.get(package, function (response) { + response.pipe(file) + response.on('end', resolve) + }) + req.on('error', function (err) { + console.log('Request error: ' + err.message) + reject(err) + }) + }) +} diff --git a/src/validate-new-release/extract-release-data.js b/src/validate-new-release/extract-release-data.js new file mode 100644 index 0000000..b1d6a29 --- /dev/null +++ b/src/validate-new-release/extract-release-data.js @@ -0,0 +1,20 @@ +/** + * scraps a fragment of xml to get the package url, MD5 etc.. + * @param {string} lines the diffed lines showing what changed in a plugins.xml file + */ +const extractReleaseData = lines => { + const { + groups: { package } + } = /(?[^ $]*)<\/package>/g.exec(lines) + + const { + groups: { md5 } + } = /md5=\"(?[^ $]*)\"/g.exec(lines) + + return { + md5, + package + } +} + +module.exports = extractReleaseData diff --git a/src/validate-new-release/index.js b/src/validate-new-release/index.js new file mode 100644 index 0000000..e33f24a --- /dev/null +++ b/src/validate-new-release/index.js @@ -0,0 +1,50 @@ +const shell = require('shelljs') +const chalk = require('chalk') +const downloadPackage = require('./download-package') +const extractReleaseData = require('./extract-release-data') +const checksumFile = require('./checkSumFile') + +module.exports = async args => { + console.log(chalk.blueBright(`Checking new release`)) + const diffFile = __dirname + '/diff.temp' + const command = shell.exec(`git diff origin/master | grep ^+[^+]`) + const changedLines = command.stdout + + // Extract release, md5 data from diff + const { package: packageUrl, md5: expectedMD5 } = extractReleaseData( + changedLines + ) + console.log(chalk.blueBright(`Package url: ${packageUrl}`)) + console.log(chalk.blueBright(`MD5: ${expectedMD5}`)) + + // Downlaod package + const downloadedFileName = __dirname + '/downloaded_package.tar.gz' + await downloadPackage(packageUrl, downloadedFileName) + + // Calculate MD5 of downloaded file + const md5 = await checksumFile(downloadedFileName) + + if (md5 === expectedMD5) { + console.log(chalk.greenBright(`The MD5 of the downloaded file is correct`)) + } else { + console.log( + chalk.redBright( + `The MD5 of the downloaded is incorrect. Expected ${expectedMD5}, Actual ${md5}` + ) + ) + shell.rm(downloadedFileName) + shell.exit(1) + } + + console.log(chalk.blueBright(`MD5 of downloaded package: ${md5}`)) + + // check contents of tar file + shell.exec(`tar -tf ${downloadedFileName}`) // ToDO .. check contents of the tar file + + shell.rm(downloadedFileName) + + if (command.code !== 0) { + shell.echo('Error: checking new release failed') + shell.exit(1) + } +}