diff --git a/.eslintignore b/.eslintignore new file mode 100644 index 0000000..40b878d --- /dev/null +++ b/.eslintignore @@ -0,0 +1 @@ +node_modules/ \ No newline at end of file diff --git a/.eslintrc.yml b/.eslintrc.yml new file mode 100644 index 0000000..3c8694b --- /dev/null +++ b/.eslintrc.yml @@ -0,0 +1,30 @@ +env: + node: true + es6: true + +parserOptions: + ecmaVersion: 2017 + +rules: + quotes: [2, double] + no-trailing-spaces: 2 + comma-spacing: [2, { before: false, after: true }] + eqeqeq: [2, always] + linebreak-style: [2, unix] + semi: [2, always] + max-len: [2, 80] + no-multi-spaces: 2 + no-useless-escape: 2 + array-bracket-spacing: [2, never] + brace-style: 2 + capitalized-comments: [2, always, { ignoreConsecutiveComments: true }] + eol-last: [2, always] + func-call-spacing: [2, never] + semi-spacing: 2 + strict: 2 + no-var: 2 + no-console: "off" + curly: 2 + indent: ["error", 2] + +extends: eslint:recommended diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..731b39c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,5 @@ +# Checklist + +- [ ] Test cases pass +- [ ] Relevant documentation has been updated +- [ ] Change has been considered for a tweet/blog diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..af2429c --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +node_modules/ +.vscode +package-lock.json diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..30ae9fc --- /dev/null +++ b/.travis.yml @@ -0,0 +1,8 @@ +language: node_js +node_js: "8" +os: linux +before_install: +# Install NVS. +- git clone --depth 1 https://github.com/jasongin/nvs ~/.nvs +- . ~/.nvs/nvs.sh +- nvs --version diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..cd482d8 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, +and distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by +the copyright owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all +other entities that control, are controlled by, or are under common +control with that entity. For the purposes of this definition, +"control" means (i) the power, direct or indirect, to cause the +direction or management of such entity, whether by contract or +otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity +exercising permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, +including but not limited to software source code, documentation +source, and configuration files. + +"Object" form shall mean any form resulting from mechanical +transformation or translation of a Source form, including but +not limited to compiled object code, generated documentation, +and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or +Object form, made available under the License, as indicated by a +copyright notice that is included in or attached to the work +(an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object +form, that is based on (or derived from) the Work and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. For the purposes +of this License, Derivative Works shall not include works that remain +separable from, or merely link (or bind by name) to the interfaces of, +the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including +the original version of the Work and any modifications or additions +to that Work or Derivative Works thereof, that is intentionally +submitted to Licensor for inclusion in the Work by the copyright owner +or by an individual or Legal Entity authorized to submit on behalf of +the copyright owner. For the purposes of this definition, "submitted" +means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, +and issue tracking systems that are managed by, or on behalf of, the +Licensor for the purpose of discussing and improving the Work, but +excluding communication that is conspicuously marked or otherwise +designated in writing by the copyright owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity +on behalf of whom a Contribution has been received by Licensor and +subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the +Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of +this License, each Contributor hereby grants to You a perpetual, +worldwide, non-exclusive, no-charge, royalty-free, irrevocable +(except as stated in this section) patent license to make, have made, +use, offer to sell, sell, import, and otherwise transfer the Work, +where such license applies only to those patent claims licensable +by such Contributor that are necessarily infringed by their +Contribution(s) alone or by combination of their Contribution(s) +with the Work to which such Contribution(s) was submitted. If You +institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work +or a Contribution incorporated within the Work constitutes direct +or contributory patent infringement, then any patent licenses +granted to You under this License for that Work shall terminate +as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the +Work or Derivative Works thereof in any medium, with or without +modifications, and in Source or Object form, provided that You +meet the following conditions: + +(a) You must give any other recipients of the Work or +Derivative Works a copy of this License; and + +(b) You must cause any modified files to carry prominent notices +stating that You changed the files; and + +(c) You must retain, in the Source form of any Derivative Works +that You distribute, all copyright, patent, trademark, and +attribution notices from the Source form of the Work, +excluding those notices that do not pertain to any part of +the Derivative Works; and + +(d) If the Work includes a "NOTICE" text file as part of its +distribution, then any Derivative Works that You distribute must +include a readable copy of the attribution notices contained +within such NOTICE file, excluding those notices that do not +pertain to any part of the Derivative Works, in at least one +of the following places: within a NOTICE text file distributed +as part of the Derivative Works; within the Source form or +documentation, if provided along with the Derivative Works; or, +within a display generated by the Derivative Works, if and +wherever such third-party notices normally appear. The contents +of the NOTICE file are for informational purposes only and +do not modify the License. You may add Your own attribution +notices within Derivative Works that You distribute, alongside +or as an addendum to the NOTICE text from the Work, provided +that such additional attribution notices cannot be construed +as modifying the License. + +You may add Your own copyright statement to Your modifications and +may provide additional or different license terms and conditions +for use, reproduction, or distribution of Your modifications, or +for any such Derivative Works as a whole, provided Your use, +reproduction, and distribution of the Work otherwise complies with +the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, +any Contribution intentionally submitted for inclusion in the Work +by You to the Licensor shall be under the terms and conditions of +this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify +the terms of any separate license agreement you may have executed +with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade +names, trademarks, service marks, or product names of the Licensor, +except as required for reasonable and customary use in describing the +origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or +agreed to in writing, Licensor provides the Work (and each +Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +implied, including, without limitation, any warranties or conditions +of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A +PARTICULAR PURPOSE. You are solely responsible for determining the +appropriateness of using or redistributing the Work and assume any +risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, +whether in tort (including negligence), contract, or otherwise, +unless required by applicable law (such as deliberate and grossly +negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, +incidental, or consequential damages of any character arising as a +result of this License or out of the use or inability to use the +Work (including but not limited to damages for loss of goodwill, +work stoppage, computer failure or malfunction, or any and all +other commercial damages or losses), even if such Contributor +has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing +the Work or Derivative Works thereof, You may choose to offer, +and charge a fee for, acceptance of support, warranty, indemnity, +or other liability obligations and/or rights consistent with this +License. However, in accepting such obligations, You may act only +on Your own behalf and on Your sole responsibility, not on behalf +of any other Contributor, and only if You agree to indemnify, +defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason +of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + +To apply the Apache License to your work, attach the following +boilerplate notice, with the fields enclosed by brackets "[]" +replaced with your own identifying information. (Don't include +the brackets!) The text should be enclosed in the appropriate +comment syntax for the file format. We also recommend that a +file or class name and description of purpose be included on the +same "printed page" as the copyright notice for easier +identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/README.md b/README.md new file mode 100644 index 0000000..1d97b7d --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# Module Curation CI + +[![Build Status](https://travis-ci.org/CloudNativeJS/module-insights-ci.svg?branch=master)](https://travis-ci.org/CloudNativeJS/module-insights-ci) diff --git a/citgm-check.sh b/citgm-check.sh new file mode 100755 index 0000000..a8a2656 --- /dev/null +++ b/citgm-check.sh @@ -0,0 +1,73 @@ +#! /bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +#Cleanup npm log files +rm -rf /tmp/npm-* +### Use CitGM to run custom-citgm-test.js on a module. ### + +echo -e "\n\nParameters:\n +WORKSPACE: $WORKSPACE\n +NODE_VERSION: $NODE_VERSION\n +OS: $OS\n +ARCH: $ARCH\n +MODULE: $MODULE\n +MODULE_VERSION: $MODULE_VERSION\n\n" + +[ -z "$NODE_VERSION" ] && echo "NODE_VERSION is undefined, bailing..." && exit 1 +[ -z "$MODULE_VERSION" ] && echo "MODULE_VERSION is undefined, bailing..." && exit 1 +[ -z "$OS" ] && echo "OS is undefined, bailing..." && exit 1 +[ -z "$ARCH" ] && echo "ARCH is undefined, bailing..." && exit 1 +[ -z "$MODULE" ] && echo "MODULE is undefined, bailing..." && exit 1 +[ -z "$CITGM_LOGLEVEL" ] && CITGM_LOGLEVEL=info + +nvs add $NODE_VERSION >/dev/null +nvs use $NODE_VERSION >/dev/null + +npm install -g citgm nyc + +set +e +citgm --verbose "$CITGM_LOGLEVEL" \ + --tap "$WORKSPACE/$MODULE.tap" \ + --customTest "$WORKSPACE/custom-citgm-test.js" \ + --lookup "$WORKSPACE/lookup.json" "$MODULE@$MODULE_VERSION" +EXIT_STATUS="$?" + +if [ "$EXIT_STATUS" != "0" ]; then + citgm --verbose "$CITGM_LOGLEVEL" \ + --tap "$WORKSPACE/$MODULE.tap" \ + --lookup "$WORKSPACE/lookup.json" "$MODULE@$MODULE_VERSION" + EXIT_STATUS="$?" +fi +set -e + +export EXIT_STATUS + +nvs add 8 >/dev/null +nvs use 8 >/dev/null +npm install >/dev/null + +export TOKEN=$(node fetch-token.js) + +node "$WORKSPACE/update-test.js" +if [ $EXIT_STATUS -eq 1 ]; then + node storing-logs.js +fi +node logout.js +#Cleanup npm log files +rm -rf /tmp/npm-* diff --git a/clean-data.js b/clean-data.js new file mode 100644 index 0000000..77ef426 --- /dev/null +++ b/clean-data.js @@ -0,0 +1,171 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); + +const HOST = process.env.HOST; +const TOKEN = process.env.TOKEN; + +const LOOKUP = require("./lookup.json"); + +function getReq (url, filterList) { + return request.get({ + uri: url, + headers: { + filter: JSON.stringify(filterList) || null + }, + followAllRedirects: true + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + process.exit(1); + } else { + return body; + } + }); +} + +function deleteReq(url) { + return request.del({ + uri: url, + headers: { + Authorization: TOKEN + }, + followAllRedirects: true, + json: true + }, (err, res) => { + if (err) { + return process.exit(1); + + } else { + return res.statusCode; + } + }); +} + +// ID's to delete +let modules = new Set(); +let moduleVersions = new Set(); +let moduleVersionTests = new Set(); +let moduleRecommendedLTSs = new Set(); + +function runDeletes () { + console.log(`Deleting ${moduleVersions.size} modules`); + modules.forEach((id) => { + deleteReq(`http://${HOST}/api/Modules/${id}`); + }); + console.log(`Deleting ${moduleVersions.size} orphan module versions`); + moduleVersions.forEach((id) => { + deleteReq(`http://${HOST}/api/ModuleVersions/${id}`); + }); + console.log(`Deleting ${moduleVersionTests.size} ` + + "orphan module version tests"); + moduleVersionTests.forEach((id) => { + deleteReq(`http://${HOST}/api/ModuleVersionTests/${id}`); + }); + console.log(`Deleting ${moduleRecommendedLTSs.size} ` + + "orphan module recommended LTSs"); + moduleRecommendedLTSs.forEach((id) => { + deleteReq(`http://${HOST}/api/ModuleRecommendedLTSs/${id}`); + }); +} + +const filter = { + "include": [ + {"ModuleVersion": "Module"}, + "Architecture", + "Distribution", + "OperatingSystem" + ] +}; + +async function cleanup () { + const MVTS = JSON.parse(await getReq(`http://${HOST}/api/ModuleVersionTests`, + filter)); + for (let mvt of MVTS) { + // Check ModuleVersion and Module + if (!mvt.ModuleVersion) { + if (!moduleVersionTests.has(mvt.id)) { + moduleVersionTests.add(mvt.id); + } + } else { + if (!mvt.ModuleVersion.Module) { + if (!moduleVersionTests.has(mvt.id)) { + moduleVersionTests.add(mvt.id); + } + if (!moduleVersions.has(mvt.ModuleVersion.id)) { + moduleVersions.add(mvt.ModuleVersion.id); + } + } + } + // Check Architecture + if (!mvt.Architecture) { + if (!moduleVersionTests.has(mvt.id)) { + moduleVersionTests.add(mvt.id); + } + } + // Check Distribution + if (!mvt.Distribution) { + if (!moduleVersionTests.has(mvt.id)) { + moduleVersionTests.add(mvt.id); + } + } + // Check OperatingSystem + if (!mvt.OperatingSystem) { + if (!moduleVersionTests.has(mvt.id)) { + moduleVersionTests.add(mvt.id); + } + } + } + + // Find orphan ModuleVersions + const MODULE_VERSIONS = JSON.parse( + await getReq(`http://${HOST}/api/ModuleVersions`, {"include": "Module"}) + ); + + for (let moduleVersion of MODULE_VERSIONS) { + if (!moduleVersion.Module) { + if (!moduleVersions.has(moduleVersion.id)) { + moduleVersions.add(moduleVersion.id); + } + } + } + + const MODULE_REC_LTS = JSON.parse( + await getReq(`http://${HOST}/api/ModuleRecommendedLTSs`, + {"include":"Module"})); + for (let mrlts of MODULE_REC_LTS) { + // Check Module + if (!mrlts.Module) { + if (!moduleRecommendedLTSs.has(mrlts.id)) { + moduleRecommendedLTSs.add(mrlts.id); + } + } + } + + const MODULES = JSON.parse( + await getReq(`http://${HOST}/api/Modules`)); + MODULES.forEach(module => { + if (!LOOKUP[module.name]) { + if (!modules.has(module.id)) { + modules.add(module.id); + } + } + }); + runDeletes(); +} + +cleanup(); diff --git a/clean-data.sh b/clean-data.sh new file mode 100755 index 0000000..c02091d --- /dev/null +++ b/clean-data.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +nvs add 8 +nvs use 8 + +npm install +export TOKEN=$(node fetch-token.js) + +node clean-data.js +node logout.js diff --git a/custom-citgm-test.js b/custom-citgm-test.js new file mode 100644 index 0000000..dbbc266 --- /dev/null +++ b/custom-citgm-test.js @@ -0,0 +1,27 @@ +"use strict"; + +/*************************************************************************** +* +* (C) Copyright IBM Corp. 2018 +* +* This program and the accompanying materials are made available +* under the terms of the Apache License v2.0 which accompanies +* this distribution. +* +* The Apache License v2.0 is available at +* http://www.opensource.org/licenses/apache2.0.php +* +* Contributors: +* Multiple authors (IBM Corp.) - initial implementation and documentation +***************************************************************************/ + +const { spawnSync } = require("child_process"); +const packageName = require(process.cwd() + "/package.json").name; + +const nyc = (process.env.OS === "win") ? "nyc.cmd" : "nyc"; + +const result = spawnSync(nyc, ["--reporter=json-summary", + `--report-dir=${process.env.WORKSPACE}/${packageName}`, "npm", "test" +]); + +process.exit(result.status); diff --git a/custom-license-test.js b/custom-license-test.js new file mode 100644 index 0000000..7245d13 --- /dev/null +++ b/custom-license-test.js @@ -0,0 +1,44 @@ +"use strict"; + +/*************************************************************************** +* +* (C) Copyright IBM Corp. 2018 +* +* This program and the accompanying materials are made available +* under the terms of the Apache License v2.0 which accompanies +* this distribution. +* +* The Apache License v2.0 is available at +* http://www.opensource.org/licenses/apache2.0.php +* +* Contributors: +* Multiple authors (IBM Corp.) - initial implementation and documentation +***************************************************************************/ + +const fs = require("fs"); +const path = require("path"); +const { spawnSync } = require("child_process"); +const packageName = require(path.join(process.cwd(), + "/package.json")).name.replace(/\//, "%2F"); + +const dir = path.join(process.env.WORKSPACE, packageName); +const checker = require(path.join(process.env.WORKSPACE, + "deps", "node_modules", "license-checker")); + +spawnSync("npm", ["prune", "--production"]); + +// Generate License Report +checker.init({start: process.cwd()}, function (err, json) { + if (err) { + throw err; + } + try { + fs.mkdirSync(dir); + } catch (err) { + if (err.code !== "EEXIST") { + throw err; + } + } + fs.writeFileSync(path.join(dir, "license.json"), + JSON.stringify(json, null, 2)); +}); diff --git a/failed-test.js b/failed-test.js new file mode 100755 index 0000000..add935b --- /dev/null +++ b/failed-test.js @@ -0,0 +1,67 @@ +"use strict"; + +/*************************************************************************** +* +* (C) Copyright IBM Corp. 2018 +* +* This program and the accompanying materials are made available +* under the terms of the Apache License v2.0 which accompanies +* this distribution. +* +* The Apache License v2.0 is available at +* http://www.opensource.org/licenses/apache2.0.php +* +* Contributors: +* Multiple authors (IBM Corp.) - initial implementation and documentation +***************************************************************************/ + +const request = require("request-promise"); +const HOST = process.env.HOST; + +function getModuleVersionTests(){ + return request.get({ + uri: `http://${HOST}/api/ModuleVersionTests`, + followAllRedirects: true, + headers: { + "filter": JSON.stringify({ + "include": [ + {"ModuleVersion":"Module"}, + "Architecture", + "Distribution", + "OperatingSystem", + "NodeVersion" + ], + "where": { + "passed": "false" + } + }) + }, + json: true + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } + return body; + }); +} + +async function findFailedTests () { + let mvts = await getModuleVersionTests(); + let params = []; + for (let mvt of mvts) { + if (mvt.ModuleVersion && mvt.ModuleVersion.Module) { + params.push([ + mvt.ModuleVersion.Module.name, + mvt.ModuleVersion.module_version, + mvt.NodeVersion.node_version, + mvt.OperatingSystem.os, + mvt.Architecture.arch, + mvt.Distribution.distro + ]); + } + } + console.log(params.join("%")); +} + +findFailedTests(); diff --git a/failed-test.sh b/failed-test.sh new file mode 100755 index 0000000..5a4c700 --- /dev/null +++ b/failed-test.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +nvs add 8 >/dev/null +nvs use 8 >/dev/null + +npm install >/dev/null +export TOKEN=$(node fetch-token.js) + +node failed-test.js +node logout.js diff --git a/failed.jenkinsfile b/failed.jenkinsfile new file mode 100644 index 0000000..2e9964c --- /dev/null +++ b/failed.jenkinsfile @@ -0,0 +1,100 @@ +#!/usr/bin/env groovy + +/*************************************************************************** +* +* (C) Copyright IBM Corp. 2018 +* +* This program and the accompanying materials are made available +* under the terms of the Apache License v2.0 which accompanies +* this distribution. +* +* The Apache License v2.0 is available at +* http://www.opensource.org/licenses/apache2.0.php +* +* Contributors: +* Multiple authors (IBM Corp.) - initial implementation and documentation +***************************************************************************/ + +properties([ + parameters([ + choice(name: 'CITGM_LOGLEVEL', choices: 'verbose\ninfo\nwarn\nsilly\nerror', defaultValue: 'verbose', description: 'This defines log level for CITGM.'), + string(defaultValue: "refs/heads/master", description: 'The remote portion of the Git refspec to fetch and test PR - refs/pull/PR_NUMBER/head', name: 'GIT_REMOTE_REF'), + string(defaultValue: "module-curation.eu-gb.mybluemix.net", description: 'module-curation-development.stage1.eu-gb.mybluemix.net || module-curation.eu-gb.mybluemix.net || IP:PORT', name: 'HOST') + ]), +]) + +withCredentials([usernamePassword(credentialsId: 'fb878c0f-2ce4-4759-9953-cfc36d59cf9b', + usernameVariable: 'API_USERNAME', passwordVariable: 'PASSWORD')]) { + + stage('PRE-CLEANUP') { + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "API_USERNAME=${env.API_USERNAME}", "PASSWORD=${env.PASSWORD}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]){ + env.citgm_parameters = sh ( + script: "./clean-data.sh" + ) + } + } + } + + stage('MC-CITGM (Test)') { + + /* - Check for new node/module version and add them to the database + - Find missing test results and return params for MC-CITGM */ + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "API_USERNAME=${env.API_USERNAME}", "PASSWORD=${env.PASSWORD}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]){ + env.citgm_parameters = sh ( + returnStdout: true, + script: "./failed-test.sh" + ) + } + } + + println "\n\n${env.citgm_parameters}\n\n" + + def resp = env.citgm_parameters.trim().tokenize('%') // Create an array of parameters + + // Create an array of buildJobs to run in parallel + def buildJobs = [:] + resp.each { str -> + def p = str.tokenize(',') + def module = "${p[0]}", module_version = "${p[1]}", node_version = "${p[2]}" + def os = "${p[3]}", arch = "${p[4]}", distro = "${p[5]}" + def parameters = [] + parameters.push(string(name: 'MODULE', value: module)) + parameters.push(string(name: 'MODULE_VERSION', value: module_version)) + parameters.push(string(name: 'NODE_VERSION', value: node_version)) + parameters.push(string(name: 'HOST', value: params.HOST)) + parameters.push(string(name: 'OSS', value: os)) + parameters.push(string(name: 'ARCHS', value: arch)) + parameters.push(string(name: 'DISTROS', value: distro)) + parameters.push(string(name: 'GIT_REMOTE_REF', value: params.GIT_REMOTE_REF)) + // parameters.push(string(name: 'CITGM_LOGLEVEL', value: params.CITGM_LOGLEVEL)) + buildJobs["$module@$module_version-$node_version($os $arch $distro)"] = { build(job: "MC-CITGM", parameters: parameters, propagate: false) } + } + + def results = parallel(buildJobs) + + def msg = '\nRESULTS:\n' + results.each { res -> + msg += "${res.key}: ${results[res.key].result}\n" + currentBuild.result = results[res.key].result + } + println msg + + } + + stage('POST-CLEANUP') { + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "API_USERNAME=${env.API_USERNAME}", "PASSWORD=${env.PASSWORD}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]){ + env.citgm_parameters = sh ( + script: "./clean-data.sh" + ) + } + } + } + + println "\n\n \\(• ◡ •)/ Pipeline COMPLETE \\(• ◡ •)/\n\n" +} diff --git a/fetch-token.js b/fetch-token.js new file mode 100644 index 0000000..515bc10 --- /dev/null +++ b/fetch-token.js @@ -0,0 +1,50 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); +const HOST = process.env.HOST; +const USERNAME = process.env.API_USERNAME; +const PASSWORD = process.env.PASSWORD; + +const DAY = 60 * 60 * 24; // Seconds * Minutes * Hours +async function userLogin() { + async function login() { + return request.post({ + url: `http://${HOST}/api/Users/Login`, + followAllRedirects: true, + body: { + "username": USERNAME, + "password": PASSWORD, + "ttl": DAY + }, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.log("Incorrect Credentials"); + process.exit(1); + } else { + return body; + } + }); + + } + + const TOKEN = await login(); + console.log(TOKEN.id); +} + +userLogin(); diff --git a/license-check.sh b/license-check.sh new file mode 100755 index 0000000..bcd987d --- /dev/null +++ b/license-check.sh @@ -0,0 +1,53 @@ +#! /bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +### Use CitGM to run custom-license-test.js on a module. ### + +echo -e "\n\nParameters:\n +WORKSPACE: $WORKSPACE\n +NODE_VERSION: $NODE_VERSION\n +MODULE: $MODULE\n +MODULE_VERSION: $MODULE_VERSION\n\n" + +[ -z "$NODE_VERSION" ] && echo "NODE_VERSION is undefined, bailing..." && exit 1 +[ -z "$MODULE_VERSION" ] && echo "MODULE_VERSION is undefined, bailing..." && exit 1 +[ -z "$MODULE" ] && echo "MODULE is undefined, bailing..." && exit 1 +[ -z "$CITGM_LOGLEVEL" ] && CITGM_LOGLEVEL=info + +nvs add $NODE_VERSION >/dev/null +nvs use $NODE_VERSION >/dev/null + +npm install -g citgm >/dev/null + +mkdir -p "$WORKSPACE/deps/node_modules" +(cd "$WORKSPACE/deps"; npm install "$WORKSPACE/deps" license-checker) + +citgm --verbose "$CITGM_LOGLEVEL" --customTest "$WORKSPACE/custom-license-test.js" --lookup "$WORKSPACE/lookup.json" "$MODULE@$MODULE_VERSION" +EXIT_STATUS="$?" + +if [ "$EXIT_STATUS" = "0" ]; then + nvs add 8 >/dev/null + nvs use 8 >/dev/null + npm install >/dev/null + export TOKEN=$(node fetch-token.js) + node update-license.js +else + echo -e "\n\nEXIT_STATUS: $EXIT_STATUS.\nCITGM exited with a non-zero exit code.\n\n" + exit 1 +fi diff --git a/logout.js b/logout.js new file mode 100644 index 0000000..fce3b0b --- /dev/null +++ b/logout.js @@ -0,0 +1,37 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); +const HOST = process.env.HOST; +const TOKEN = process.env.TOKEN; + +if (TOKEN === null || TOKEN === "Incorrect Credentials") { + console.log("TOKEN is not defined"); + process.exit(1); +} + +request.post({ + uri: `http://${HOST}/api/Users/logout`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + } +}, (err, res) => { + if (err || res.statusCode !== 204) { + process.exit(1); + } +}); diff --git a/lookup.json b/lookup.json new file mode 100644 index 0000000..8f6cbcd --- /dev/null +++ b/lookup.json @@ -0,0 +1,429 @@ +{ + "@cloudant/cloudant": { + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "@cloudnative/health": { + "npm": true, + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "@cloudnative/health-connect": { + "npm": true, + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "appmetrics": { + "npm": true, + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "appmetrics-dash": { + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "appmetrics-prometheus": { + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "appmetrics-zipkin": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "async": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "bluebird": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "body-parser": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "cfenv": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "cheerio": { + "recommended-major-versions": { + "10.x": "0.x", + "6.x": "0.x", + "8.x": "0.x" + } + }, + "classnames": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "commander": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "compression": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "connect-multiparty": { + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "cookie-parser": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "cors": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "debug": { + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "dotenv": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "6.x", + "6.x": "6.x", + "8.x": "6.x" + } + }, + "errorhandler": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "express": { + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "express-gateway": { + "recommended-major-versions": { + "10.x": "1.x", + "8.x": "1.x" + } + }, + "express-session": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "file-extension": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "file-loader": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "fs-extra": { + "recommended-major-versions": { + "10.x": "7.x", + "6.x": "7.x", + "8.x": "7.x" + } + }, + "glob": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "7.x", + "6.x": "7.x", + "8.x": "7.x" + } + }, + "gulp-util": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "hapi": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "17.x", + "6.x": "16.x", + "8.x": "17.x" + } + }, + "js-yaml": { + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "jsonwebtoken": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "8.x", + "6.x": "8.x", + "8.x": "8.x" + } + }, + "koa": { + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "1.x", + "8.x": "2.x" + } + }, + "lodash": { + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "log4js": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "loopback": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "mime": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "minimist": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "mocha": { + "envVar": { + "CHROME_BIN": "/usr/bin/chromium-browser" + }, + "recommended-major-versions": { + "10.x": "5.x", + "6.x": "5.x", + "8.x": "5.x" + } + }, + "moment": { + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "node-report": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "passport": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "0.x", + "6.x": "0.x", + "8.x": "0.x" + } + }, + "q": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "redis": { + "prefix": "v.", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "request": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "request-promise": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "4.x", + "6.x": "4.x", + "8.x": "4.x" + } + }, + "rimraf": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "semver": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "5.x", + "6.x": "5.x", + "8.x": "5.x" + } + }, + "socket.io": { + "recommended-major-versions": { + "10.x": "2.x", + "6.x": "2.x", + "8.x": "2.x" + } + }, + "strong-supervisor": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "6.x", + "6.x": "6.x", + "8.x": "6.x" + } + }, + "through2": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "underscore": { + "recommended-major-versions": { + "10.x": "1.x", + "6.x": "1.x", + "8.x": "1.x" + } + }, + "uuid": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "winston": { + "recommended-major-versions": { + "10.x": "3.x", + "6.x": "3.x", + "8.x": "3.x" + } + }, + "xhr2": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "0.x", + "6.x": "0.x", + "8.x": "0.x" + } + }, + "yargs": { + "prefix": "v", + "recommended-major-versions": { + "10.x": "12.x", + "6.x": "12.x", + "8.x": "12.x" + } + } +} diff --git a/main.jenkinsfile b/main.jenkinsfile new file mode 100644 index 0000000..7263bde --- /dev/null +++ b/main.jenkinsfile @@ -0,0 +1,145 @@ +#!/usr/bin/env groovy + +/*************************************************************************** +* +* (C) Copyright IBM Corp. 2018 +* +* This program and the accompanying materials are made available +* under the terms of the Apache License v2.0 which accompanies +* this distribution. +* +* The Apache License v2.0 is available at +* http://www.opensource.org/licenses/apache2.0.php +* +* Contributors: +* Multiple authors (IBM Corp.) - initial implementation and documentation +***************************************************************************/ + +properties([ + parameters([ + choice(name: 'CITGM_LOGLEVEL', choices: 'verbose\ninfo\nwarn\nsilly\nerror', defaultValue: 'verbose', description: 'This defines log level for CITGM.'), + string(defaultValue: "module-curation.eu-gb.mybluemix.net", description: 'module-curation-development.stage1.eu-gb.mybluemix.net || module-curation.eu-gb.mybluemix.net || IP:PORT', name: 'HOST'), + string(defaultValue: "refs/heads/master", description: 'The remote portion of the Git refspec to fetch and test PR - refs/pull/PR_NUMBER/head', name: 'GIT_REMOTE_REF'), + string(name: 'STARTFROM', defaultValue: '1', description: '''Where to start from: 1=PRE-CLEANUP, 2=MC-CITGM, 3=MC-License, 4=POST-CLEANUP''') + ]), +]) + +withCredentials([usernamePassword(credentialsId: 'fb878c0f-2ce4-4759-9953-cfc36d59cf9b', + usernameVariable: 'API_USERNAME', passwordVariable: 'PASSWORD')]) { + + def STARTFROM = params.STARTFROM.toInteger() + + stage('PRE-CLEANUP') { if (STARTFROM <= 1) { + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "API_USERNAME=${env.API_USERNAME}", "PASSWORD=${env.PASSWORD}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]){ + env.citgm_parameters = sh ( + script: "./clean-data.sh" + ) + } + } + } } + + stage('MC-CITGM (Test + Coverage)') { if (STARTFROM <= 2) { + + /* - Check for new node/module version and add them to the database + - Find missing test results and return params for MC-CITGM */ + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "API_USERNAME=${env.API_USERNAME}", "PASSWORD=${env.PASSWORD}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]){ + env.citgm_parameters = sh ( + returnStdout: true, + script: "./missing-test.sh" + ) + } + } + + println "\n\n${env.citgm_parameters}\n\n" + + def resp = env.citgm_parameters.trim().tokenize('%') // Create an array of parameters + + // Create an array of buildJobs to run in parallel + def buildJobs = [:] + resp.each { str -> + def p = str.tokenize(',') + def module = "${p[0]}", module_version = "${p[1]}", node_version = "${p[2]}" + def os = "${p[3]}", arch = "${p[4]}", distro = "${p[5]}" + def parameters = [] + parameters.push(string(name: 'MODULE', value: module)) + parameters.push(string(name: 'MODULE_VERSION', value: module_version)) + parameters.push(string(name: 'NODE_VERSION', value: node_version)) + parameters.push(string(name: 'HOST', value: params.HOST)) + parameters.push(string(name: 'OSS', value: os)) + parameters.push(string(name: 'ARCHS', value: arch)) + parameters.push(string(name: 'DISTROS', value: distro)) + parameters.push(string(name: 'GIT_REMOTE_REF', value: params.GIT_REMOTE_REF)) + // parameters.push(string(name: 'CITGM_LOGLEVEL', value: params.CITGM_LOGLEVEL)) + buildJobs["$module@$module_version-$node_version($os $arch $distro)"] = { build(job: "MC-CITGM", parameters: parameters, propagate: false) } + } + + def results = parallel(buildJobs) + + def msg = '\nRESULTS:\n' + results.each { res -> + msg += "${res.key}: ${results[res.key].result}\n" + currentBuild.result = results[res.key].result + } + println msg + + } } + + stage('MC-License') { if (STARTFROM <= 3) { + + /* Find missing licenses */ + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]) { + env.license_params = sh ( + returnStdout: true, + script: "./missing-license.sh" + ) + } + } + + def license_params = env.license_params.trim().tokenize('%') // Create an array of parameters + + // Create an array of buildJobs to run in parallel + def buildJobs = [:]; + license_params.each { str -> + def p = str.tokenize(',') + def module = "${p[0].trim()}", module_version = "${p[1].trim()}" + def parameters = [] + parameters.push(string(name: 'MODULE', value: module)) + parameters.push(string(name: 'MODULE_VERSION', value: module_version)) + parameters.push(string(name: 'NODE_VERSION', value: "8")) + parameters.push(string(name: 'HOST', value: params.HOST)) + parameters.push(string(name: 'ENV', value: params.ENV)) + parameters.push(string(name: 'GIT_REMOTE_REF', value: params.GIT_REMOTE_REF)) + // parameters.push(string(name: 'CITGM_LOGLEVEL', value: params.CITGM_LOGLEVEL)) + buildJobs["$module@$module_version"] = { build(job: "MC-License-Check", parameters: parameters, propagate: false) } + } + + def results = parallel(buildJobs) + + def msg = '\nRESULTS:\n' + results.each { res -> + msg += "${res.key}: ${results[res.key].result}\n" + currentBuild.result = results[res.key].result + } + println msg + + } } + + stage('POST-CLEANUP') { if (STARTFROM <= 4) { + node ('linux && x64') { + checkout scm + withEnv(["HOST=${params.HOST}", "API_USERNAME=${env.API_USERNAME}", "PASSWORD=${env.PASSWORD}", "GIT_REMOTE_REF=${params.GIT_REMOTE_REF}"]){ + env.citgm_parameters = sh ( + script: "./clean-data.sh" + ) + } + } + } } + + println "\n\n \\(• ◡ •)/ Pipeline COMPLETE \\(• ◡ •)/\n\n" +} diff --git a/missing-license.js b/missing-license.js new file mode 100644 index 0000000..42394f8 --- /dev/null +++ b/missing-license.js @@ -0,0 +1,53 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); +const HOST = process.env.HOST; + +function getModuleVersions () { + const filter = { + "where": { + "deps_licenses_count": null + }, + "include": ["Module"] + }; + return request.get({ + url: `http://${HOST}/api/ModuleVersions`, + followAllRedirects: true, + headers: { + filter: JSON.stringify(filter) + }, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } else { + return body; + } + }); +} + +async function missingLicenseCheck () { + const params = []; + const moduleVersions = await getModuleVersions(); + moduleVersions.forEach(moduleVersion => { + params.push(`${moduleVersion.Module.name},${moduleVersion.module_version}`); + }); + console.log(params.join("%")); +} +missingLicenseCheck(); diff --git a/missing-license.sh b/missing-license.sh new file mode 100755 index 0000000..e4bb47b --- /dev/null +++ b/missing-license.sh @@ -0,0 +1,25 @@ +#!/bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +nvs add 8 >/dev/null +nvs use 8 >/dev/null + +npm install >/dev/null + +node missing-license.js diff --git a/missing-test.js b/missing-test.js new file mode 100644 index 0000000..1739c0f --- /dev/null +++ b/missing-test.js @@ -0,0 +1,330 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const { + spawnSync +} = require("child_process"); +const request = require("request-promise"); +const fs = require("fs"); +const path = require("path"); +const lookup = JSON.parse(fs.readFileSync( + path.join(__dirname, "/lookup.json") +)); +const HOST = process.env.HOST; +const TOKEN = process.env.TOKEN; +const TIMESTAMP = new Date(); +let params = []; + +if (TOKEN === null || TOKEN === "Incorrect Credentials") { + console.error("TOKEN is not defined"); + process.exit(1); +} + +function check(e, r, b) { + if (e || r.statusCode !== 200) { + console.error(b.error); + process.exit(1); + } else { + return b; + } +} + +/* +Find missing test results for the latest module_version and +latest node_version (per LTS). +*/ +function getLatestNodeVersion() { + return request.get({ + url: `http://${HOST}/api/LTSVersions/latestNodeVersion`, + followAllRedirects: true + }, (err, res, body) => { + check(err, res, body); + }); +} + +// Fetch the list of valid platforms +function getValidPlatforms() { + return request.get({ + url: `http://${HOST}/api/ValidPlatforms`, + followAllRedirects: true + }, (err, res, body) => { + check(err, res, body); + }); +} + +// Adds a new node version to the DB +function addNodeVersion(nodeVersion) { + let LTSVersion = nodeVersion.split(".")[0].replace("v", ""); + request.post({ + url: `http://${HOST}/api/NodeVersions`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: { + "node_version": nodeVersion, + "lts_version": LTSVersion + }, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + check(err, res, body); + }); +} + +function getModule(moduleName) { + return request.get({ + url: `http://${HOST}/api/Modules`, + headers: { + filter: JSON.stringify({ + "where": { + "name": moduleName + } + }) + }, + followAllRedirects: true + }, (err, res, body) => { + check(err, res, body); + }); +} + +// Adds a new module version to the DB +function addModule(moduleName) { + return request.post({ + url: `http://${HOST}/api/Modules`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: { + "name": moduleName + }, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + check(err, res, body); + }); +} + +async function getModuleVersion(moduleName, moduleVersion) { + const module = await getModule(moduleName); + const moduleID = (module) ? JSON.parse(module)[0].id : null; + return request.get({ + url: `http://${HOST}/api/ModuleVersions`, + headers: { + filter: JSON.stringify({ + "where": { + "module_id": moduleID, + "module_version": moduleVersion + } + }) + }, + followAllRedirects: true, + json: true + }, (err, res, body) => { + check(err, res, body); + }); +} + +function addModuleVersion(moduleName, moduleVersion) { + let license = spawnSync("npm", ["view", moduleName, + "license" + ]).stdout.toString().trim(); + return request.post({ + url: `http://${HOST}/api/ModuleVersions`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: { + "module": moduleName, + "module_version": moduleVersion, + "license": license || null, + "min_dep_license": license || null, + "timestamp": TIMESTAMP + }, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + check(err, res, body); + }); +} + +function addRecommendedVersions(moduleName) { + let recMajorVersions = lookup[moduleName]["recommended-major-versions"]; + for (let recVersion in recMajorVersions) { + let ltsVersion = recVersion.split(".")[0]; + request.post({ + url: `http://${HOST}/api/ModuleRecommendedLTSs`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: { + "lts_version": ltsVersion, + "module": moduleName, + "recommended_version": recMajorVersions[recVersion] + }, + json: true // Automatically stringifies the body to JSON + }, (err) => { + if (err) { + return console.error(err); + } + }); + } +} + +// Fetch all tests for each module +function getModuleVersionTests(moduleName, moduleVersion, ltsVersion) { + let filter = { + "where": { + "module": moduleName, + "module_version": moduleVersion, + "node_version": ltsVersion.latestNodeVersion + }, + "include": ["Architecture", "Distribution", "OperatingSystem"] + }; + return request.get({ + url: `http://${HOST}/api/ModuleVersionTests`, + followAllRedirects: true, + headers: { + filter: JSON.stringify(filter) + }, + json: true + }, (err, res, body) => { + check(err, res, body); + }); +} + +async function missingTests() { + + const ltsVersions = JSON.parse(await getLatestNodeVersion()); + const validPlatforms = JSON.parse(await getValidPlatforms()); + + // Add a new Node Versions + if (ltsVersions[0].latestNodeVersion !== process.env.node6) { + await addNodeVersion(process.env.node6); + } + if (ltsVersions[1].latestNodeVersion !== process.env.node8) { + await addNodeVersion(process.env.node8); + } + if (ltsVersions[2].latestNodeVersion !== process.env.node10) { + await addNodeVersion(process.env.node10); + } + + async function checkModuleExists(moduleName, moduleVersion) { + let moduleExists = await getModule(moduleName); + if (JSON.parse(moduleExists).length === 0) { + await addModule(moduleName); + await addModuleVersion(moduleName, moduleVersion); + await addRecommendedVersions(moduleName); + } + } + + // Add a new Module Versions + let modules = []; + + for (let i in Object.keys(lookup)) { + const moduleName = Object.keys(lookup)[i]; + const moduleData = lookup[moduleName]; + + const moduleVersions = new Set(); + // Add Latest Module Version + const latestModuleVersion = + spawnSync("npm", ["view", moduleName, "version"]) + .stdout.toString().trim(); + moduleVersions.add(latestModuleVersion); + + // Add Latest Module Version for each recommended major version + const recMajorVersions = moduleData["recommended-major-versions"]; + Object.keys(recMajorVersions).forEach(lts => { + const recMajVersion = recMajorVersions[lts]; + const allVersionsForMajor = + spawnSync("npm", ["view", `${moduleName}@${recMajVersion}`, "version"]) + .stdout.toString().trim().split(" "); + // Latest Module Version for Major + const moduleVersionForMajor = + allVersionsForMajor[allVersionsForMajor.length - 1].replace(/'/g, ""); + + // Add required module version is it isnt a duplicate + if (!moduleVersions.has(moduleVersionForMajor)) { + moduleVersions.add(moduleVersionForMajor); + } + }); + + // Array.from() converts a set to an Array + modules.push({ + name: moduleName, + versions: Array.from(moduleVersions) + }); + + //Add ModuleVersions + await checkModuleExists(moduleName, latestModuleVersion); + moduleVersions.forEach(async moduleVersion => { + const resp = await getModuleVersion(moduleName, moduleVersion); + if (resp.length === 0) { + await addModuleVersion(moduleName, moduleVersion); + } + }); + } + + await findMissingTests(); + console.log(params.join("%")); + + async function findMissingTests() { + for (let i in modules) { + let module = modules[i]; + for (let j in ltsVersions) { + let ltsVersion = ltsVersions[j]; + for (let k in module.versions) { + let moduleVersion = module.versions[k]; + const mvt = + await getModuleVersionTests(module.name, moduleVersion, ltsVersion); + if (mvt.length === 0) { + // Run on all platforms + params.push([module.name, moduleVersion, + ltsVersion.latestNodeVersion, "all", "all", "all" + ]); + } else if (mvt.length !== validPlatforms.length) { + /* + Don't run if no missing files. + IF there are => for each valid platform check if + there is an existing test result + */ + validPlatforms.forEach(plat => { + let exist = false; + try { + mvt.forEach(test => { + if (test.OperatingSystem.os === plat.os && + test.Architecture.arch === plat.arch && + test.Distribution.distro === plat.distro) { + exist = true; + } + }); + } catch (e) { + console.error(e); + } + if (!exist) { + params.push([module.name, moduleVersion, + ltsVersion.latestNodeVersion, plat.os, plat.arch, plat.distro + ]); + } + }); + } + } + } + } + } +} +missingTests(); diff --git a/missing-test.sh b/missing-test.sh new file mode 100755 index 0000000..b0fada2 --- /dev/null +++ b/missing-test.sh @@ -0,0 +1,36 @@ +#!/bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +nvs add 6 >/dev/null +nvs use 6 >/dev/null +export node6="$(node -v)" + +nvs add 8 >/dev/null +nvs use 8 >/dev/null +export node8="$(node -v)" + +nvs add 10 >/dev/null +nvs use 10 >/dev/null +export node10="$(node -v)" + +npm install >/dev/null +export TOKEN=$(node fetch-token.js) + +node missing-test.js +node logout.js diff --git a/package.json b/package.json new file mode 100644 index 0000000..9b8274f --- /dev/null +++ b/package.json @@ -0,0 +1,21 @@ +{ + "name": "module-insights-ci", + "version": "0.1.0", + "license": "Apache-2.0", + "repository": { + "type": "git", + "url": "https://github.com/CloudNativeJS/module-insights-ci" + }, + "scripts": { + "lint": "eslint **/*.js", + "test": "npm run lint && bash ./test/test.sh" + }, + "dependencies": { + "ibm-cos-sdk": "^1.2.3", + "request": "^2.87.0", + "request-promise": "^4.2.2" + }, + "devDependencies": { + "eslint": "^3.17.1" + } +} diff --git a/setup-npm.sh b/setup-npm.sh new file mode 100644 index 0000000..4b9927e --- /dev/null +++ b/setup-npm.sh @@ -0,0 +1,34 @@ +#!/bin/bash -ex + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +# Redirects npm directories into the WORKSPACE and sets up artifactory. +# Script should be sourced. +rm -rf "$WORKSPACE"/npm + +export npm_config_userconfig="$WORKSPACE"/npm/npmrc +mkdir -p "$WORKSPACE"/npm/npm-{cache,devdir,tmp} + +cat <$WORKSPACE/npm/npmrc +tmpdir=$WORKSPACE/npm/npm-tmp +cache=$WORKSPACE/npm/npm-cache +devdir=$WORKSPACE/npm/npm-devdir +@wicked:registry=https://na.artifactory.swg-devops.com/artifactory/api/npm/wicked-npm-local/ +//na.artifactory.swg-devops.com/artifactory/api/npm/wicked-npm-local/:_password="\${NPM_TOKEN}" +//na.artifactory.swg-devops.com/artifactory/api/npm/wicked-npm-local/:username=jsbuild@ca.ibm.com +//na.artifactory.swg-devops.com/artifactory/api/npm/wicked-npm-local/:email=jsbuild@ca.ibm.com +//na.artifactory.swg-devops.com/artifactory/api/npm/wicked-npm-local/:always-auth=true +!!EOF diff --git a/setup-nvs.sh b/setup-nvs.sh new file mode 100644 index 0000000..70887a4 --- /dev/null +++ b/setup-nvs.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +if [ -z $NVS_HOME ]; then + export NVS_HOME="$HOME/.nvs" +fi + +[ -s "$NVS_HOME/nvs.sh" ] && . "$NVS_HOME/nvs.sh" +export NVS_HOME=$HOME/.nvs + +if ! [ $(command -v nvs) ]; then + echo Please install nvs + exit 1 +fi diff --git a/storing-logs.js b/storing-logs.js new file mode 100644 index 0000000..90cf395 --- /dev/null +++ b/storing-logs.js @@ -0,0 +1,64 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const IBMCloudObjectStore = require("ibm-cos-sdk"); +const fs = require('fs'); + +const BUCKET = "module-insights-logs"; +const API_KEY = process.env.API_KEY; +const INSTANCE_ID = process.env.INSTANCE_ID; +const MODULE = process.env.MODULE; +const MODULE_VERSION = process.env.MODULE_VERSION; +const OS = process.env.OS; +const ARCH = process.env.ARCH; +const NODE_VERSION = process.env.NODE_VERSION; +const WORKSPACE = process.env.WORKSPACE; +const LOG_FILE = `${WORKSPACE}/${MODULE}.tap`; +const FILE_NAME= `${MODULE}@${MODULE_VERSION}-${OS}-${ARCH}-${NODE_VERSION}.txt`; + +const config = { + endpoint: "s3-api.us-geo.objectstorage.softlayer.net", + apiKeyId: API_KEY, + ibmAuthEndpoint: "https://iam.bluemix.net/identity/token", + serviceInstanceId: INSTANCE_ID +}; + +const objectStoreClient = new IBMCloudObjectStore.S3(config); + +function createLogFile(BUCKET, FILE_NAME, LOG_FILE) { + console.log(`\nCreating new item: ${FILE_NAME}`); + const body = fs.readFileSync(LOG_FILE, 'utf8') + return objectStoreClient.putObject({ + Bucket: BUCKET, + Key: FILE_NAME, + Body: body, + ACL: "public-read" + }).promise() + .then(() => { + return console.log(`Item: ${FILE_NAME} created!`); + }) + .catch((e) => { + return console.log(`ERROR: ${e.code} - ${e.message}\n`); + }); +} + +async function creating(){ + await createLogFile(BUCKET, FILE_NAME, LOG_FILE); + console.log("Finshed Adding Logs"); + process.exit(0); +} +creating(); diff --git a/test-data.sh b/test-data.sh new file mode 100644 index 0000000..89af0bd --- /dev/null +++ b/test-data.sh @@ -0,0 +1,192 @@ +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +export TOKEN=$(node fetch-token.js); + +# This is done by node initialise.js + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"arch": "x64"}' "http://${HOST}/api/Architectures" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"distro": "ubuntu-16.04"}' "http://${HOST}/api/Distributions" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"os": "linux"}' "http://${HOST}/api/OperatingSystems" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"os": "linux","arch": "x64","distro": "ubuntu-16.04"}' "http://${HOST}/api/ValidPlatforms" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"node_version": "v6.14.3","lts_version_id": 1}' "http://${HOST}/api/NodeVersions" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"node_version": "v8.11.3","lts_version_id": 2}' "http://${HOST}/api/NodeVersions" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"lts_version": 6}' "http://${HOST}/api/LTSVersions" + +# curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +# -d '{"lts_version": 8}' "http://${HOST}/api/LTSVersions" + +echo "#------------------------------- New Node Versions -------------------------------\n" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"node_version": "v6.14.4","lts_version_id": 1}' "http://${HOST}/api/NodeVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"node_version": "v8.12.0","lts_version_id": 2}' "http://${HOST}/api/NodeVersions" + +echo "\n\nAdded New Node Versions" + +echo "#------------------------------- @Cloudant/cloudant -------------------------------\n" + +curl -X POST --header 'Content-Type: application/json' -H "Authorization: ${TOKEN}" \ +-d '{"name":"@cloudant/cloudant","stability":"LTS Adopted","commercial_support":["IBM"]}' "http://${HOST}/api/Modules" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "@cloudant/cloudant","module_major_version": "2.x","module_version": "2.4.0","license": "Apache-2.0","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "@cloudant/cloudant","module_major_version": "2.x","module_version": "2.3.0","license": "Apache-2.0","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 2,"os_id": 2,"node_version_id": 3,"module": "@cloudant/cloudant","module_version": "2.4.0","code_coverage": 92,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 3,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "@cloudant/cloudant","module_version": "2.4.0","code_coverage": 92,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 2,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "@cloudant/cloudant","module_version": "2.4.0","code_coverage": 92,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "@cloudant/cloudant","module_version": "2.4.0","code_coverage": 92,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "@cloudant/cloudant","module_version": "2.4.0","code_coverage": 92,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 2,"os_id": 2,"node_version_id": 3,"module": "@cloudant/cloudant","module_version": "2.3.0","code_coverage": 92,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 1,"module": "@cloudant/cloudant","recommended_version": "2.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 2,"module": "@cloudant/cloudant","recommended_version": "2.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +echo "\n\nAdded @cloudant/cloudant" + +echo "#------------------------------- Appmetrics -------------------------------\n" + +curl -X POST --header 'Content-Type: application/json' -H "Authorization: ${TOKEN}" \ +-d '{"name":"appmetrics","stability":"LTS Adopted","commercial_support":["IBM"]}' "http://${HOST}/api/Modules" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "appmetrics","module_major_version": "4.x","module_version": "4.0.0","license": "Apache-2.0","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "appmetrics","module_version": "4.0.0","code_coverage": 50,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "appmetrics","module_version": "4.0.0","code_coverage": 50,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 2,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "appmetrics","module_version": "4.0.0","code_coverage": 50,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 2,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "appmetrics","module_version": "4.0.0","code_coverage": 50,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 1,"module": "appmetrics","recommended_version": "4.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 2,"module": "appmetrics","recommended_version": "4.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +echo "\n\nAdded Appmetrics" + +echo "# ------------------------------- Commander -------------------------------\n" + +curl -X POST --header 'Content-Type: application/json' -H "Authorization: ${TOKEN}" \ +-d '{"name":"commander","stability":"","commercial_support":"[]"}' "http://${HOST}/api/Modules" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "commander","module_major_version": "2.x","module_version": "2.16.0","license": "MIT","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "commander","module_major_version": "2.x","module_version": "2.17.1","license": "MIT","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "commander","module_version": "2.16.0","code_coverage": 94,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "commander","module_version": "2.16.0","code_coverage": 94,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 3,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "commander","module_version": "2.17.1","code_coverage": 87,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 3,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "commander","module_version": "2.17.1","code_coverage": 87,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 1,"module": "commander","recommended_version": "2.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 2,"module": "commander","recommended_version": "2.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +echo "\n\nAdded Commander" + +echo "#------------------------------- Bluebird -------------------------------\n" + +curl -X POST --header 'Content-Type: application/json' -H "Authorization: ${TOKEN}" \ +-d '{"name":"bluebird","stability":"","commercial_support":"[]"}' "http://${HOST}/api/Modules" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "bluebird","module_major_version": "3.x","module_version": "3.5.1","license": "MIT","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "bluebird","module_version": "3.5.1","code_coverage": 25,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "bluebird","module_version": "3.5.1","code_coverage": 25,"passed": false}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 1,"module": "bluebird","recommended_version": "3.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 2,"module": "bluebird","recommended_version": "3.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +echo "\n\nAdded Bluebird" + +echo "#------------------------------- Async -------------------------------\n" + +curl -X POST --header 'Content-Type: application/json' -H "Authorization: ${TOKEN}" \ +-d '{"name":"async","stability":"","commercial_support":"[]"}' "http://${HOST}/api/Modules" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"module": "async","module_major_version": "2.x","module_version": "2.6.1","license": "MIT","deps_ids": "string","deps_licenses": "string","deps_licenses_count": 0,"timestamp": "2018-08-22T12:43:53.871Z"}' "http://${HOST}/api/ModuleVersions" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 3,"module": "async","module_version": "2.6.1","code_coverage": 98,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"arch_id": 1,"distro_id": 1,"os_id": 1,"node_version_id": 4,"module": "async","module_version": "2.6.1","code_coverage": 98,"passed": true}' "http://${HOST}/api/ModuleVersionTests" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 1,"module": "async","recommended_version": "2.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +curl -X POST --header 'Content-Type: application/json' --header "Authorization: ${TOKEN}" \ +-d '{"lts_version_id": 2,"module": "async","recommended_version": "2.x"}' "http://${HOST}/api/ModuleRecommendedLTSs" + +echo "\n\nAdded Async" diff --git a/test/test.sh b/test/test.sh new file mode 100755 index 0000000..87b4f9c --- /dev/null +++ b/test/test.sh @@ -0,0 +1,134 @@ +#!/bin/bash + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +export HOST=module-curation-staging.eu-gb.mybluemix.net +export API_USERNAME=admin +export WORKSPACE=$PWD +export NVS_HOME=$HOME/.nvs + +if [ -z $PASSWORD ]; then + echo PASSWORD is not defined so skipping the test suite + exit 0 +else + . ./setup-nvs.sh + nvs add 6 + nvs add 8 + + ##################################### + echo "\n##### Testing fetch-token.js #####" + nvs use 8 + npm install >/dev/null + export TOKEN=$(node fetch-token.js) + + if [ "$TOKEN" == null ] || [ "$TOKEN" == "Incorrect Credentials" ]; then + echo "✘ fetch-token.js failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + ##################################### + echo "\n##### Testing missing-test.sh #####" + . ./missing-test.sh + if [ $? != 0 ]; then + echo "✘ missing-test.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + unset node6 + unset node8 + ##################################### + echo "\n##### Testing citgm-check.sh #####" + + export OS=linux + export ARCH=x64 + export DISTRO=ubuntu-16.04 + export MODULE=async + export MODULE_VERSION=2.6.1 + export NODE_VERSION=v8.12.0 + + . ./citgm-check.sh + if [ $? != 0 ]; then + echo "✘ citgm-check.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + unset OS ARCH DISTRO MODULE MODULE_VERSION NODE_VERSION + ##################################### + echo "\n##### Testing clean-data.sh #####" + + . ./clean-data.sh + if [ $? != 0 ]; then + echo "✘ clean-data.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + ##################################### + echo "\n##### Testing failed-test.sh #####" + + . ./failed-test.sh + if [ $? != 0 ]; then + echo "✘ failed-test.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + ##################################### + echo "\n##### Testing missing-license.sh #####" + + . ./missing-license.sh + if [ $? != 0 ]; then + echo "✘ missing-license.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + ##################################### + echo "\n##### Testing license-check.sh #####" + + export MODULE=@cloudnative/health + export MODULE_VERSION=1.0.0 + export NODE_VERSION=v6.14.3 + + . ./license-check.sh + if [ $? != 0 ]; then + echo "✘ license-check.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + unset MODULE MODULE_VERSION NODE_VERSION + ##################################### + echo "\n##### Testing update-recommended-versions.sh #####" + + . ./update-recommended-versions.sh + if [ $? != 0 ]; then + echo "✘ update-recommended-versions.sh failed with non-zero exit code" + FAILED="true" + else + echo "✓ Test Passed!" + fi + + if [ $FAILED == "true" ]; then + echo "\nTests failed" + exit 1 + else + echo "\n✓ All tests passed" + fi +fi diff --git a/update-license.js b/update-license.js new file mode 100644 index 0000000..e01a0ef --- /dev/null +++ b/update-license.js @@ -0,0 +1,139 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); +const [moduleName, moduleVersion] = [process.env.MODULE, + process.env.MODULE_VERSION]; +const HOST = process.env.HOST; +const TOKEN = process.env.TOKEN; + +// Throw an error if license report doesn't exist +let json; +try { + json = require(`${process.env.WORKSPACE}/`+ + `${moduleName.replace(/\//, "%2F")}/license.json`); +} catch (e) { + throw e; +} + +/* Create an object with all dependencies and their license +e.g {"request@2.85.0": "MIT", "appmetrics@4.0.0": "Apache2.0"} */ +let obj = {}; +Object.keys(json).forEach(depModule => { + let depModuleName = depModule.split("@")[0]; + if (depModuleName !== moduleName) { + obj[depModule] = json[depModule].licenses; + } else { + delete obj.depModule; + } +}); + +function check (e, r, b) { + if (e || r.statusCode !== 200) { + console.error(b.error); + process.exit(1); + } else { + return b; + } +} + +/* Posting an object (like above) to /api/DepModuleVersions/add: + - adds all dep licenses at once if they don't exist + - returns an object with all of the depIds and distinct depsLicenses. */ +function updateDepModuleVersions () { + return request.post({ + url: `http://${HOST}/api/DepModuleVersions/add`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: { + obj: obj + }, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + licenseObj(body); + check(err, res, body); + }); +} + +function getModuleVersionID() { + return request.get({ + url: `http://${HOST}/api/ModuleVersions`, + followAllRedirects: true, + headers: { + filter: JSON.stringify({ + "where": { + "module": moduleName, + "module_version": moduleVersion + } + }) + }, + json: true + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } + return body; + }); +} + +async function patchLicense(licenseObj) { + const id = await getModuleVersionID(licenseObj); + if (id[0] && id[0].id) { + licenseObj.id = id[0].id; + } + console.log(licenseObj); + return request.patch({ + url: `http://${HOST}/api/ModuleVersions`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: licenseObj, + json: true + }, (err, res) => { + if (err || res.statusCode !== 200) { + process.exit(1); + } + }); +} + +async function licenseObj (retObj) { + const depsIds = retObj.ids || " "; + const depsLicenses = retObj.licenses || " "; + // Number of unique licenses + const depsLicensesCount = retObj.licenses_count || " "; + const licenseObj = { + module: moduleName, + module_version: moduleVersion, + deps_ids: depsIds, + deps_licenses: depsLicenses, + deps_licenses_count: depsLicensesCount + }; + await patchLicense(licenseObj); +} + +async function updateLicense () { + try { + await updateDepModuleVersions(); + } catch (err) { + console.log(err); + } +} +updateLicense(); diff --git a/update-recommended-versions.js b/update-recommended-versions.js new file mode 100644 index 0000000..0d556b4 --- /dev/null +++ b/update-recommended-versions.js @@ -0,0 +1,177 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); +const lookup = require("./lookup.json"); + +const HOST = process.env.HOST; +const TOKEN = process.env.TOKEN; + +function getModule(moduleName) { + return request.get({ + url: `http://${HOST}/api/Modules`, + followAllRedirects: true, + json: true, + headers: { + "filter": JSON.stringify({ + "where": { + "name": moduleName + } + }) + } + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } + }); +} + +function getLTSs() { + return request.get({ + url: `http://${HOST}/api/LTSVersions`, + followAllRedirects: true, + json: true + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } + }); +} + +function getRecommendedVersion(lts_version_id, module_id) { + return request.get({ + url: `http://${HOST}/api/ModuleRecommendedLTSs`, + followAllRedirects: true, + json: true, + headers: { + "filter": JSON.stringify({ + "where": { + "lts_version_id": lts_version_id, + "module_id": module_id + } + }) + } + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } + return body; + }); +} + +function getCurrentRecommendedVersions() { + return request.get({ + url: `http://${HOST}/api/ModuleRecommendedLTSs`, + followAllRedirects: true, + headers: { + "filter": JSON.stringify({ + "include": [ + "LTSVersion", + "Module" + ] + }) + } + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } + return body; + }); +} + +function addRecommendedVersion(data) { + return request.post({ + url: `http://${HOST}/api/ModuleRecommendedLTSs`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: data, + json: true // Automatically stringifies the body to JSON + }, (err, res) => { + if (err || res.statusCode !== 200) { + process.exit(1); + } + }); +} + +function patchRecommendedVersion(data) { + return request.patch({ + url: `http://${HOST}/api/ModuleRecommendedLTSs`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: data, + json: true // Automatically stringifies the body to JSON + }, (err, res) => { + if (err || res.statusCode !== 200) { + process.exit(1); + } + }); +} + +async function updateRecommendedVersions() { + const recommendedVersions = await getCurrentRecommendedVersions(); + JSON.parse(recommendedVersions).forEach((recommendedVersion) => { + const moduleName = recommendedVersion.Module.name; + const recMajorVersion = "recommended-major-versions"; + const recLTSVersion = + `${recommendedVersion.LTSVersion.lts_version}.x`; + const lookupVersion = lookup[moduleName][recMajorVersion][recLTSVersion]; + if (recommendedVersion.recommended_version !== lookupVersion) { + patchRecommendedVersion({ + "lts_version_id": recommendedVersion.lts_version_id, + "module_id": recommendedVersion.module_id, + "recommended_version": lookupVersion, + "id": recommendedVersion.id + }); + } + }); +} + +async function addMissingRecommendedVersions() { + const ltsVersions = await getLTSs(); + Object.keys(lookup).forEach(async (moduleName) => { + ltsVersions.forEach(async (lts) => { + const ltsMajor = `${lts.lts_version}.x` + const recMajorVersion = lookup[moduleName]["recommended-major-versions"][ltsMajor]; + if (recMajorVersion) { + const module = await getModule(moduleName); + const module_id = module[0].id; + const currentRecommended = await getRecommendedVersion(lts.id, module_id); + if (currentRecommended.length == 0) { + addRecommendedVersion({ + "lts_version_id": lts.id, + "module_id": module_id, + "recommended_version": recMajorVersion + }); + } + } else { + console.log(`${moduleName} doesn't have a recommended major version for node ${ltsMajor} in the lookup`); + } + }); + }); +} + +updateRecommendedVersions(); +addMissingRecommendedVersions(); + diff --git a/update-recommended-versions.sh b/update-recommended-versions.sh new file mode 100644 index 0000000..705f625 --- /dev/null +++ b/update-recommended-versions.sh @@ -0,0 +1,27 @@ +#!/bin/bash -e + +############################################################################ +# +# (C) Copyright IBM Corp. 2018 +# +# This program and the accompanying materials are made available +# under the terms of the Apache License v2.0 which accompanies +# this distribution. +# +# The Apache License v2.0 is available at +# http://www.opensource.org/licenses/apache2.0.php +# +# Contributors: +# Multiple authors (IBM Corp.) - initial implementation and documentation +############################################################################ + +. ./setup-nvs.sh + +nvs add 8 +nvs use 8 + +npm install +export TOKEN=$(node fetch-token.js) + +node update-recommended-versions.js +node logout.js diff --git a/update-test.js b/update-test.js new file mode 100644 index 0000000..41a3406 --- /dev/null +++ b/update-test.js @@ -0,0 +1,121 @@ +"use strict"; + +/*************************************************************************** + * + * (C) Copyright IBM Corp. 2018 + * + * This program and the accompanying materials are made available + * under the terms of the Apache License v2.0 which accompanies + * this distribution. + * + * The Apache License v2.0 is available at + * http://www.opensource.org/licenses/apache2.0.php + * + * Contributors: + * Multiple authors (IBM Corp.) - initial implementation and documentation + ***************************************************************************/ + +const request = require("request-promise"); + +const [OS, ARCH, DISTRO] = [process.env.OS, process.env.ARCH, + process.env.DISTRO]; +const [moduleName, moduleVersion] = [process.env.MODULE, + process.env.MODULE_VERSION]; +const nodeVersion = process.env.NODE_VERSION; +const HOST = process.env.HOST; +const TOKEN = process.env.TOKEN; +const passResult = (parseInt(process.env.EXIT_STATUS) === 0); + +if (TOKEN === null || TOKEN === "Incorrect Credentials") { + console.error("TOKEN is not defined"); + process.exit(1); +} + +console.log(`\nUpdating HOST: ${HOST}` + + `\nModule: ${moduleName},` + + `Module Verison: ${moduleVersion},` + + `Node Version: ${nodeVersion}` + + `\nPass: ${passResult}` +); + +/* +Catch an error if coverage-summary.json +doesn't exist and set codeCoverage to 0 +*/ +let codeCoverage; +try { + const coverageSummary = require(`${process.env.WORKSPACE}/` + + `${moduleName}/coverage-summary.json`); + codeCoverage = parseInt( + coverageSummary.total.lines.pct.toString().split(".")[0] + ); + console.log("Coverage summary:", coverageSummary); +} catch (e) { + codeCoverage = 0; +} + +function getModuleVersionTestID() { + return request.get({ + url: `http://${HOST}/api/ModuleVersionTests`, + followAllRedirects: true, + headers: { + filter: JSON.stringify({ + "where": { + "module": moduleName, + "module_version": moduleVersion, + "node_version": nodeVersion, + "os": OS, + "arch": ARCH, + "distro": DISTRO + } + }) + }, + json: true + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } else { + return body; + } + }); +} + +async function patchModuleVersionTest () { + const id = await getModuleVersionTestID(); + let data = { + "module": moduleName, + "module_version": moduleVersion, + "node_version": nodeVersion, + "os": OS, + "arch": ARCH, + "distro": DISTRO, + "passed": passResult, + "code_coverage": codeCoverage || 0 + }; + if (id[0] && id[0].id) { + data.id = id[0].id; + } + return request.patch({ + url: `http://${HOST}/api/ModuleVersionTests`, + followAllRedirects: true, + headers: { + "Authorization": TOKEN + }, + body: data, + json: true // Automatically stringifies the body to JSON + }, (err, res, body) => { + if (err || res.statusCode !== 200) { + console.error(body.error); + process.exit(1); + } else { + console.log("Adding Module Version Test :", body); + return body; + } + }); +} + +async function updateTest() { + await patchModuleVersionTest(); +} +updateTest();