diff --git a/index.js b/index.js index 8d3c243..291fc62 100644 --- a/index.js +++ b/index.js @@ -15,7 +15,8 @@ const phaseMerge = require('./lib/phase/merge'); const { flattenSharedIntoJobs, handleMergeSharedStepsAnnotation, - flattenPhase: phaseFlatten + flattenPhase: phaseFlatten, + flattenTemplates } = require('./lib/phase/flatten'); const phaseValidateFunctionality = require('./lib/phase/functional'); const phaseGeneratePermutations = require('./lib/phase/permutation'); @@ -314,7 +315,26 @@ async function parsePipelineTemplate({ yaml }) { return pipelineTemplate; } +/** + * Generates pipeline template configuration for the validator + * @method validatePipelineTemplate + * @param {Object} config + * @param {String} config.yaml Pipeline Template + * @param {TemplateFactory} config.templateFactory Template Factory to get templates + * @return {Object} Pipeline Template + */ +async function validatePipelineTemplate({ yaml, templateFactory }) { + const pipelineTemplate = await parsePipelineTemplate({ yaml }); + // Merge template steps for validator + const { newJobs } = await flattenTemplates(pipelineTemplate.config, templateFactory, true); + + pipelineTemplate.config.jobs = newJobs; + + return pipelineTemplate; +} + module.exports = { parsePipelineYaml, - parsePipelineTemplate + parsePipelineTemplate, + validatePipelineTemplate }; diff --git a/lib/phase/flatten.js b/lib/phase/flatten.js index 99535b6..2b42ab3 100644 --- a/lib/phase/flatten.js +++ b/lib/phase/flatten.js @@ -358,9 +358,18 @@ function handleMergeSharedStepsAnnotation({ sharedConfig, jobConfig, template }) * @param {TemplateFactory} templateFactory Template Factory to get templates * @param {Object} sharedConfig Shared configuration * @param {Object} pipelineParameters Pipeline level parameters + * @param {Boolean} [isPipelineTemplate] If the current template is pipeline template or not * @return {Promise} */ -function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sharedConfig, pipelineParameters }) { +async function mergeTemplateIntoJob({ + jobName, + jobConfig, + newJobs, + templateFactory, + sharedConfig, + pipelineParameters, + isPipelineTemplate +}) { let oldJob = jobConfig; // Try to get the template @@ -390,7 +399,9 @@ function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sh let warnings = []; // merge shared steps into oldJob - oldJob = handleMergeSharedStepsAnnotation({ sharedConfig, jobConfig: oldJob, template }); + if (!isPipelineTemplate) { + oldJob = handleMergeSharedStepsAnnotation({ sharedConfig, jobConfig: oldJob, template }); + } // Include parameters from the template only if it not overwritten either in pipeline or job parameters if (newJob.parameters !== undefined) { @@ -409,7 +420,7 @@ function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sh } else { delete newJob.parameters; } - } else { + } else if (oldJob.parameters !== undefined) { newJob.parameters = oldJob.parameters; } @@ -435,11 +446,12 @@ function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sh * Goes through each job and if template is specified, then merge into job config * * @method flattenTemplates - * @param {Object} doc Document that went through structural parsing - * @param {TemplateFactory} templateFactory Template Factory to get templates + * @param {Object} doc Document that went through structural parsing + * @param {TemplateFactory} templateFactory Template Factory to get templates + * @param {Boolean} [isPipelineTemplate] If the current template is pipeline template or not * @return {Promise} Resolves to new object with jobs after merging templates */ -function flattenTemplates(doc, templateFactory) { +async function flattenTemplates(doc, templateFactory, isPipelineTemplate) { const newJobs = {}; const templates = []; const { jobs, shared, parameters } = doc; @@ -466,7 +478,8 @@ function flattenTemplates(doc, templateFactory) { newJobs, templateFactory, sharedConfig: shared, - pipelineParameters: parameters + pipelineParameters: parameters, + isPipelineTemplate }) ); } else { @@ -754,5 +767,6 @@ function flattenPhase(parsedDoc, templateFactory) { module.exports = { flattenPhase, flattenSharedIntoJobs, - handleMergeSharedStepsAnnotation + handleMergeSharedStepsAnnotation, + flattenTemplates }; diff --git a/package.json b/package.json index daf20d3..967982e 100644 --- a/package.json +++ b/package.json @@ -56,7 +56,7 @@ "joi": "^17.7.0", "js-yaml": "^4.1.0", "keymbinatorial": "^2.0.0", - "screwdriver-data-schema": "^23.3.1", + "screwdriver-data-schema": "^23.3.2", "screwdriver-notifications-email": "^3.0.1", "screwdriver-notifications-slack": "^5.0.0", "screwdriver-workflow-parser": "^4.3.0", diff --git a/test/data/pipeline-template-invalid.yaml b/test/data/pipeline-template-invalid.yaml index 8be7387..5601925 100644 --- a/test/data/pipeline-template-invalid.yaml +++ b/test/data/pipeline-template-invalid.yaml @@ -6,5 +6,4 @@ jobs: - install: npm install requires: - ~pr - - ~commit - \ No newline at end of file + - ~commit \ No newline at end of file diff --git a/test/data/validate-pipeline-template-invalid.yaml b/test/data/validate-pipeline-template-invalid.yaml new file mode 100644 index 0000000..f9309eb --- /dev/null +++ b/test/data/validate-pipeline-template-invalid.yaml @@ -0,0 +1,14 @@ +namespace: template_namespace +name: template_name +version: 1.2.3 +description: template description +maintainer: name@domain.org +config: + shared: + image: node:20 + environment: + FOO: foo + parameters: + user: + value: sd-bot + description: User running build \ No newline at end of file diff --git a/test/data/validate-pipeline-template-with-job-template.json b/test/data/validate-pipeline-template-with-job-template.json new file mode 100644 index 0000000..3cb6de9 --- /dev/null +++ b/test/data/validate-pipeline-template-with-job-template.json @@ -0,0 +1,112 @@ +{ + "config": { + "annotations": { + "screwdriver.cd/chainPR": false, + "screwdriver.cd/restrictPR": "none" + }, + "jobs": { + "extra": { + "annotations": {}, + "environment": { + "FOO": "BAR" + }, + "image": "node:20", + "requires": [ + "main" + ], + "secrets": [], + "settings": {}, + "sourcePaths": [], + "steps": [ + { + "name": "echo \"pipeline template test\"" + } + ] + }, + "main": { + "annotations": {}, + "environment": { + "BAR": "foo", + "FOO": "BAR", + "SD_TEMPLATE_FULLNAME": "mytemplate", + "SD_TEMPLATE_NAME": "mytemplate", + "SD_TEMPLATE_NAMESPACE": "", + "SD_TEMPLATE_VERSION": "1.2.3" + }, + "image": "golang", + "requires": [ + "main" + ], + "secrets": [ + "GIT_KEY" + ], + "settings": { + "email": "foo@example.com" + }, + "sourcePaths": [], + "steps": [ + { + "install": "npm install" + }, + { + "test": "npm test" + } + ], + "templateId": 7754 + }, + "other": { + "annotations": {}, + "environment": { + "BAR": "foo", + "FOO": "BAR", + "SD_TEMPLATE_FULLNAME": "mytemplate", + "SD_TEMPLATE_NAME": "mytemplate", + "SD_TEMPLATE_NAMESPACE": "", + "SD_TEMPLATE_VERSION": "1.2.3" + }, + "image": "golang", + "requires": [ + "main" + ], + "secrets": [ + "GIT_KEY" + ], + "settings": { + "email": "foo@example.com" + }, + "sourcePaths": [], + "steps": [ + { + "install": "npm install" + }, + { + "test": "npm test" + } + ], + "templateId": 7754 + } + }, + "parameters": { + "nameA": "value1" + }, + "subscribe": { + "scmUrls": [ + { + "https://github.com/VonnyJap/python-zero-to-hero.git": [ + "~pr" + ] + }, + { + "https://github.com/VonnyJap/sshca.git": [ + "~pr" + ] + } + ] + } + }, + "description": "An example pipeline template for testing golang files", + "maintainer": "foo@bar.com", + "name": "example-template", + "namespace": "sd-test", + "version": "1.0.0" +} \ No newline at end of file diff --git a/test/data/validate-pipeline-template-with-job-template.yaml b/test/data/validate-pipeline-template-with-job-template.yaml new file mode 100644 index 0000000..0cfca4c --- /dev/null +++ b/test/data/validate-pipeline-template-with-job-template.yaml @@ -0,0 +1,31 @@ +namespace: sd-test +name: example-template +version: '1.0.0' +description: An example pipeline template for testing golang files +maintainer: foo@bar.com +config: + parameters: + nameA: "value1" + annotations: + screwdriver.cd/restrictPR: none + screwdriver.cd/chainPR: false + subscribe: + scmUrls: + - https://github.com/VonnyJap/python-zero-to-hero.git: ['~pr'] + - https://github.com/VonnyJap/sshca.git: ['~pr'] + shared: + image: golang + environment: + FOO: "BAR" + requires: [main] + steps: + - name: echo "bang" + jobs: + main: + template: sd/noop@1.0.0 + extra: + image: node:20 + steps: + - name: echo "pipeline template test" + other: + template: sd/noop@1.0.0 \ No newline at end of file diff --git a/test/index.test.js b/test/index.test.js index 032aed8..1a12680 100644 --- a/test/index.test.js +++ b/test/index.test.js @@ -4,7 +4,7 @@ const { assert } = require('chai'); const fs = require('fs'); const path = require('path'); const sinon = require('sinon'); -const { parsePipelineTemplate, parsePipelineYaml: parser } = require('..'); +const { parsePipelineTemplate, parsePipelineYaml: parser, validatePipelineTemplate } = require('..'); const pipelineId = 111; sinon.assert.expose(assert, { prefix: '' }); @@ -260,8 +260,6 @@ describe('config parser', () => { triggerFactory, pipelineId }).then(data => { - console.log(data); - console.log(data.errors); assert.match( data.errors[0], /Error: main job has invalid requires: baz. Triggers must be jobs from canary stage./ @@ -736,7 +734,7 @@ describe('config parser', () => { pipelineTemplateTagFactory: pipelineTemplateTagFactoryMock, pipelineTemplateVersionFactory: pipelineTemplateVersionFactoryMock }).then(data => { - assert.deepEqual(data.errors[0], 'ValidationError: "jobs" is not allowed'); + assert.deepEqual(data.errors[0], 'ValidationError: "jobs.main.steps" is not allowed'); })); it('returns error if pipeline template does not exist', () => { @@ -1146,4 +1144,21 @@ describe('config parser', () => { assert.match(err.toString(), /[ValidationError]: "config.jobs" is required/); })); }); + + describe('validate pipeline template', () => { + it('flattens pipeline template for the validator and pulls in job template steps', () => + validatePipelineTemplate({ + yaml: loadData('validate-pipeline-template-with-job-template.yaml'), + templateFactory: templateFactoryMock + }).then(data => { + assert.deepEqual(data, JSON.parse(loadData('validate-pipeline-template-with-job-template.json'))); + })); + it('throws error if pipeline template is invalid', () => + validatePipelineTemplate({ + yaml: loadData('validate-pipeline-template-invalid.yaml'), + templateFactory: templateFactoryMock + }).then(assert.fail, err => { + assert.match(err.toString(), /[ValidationError]: "config.jobs" is required/); + })); + }); });