feat(3124): Merge job template steps for pipeline template validation [1]

index.js

@@ -15,7 +15,8 @@ const phaseMerge = require('./lib/phase/merge');
 const {
     flattenSharedIntoJobs,
     handleMergeSharedStepsAnnotation,
-    flattenPhase: phaseFlatten
+    flattenPhase: phaseFlatten,
+    flattenTemplates
 } = require('./lib/phase/flatten');
 const phaseValidateFunctionality = require('./lib/phase/functional');
 const phaseGeneratePermutations = require('./lib/phase/permutation');
@@ -314,7 +315,26 @@ async function parsePipelineTemplate({ yaml }) {
     return pipelineTemplate;
 }
 
+/**
+ * Generates pipeline template configuration for the validator
+ * @method validatePipelineTemplate
+ * @param   {Object}          config
+ * @param   {String}          config.yaml             Pipeline Template
+ * @param   {TemplateFactory} config.templateFactory  Template Factory to get templates
+ * @return  {Object}                                  Pipeline Template
+ */
+async function validatePipelineTemplate({ yaml, templateFactory }) {
+    const pipelineTemplate = await parsePipelineTemplate({ yaml });
+    // Merge template steps for validator
+    const { newJobs } = await flattenTemplates(pipelineTemplate.config, templateFactory, true);
+
+    pipelineTemplate.config.jobs = newJobs;
+
+    return pipelineTemplate;
+}
+
 module.exports = {
     parsePipelineYaml,
-    parsePipelineTemplate
+    parsePipelineTemplate,
+    validatePipelineTemplate
 };

lib/phase/flatten.js

@@ -358,9 +358,18 @@ function handleMergeSharedStepsAnnotation({ sharedConfig, jobConfig, template })
  * @param  {TemplateFactory}  templateFactory       Template Factory to get templates
  * @param  {Object}           sharedConfig          Shared configuration
  * @param  {Object}           pipelineParameters    Pipeline level parameters
+ * @param  {Boolean}          [isPipelineTemplate]  If the current template is pipeline template or not
  * @return {Promise}
  */
-function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sharedConfig, pipelineParameters }) {
+async function mergeTemplateIntoJob({
+    jobName,
+    jobConfig,
+    newJobs,
+    templateFactory,
+    sharedConfig,
+    pipelineParameters,
+    isPipelineTemplate
+}) {
     let oldJob = jobConfig;
 
     // Try to get the template
@@ -390,7 +399,9 @@ function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sh
         let warnings = [];
 
         // merge shared steps into oldJob
-        oldJob = handleMergeSharedStepsAnnotation({ sharedConfig, jobConfig: oldJob, template });
+        if (!isPipelineTemplate) {
+            oldJob = handleMergeSharedStepsAnnotation({ sharedConfig, jobConfig: oldJob, template });
+        }
 
         // Include parameters from the template only if it not overwritten either in pipeline or job parameters
         if (newJob.parameters !== undefined) {
@@ -409,7 +420,7 @@ function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sh
             } else {
                 delete newJob.parameters;
             }
-        } else {
+        } else if (oldJob.parameters !== undefined) {
             newJob.parameters = oldJob.parameters;
         }
 
@@ -435,11 +446,12 @@ function mergeTemplateIntoJob({ jobName, jobConfig, newJobs, templateFactory, sh
  * Goes through each job and if template is specified, then merge into job config
  *
  * @method flattenTemplates
- * @param  {Object}           doc               Document that went through structural parsing
- * @param  {TemplateFactory}  templateFactory   Template Factory to get templates
+ * @param  {Object}           doc                   Document that went through structural parsing
+ * @param  {TemplateFactory}  templateFactory       Template Factory to get templates
+ * @param  {Boolean}          [isPipelineTemplate]  If the current template is pipeline template or not
  * @return {Promise}          Resolves to new object with jobs after merging templates
  */
-function flattenTemplates(doc, templateFactory) {
+async function flattenTemplates(doc, templateFactory, isPipelineTemplate) {
     const newJobs = {};
     const templates = [];
     const { jobs, shared, parameters } = doc;
@@ -466,7 +478,8 @@ function flattenTemplates(doc, templateFactory) {
                     newJobs,
                     templateFactory,
                     sharedConfig: shared,
-                    pipelineParameters: parameters
+                    pipelineParameters: parameters,
+                    isPipelineTemplate
                 })
             );
         } else {
@@ -754,5 +767,6 @@ function flattenPhase(parsedDoc, templateFactory) {
 module.exports = {
     flattenPhase,
     flattenSharedIntoJobs,
-    handleMergeSharedStepsAnnotation
+    handleMergeSharedStepsAnnotation,
+    flattenTemplates
 };

package.json

@@ -56,7 +56,7 @@
     "joi": "^17.7.0",
     "js-yaml": "^4.1.0",
     "keymbinatorial": "^2.0.0",
-    "screwdriver-data-schema": "^23.3.1",
+    "screwdriver-data-schema": "^23.3.2",
     "screwdriver-notifications-email": "^3.0.1",
     "screwdriver-notifications-slack": "^5.0.0",
     "screwdriver-workflow-parser": "^4.3.0",

test/data/pipeline-template-invalid.yaml

@@ -6,5 +6,4 @@ jobs:
             - install: npm install
         requires:
             - ~pr
-            - ~commit
-
+            - ~commit  

test/data/validate-pipeline-template-invalid.yaml

@@ -0,0 +1,14 @@
+namespace: template_namespace
+name: template_name
+version: 1.2.3
+description: template description
+maintainer: [email protected]
+config:
+  shared:
+    image: node:20
+    environment:
+      FOO: foo
+  parameters:
+    user:
+      value: sd-bot
+      description: User running build

test/data/validate-pipeline-template-with-job-template.json

@@ -0,0 +1,112 @@
+{
+  "config": {
+    "annotations": {
+      "screwdriver.cd/chainPR": false,
+      "screwdriver.cd/restrictPR": "none"
+    },
+    "jobs": {
+      "extra": {
+        "annotations": {},
+        "environment": {
+          "FOO": "BAR"
+        },
+        "image": "node:20",
+        "requires": [
+          "main"
+        ],
+        "secrets": [],
+        "settings": {},
+        "sourcePaths": [],
+        "steps": [
+          {
+            "name": "echo \"pipeline template test\""
+          }
+        ]
+      },
+      "main": {
+        "annotations": {},
+        "environment": {
+          "BAR": "foo",
+          "FOO": "BAR",
+          "SD_TEMPLATE_FULLNAME": "mytemplate",
+          "SD_TEMPLATE_NAME": "mytemplate",
+          "SD_TEMPLATE_NAMESPACE": "",
+          "SD_TEMPLATE_VERSION": "1.2.3"
+        },
+        "image": "golang",
+        "requires": [
+          "main"
+        ],
+        "secrets": [
+          "GIT_KEY"
+        ],
+        "settings": {
+          "email": "[email protected]"
+        },
+        "sourcePaths": [],
+        "steps": [
+          {
+            "install": "npm install"
+          },
+          {
+            "test": "npm test"
+          }
+        ],
+        "templateId": 7754
+      },
+      "other": {
+        "annotations": {},
+        "environment": {
+          "BAR": "foo",
+          "FOO": "BAR",
+          "SD_TEMPLATE_FULLNAME": "mytemplate",
+          "SD_TEMPLATE_NAME": "mytemplate",
+          "SD_TEMPLATE_NAMESPACE": "",
+          "SD_TEMPLATE_VERSION": "1.2.3"
+        },
+        "image": "golang",
+        "requires": [
+          "main"
+        ],
+        "secrets": [
+          "GIT_KEY"
+        ],
+        "settings": {
+          "email": "[email protected]"
+        },
+        "sourcePaths": [],
+        "steps": [
+          {
+            "install": "npm install"
+          },
+          {
+            "test": "npm test"
+          }
+        ],
+        "templateId": 7754
+      }
+    },
+    "parameters": {
+      "nameA": "value1"
+    },
+    "subscribe": {
+      "scmUrls": [
+        {
+          "https://github.com/VonnyJap/python-zero-to-hero.git": [
+            "~pr"
+          ]
+        },
+        {
+          "https://github.com/VonnyJap/sshca.git": [
+            "~pr"
+          ]
+        }
+      ]
+    }
+  },
+  "description": "An example pipeline template for testing golang files",
+  "maintainer": "[email protected]",
+  "name": "example-template",
+  "namespace": "sd-test",
+  "version": "1.0.0"
+}

test/data/validate-pipeline-template-with-job-template.yaml

@@ -0,0 +1,31 @@
+namespace: sd-test
+name: example-template
+version: '1.0.0'
+description: An example pipeline template for testing golang files
+maintainer: [email protected]
+config:
+  parameters:
+    nameA: "value1"
+  annotations:
+    screwdriver.cd/restrictPR: none
+    screwdriver.cd/chainPR: false
+  subscribe:
+    scmUrls:
+    - https://github.com/VonnyJap/python-zero-to-hero.git: ['~pr']
+    - https://github.com/VonnyJap/sshca.git: ['~pr']
+  shared:
+    image: golang
+    environment:
+      FOO: "BAR"
+    requires: [main]
+    steps:
+      - name: echo "bang"
+  jobs:
+    main:
+      template: sd/[email protected]
+    extra:
+      image: node:20
+      steps:
+        - name: echo "pipeline template test"
+    other:
+      template: sd/[email protected]

test/index.test.js

@@ -4,7 +4,7 @@ const { assert } = require('chai');
 const fs = require('fs');
 const path = require('path');
 const sinon = require('sinon');
-const { parsePipelineTemplate, parsePipelineYaml: parser } = require('..');
+const { parsePipelineTemplate, parsePipelineYaml: parser, validatePipelineTemplate } = require('..');
 const pipelineId = 111;
 
 sinon.assert.expose(assert, { prefix: '' });
@@ -260,8 +260,6 @@ describe('config parser', () => {
                         triggerFactory,
                         pipelineId
                     }).then(data => {
-                        console.log(data);
-                        console.log(data.errors);
                         assert.match(
                             data.errors[0],
                             /Error: main job has invalid requires: baz. Triggers must be jobs from canary stage./
@@ -736,7 +734,7 @@ describe('config parser', () => {
                         pipelineTemplateTagFactory: pipelineTemplateTagFactoryMock,
                         pipelineTemplateVersionFactory: pipelineTemplateVersionFactoryMock
                     }).then(data => {
-                        assert.deepEqual(data.errors[0], 'ValidationError: "jobs" is not allowed');
+                        assert.deepEqual(data.errors[0], 'ValidationError: "jobs.main.steps" is not allowed');
                     }));
 
                 it('returns error if pipeline template does not exist', () => {
@@ -1146,4 +1144,21 @@ describe('config parser', () => {
                 assert.match(err.toString(), /[ValidationError]: "config.jobs" is required/);
             }));
     });
+
+    describe('validate pipeline template', () => {
+        it('flattens pipeline template for the validator and pulls in job template steps', () =>
+            validatePipelineTemplate({
+                yaml: loadData('validate-pipeline-template-with-job-template.yaml'),
+                templateFactory: templateFactoryMock
+            }).then(data => {
+                assert.deepEqual(data, JSON.parse(loadData('validate-pipeline-template-with-job-template.json')));
+            }));
+        it('throws error if pipeline template is invalid', () =>
+            validatePipelineTemplate({
+                yaml: loadData('validate-pipeline-template-invalid.yaml'),
+                templateFactory: templateFactoryMock
+            }).then(assert.fail, err => {
+                assert.match(err.toString(), /[ValidationError]: "config.jobs" is required/);
+            }));
+    });
 });