Skip to content

Commit 8ab8b9f

Browse files
itai-codefreshoren-codefresh
itai-codefresh
authored and
oren-codefresh
committed
refactor run pipeline command (#265)
* refactor run pipeline command * support passing variables on command line when running local engine * WIP * add progress and fix defects * Support no cache and reset volume for local run * no cf cache support * Fix context defect * fixed noisy docker pull * Fixed pulling engine log * Defect fixing * Support enable notifications when running locally * Upgrade version * merge
1 parent 581e626 commit 8ab8b9f

File tree

7 files changed

+322
-170
lines changed

7 files changed

+322
-170
lines changed

‎lib/interface/cli/commands/pipeline/run.base.js

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
const _ = require('lodash');
2+
const Promise = require('bluebird');
3+
const { prepareKeyValueFromCLIEnvOption } = require('../../helpers/general');
4+
const { pipeline, log } = require('../../../../logic').api;
5+
const { validatePipelineYaml } = require('../../helpers/validation');
6+
const { printResult } = require('../root/validate.cmd');
7+
const CFError = require('cf-errors');
8+
9+
class RunBaseCommand {
10+
constructor(argv) {
11+
this.argv = argv;
12+
this.executionRequests = [];
13+
}
14+
async run() {
15+
await this.preRunRequest();
16+
const variablesFromFile = this.argv['var-file'];
17+
const pipelineName = this.argv.name;
18+
const { branch, sha } = this.argv;
19+
const noCache = this.argv['no-cache'];
20+
const enableNotifications = this.argv['enable-notifications'];
21+
const resetVolume = this.argv['reset-volume'];
22+
const userYamlDescriptor = this.argv.yaml;
23+
const contexts = this.argv.context;
24+
const noCfCache = this.argv['no-cf-cache'];
25+
26+
const executionRequestTemplate = {
27+
pipelineName,
28+
options: {
29+
noCache,
30+
resetVolume,
31+
branch,
32+
sha,
33+
enableNotifications,
34+
userYamlDescriptor,
35+
noCfCache,
36+
},
37+
};
38+
if (variablesFromFile) {
39+
_.forEach(variablesFromFile, (variables) => {
40+
const request = _.cloneDeep(executionRequestTemplate);
41+
request.options.variables = variables;
42+
this.executionRequests.push(request);
43+
});
44+
} else {
45+
const variables = prepareKeyValueFromCLIEnvOption(this.argv.variable);
46+
const request = _.cloneDeep(executionRequestTemplate);
47+
request.options.variables = variables;
48+
request.options.contexts = contexts;
49+
this.executionRequests.push(request);
50+
}
51+
52+
const results = await Promise.all(this.executionRequests.map(request => this.runImpl(request)));
53+
const findMaxReducer = (accumulator, currentValue) => (currentValue > accumulator ? currentValue : accumulator);
54+
const exitCode = results.reduce(findMaxReducer);
55+
await this.postRunRequest();
56+
return exitCode;
57+
// let p = Promise.resolve();
58+
// this.executionRequests.forEach(request => p = p.then(() => this.runImpl(request)));
59+
// await p;
60+
}
61+
62+
// eslint-disable-next-line class-methods-use-this
63+
async runImpl() {
64+
throw new Error('To be implemented in the derived class');
65+
}
66+
// eslint-disable-next-line class-methods-use-this
67+
async preRunAll() {
68+
const pipelineName = this.argv.name;
69+
const userYamlDescriptor = this.argv.yaml;
70+
try {
71+
await pipeline.getPipelineByName(pipelineName);
72+
} catch (err) {
73+
throw new CFError({
74+
message: `Passed pipeline id: ${pipelineName} does not exist`,
75+
});
76+
}
77+
78+
if (userYamlDescriptor) {
79+
const result = await validatePipelineYaml(undefined, userYamlDescriptor);
80+
printResult(result);
81+
}
82+
}
83+
// eslint-disable-next-line class-methods-use-this
84+
get isParalel() {
85+
return true;
86+
}
87+
// eslint-disable-next-line class-methods-use-this
88+
async preRunRequest() {
89+
return Promise.resolve();
90+
}
91+
// eslint-disable-next-line class-methods-use-this
92+
async postRunRequest() {
93+
return Promise.resolve();
94+
}
95+
// eslint-disable-next-line class-methods-use-this
96+
async postRunAll() {
97+
return Promise.resolve();
98+
}
99+
async writeToLog(workflowId) {
100+
if (this.executionRequests.length === 1) {
101+
if (this.argv.detach) {
102+
console.log(workflowId);
103+
} else {
104+
await log.showWorkflowLogs(workflowId, true);
105+
}
106+
} else {
107+
console.log(workflowId);
108+
}
109+
}
110+
}
111+
module.exports = RunBaseCommand;

‎lib/interface/cli/commands/pipeline/run.cf.js

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
const { workflow, pipeline } = require('../../../../logic').api;
2+
const RunBaseCommand = require('./run.base');
3+
4+
class RunExternalCommand extends RunBaseCommand {
5+
async runImpl(request) {
6+
const { pipelineName, options } = request;
7+
this.workflowId = await pipeline.runPipelineByName(pipelineName, options);
8+
await this.writeToLog(this.workflowId);
9+
const workflowInstance = await workflow.getWorkflowById(this.workflowId);
10+
switch (workflowInstance.getStatus()) {
11+
case 'success':
12+
return 0;
13+
case 'error':
14+
return 1;
15+
case 'terminated':
16+
return 2;
17+
default:
18+
return 100;
19+
}
20+
}
21+
}
22+
module.exports = RunExternalCommand;

‎lib/interface/cli/commands/pipeline/run.cmd.js

Lines changed: 18 additions & 168 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,16 @@
11
const debug = require('debug')('codefresh:cli:run:pipeline');
22
const Command = require('../../Command');
3-
const _ = require('lodash');
4-
const CFError = require('cf-errors');
5-
const { prepareKeyValueFromCLIEnvOption, crudFilenameOption } = require('../../helpers/general');
6-
const { workflow, pipeline, log } = require('../../../../logic').api;
7-
const authManager = require('../../../../logic').auth.manager;
8-
const Docker = require('dockerode');
9-
const { validatePipelineYaml } = require('../../helpers/validation');
10-
const { printResult } = require('../root/validate.cmd');
11-
const DEFAULTS = require('../../defaults');
12-
const path = require('path');
13-
const regex = /##[0-9a-f]{24}##/i;
14-
const imageName = 'codefresh/engine:master';
3+
const { crudFilenameOption } = require('../../helpers/general');
4+
const RunLocalCommand = require('./run.local');
5+
const RunExternalCommand = require('./run.cf');
156

16-
17-
function _customizer(objValue, srcValue) {
18-
if (Array.isArray(objValue)) {
19-
return _.compact(objValue.concat(srcValue));
7+
function getCommandFlavor(argv) {
8+
if (argv.local) {
9+
return new RunLocalCommand(argv);
2010
}
11+
return new RunExternalCommand(argv);
2112
}
13+
2214
const run = new Command({
2315
root: true,
2416
command: 'run <name>',
@@ -80,6 +72,11 @@ const run = new Command({
8072
describe: 'Use your file system as volume in local run',
8173
alias: 'lv',
8274
})
75+
.option('no-cf-cache', {
76+
describe: 'Ignore Codefresh cache optimizations',
77+
alias: 'ncfc',
78+
default: false,
79+
})
8380
.example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master', 'Defining the source control context using a branch')
8481
.example('codefresh run PIPELINE_ID | PIPELINE_NAME -s=52b992e783d2f84dd0123c70ac8623b4f0f938d1', 'Defining the source control context using a commit')
8582
.example('codefresh run PIPELINE_ID | PIPELINE_NAME -b=master -v key1=value1 -v key2=value2', 'Setting variables through the command')
@@ -102,158 +99,11 @@ const run = new Command({
10299
return yargs;
103100
},
104101
handler: async (argv) => {
105-
const pipelineName = argv.name;
106-
const branch = argv.branch;
107-
const sha = argv.sha;
108-
const noCache = argv['no-cache'];
109-
const enableNotifications = argv['enable-notifications'];
110-
const resetVolume = argv['reset-volume'];
111-
const variablesFromFile = argv['var-file'];
112-
const contexts = argv.context;
113-
const userYamlDescriptor = argv.yaml;
114-
const local = argv.local;
115-
const localVolume = argv['local-volume'] === true ? path.join(DEFAULTS.CODEFRESH_PATH, pipelineName) : argv['local-volume'];
116-
117-
try {
118-
await pipeline.getPipelineByName(pipelineName);
119-
} catch (err) {
120-
throw new CFError({
121-
message: `Passed pipeline id: ${pipelineName} does not exist`,
122-
});
123-
}
124-
125-
if (userYamlDescriptor) {
126-
const result = await validatePipelineYaml(undefined, userYamlDescriptor);
127-
printResult(result);
128-
}
129-
130-
if (local) {
131-
const docker = new Docker();
132-
const cleanupActions = [];
133-
const injectedOpts = {};
134-
// TODO : Move to per command's handler so each command will be handled in a seperate handler
135-
if (userYamlDescriptor) {
136-
injectedOpts.Env = [`OVERRIDE_WORKFLOW_YAML=${userYamlDescriptor}`];
137-
}
138-
139-
console.log(`Updating Codefresh engine ==>`);
140-
docker.pull(imageName, (err, stream) => {
141-
docker.modem.followProgress(stream, onFinished, onProgress);
142-
function onFinished(err) {
143-
if (!err) {
144-
console.log('Finished Update.\n');
145-
146-
if (localVolume) {
147-
injectedOpts.Env = [`EXTERNAL_WORKSPACE=${localVolume}`];
148-
console.log(`\nUsing ${localVolume} as a local volume.\n`);
149-
}
150-
151-
const currentContext = authManager.getCurrentContext();
152-
console.log(`Running pipeline: ${pipelineName}`);
153-
154-
docker.run(imageName, [], [], _.mergeWith({
155-
Env: [
156-
`ACCESS_TOKEN=${currentContext.token}`,
157-
`PIPELINE_ID=${pipelineName}`, `BRANCH=${branch}`,
158-
`CF_HOST=${currentContext.url}`,
159-
'DOCKER_SOCKET_PATH=/var/run/docker.sock',
160-
'NO_EXT_MONITOR=true',
161-
],
162-
Hostconfig: {
163-
Binds: [
164-
'/var/run/docker.sock:/var/run/docker.sock',
165-
],
166-
},
167-
}, injectedOpts, _customizer), (err, data) => {
168-
cleanupActions.forEach((action) => {
169-
try {
170-
action();
171-
} catch (error) {
172-
console.error(err);
173-
}
174-
});
175-
if (err) {
176-
return console.error(err);
177-
}
178-
process.exit(data.StatusCode);
179-
}).on('stream', (stream) => {
180-
stream.on('data', (chunk) => {
181-
const line = chunk.toString();
182-
const include = line.match(regex);
183-
if (include) {
184-
const workflowId = include[0].substring(2, include[0].length - 2);
185-
log.showWorkflowLogs(workflowId, true)
186-
.then(() => Promise.resolve());
187-
}
188-
});
189-
});
190-
} else {
191-
console.log(err);
192-
process.exit(1);
193-
}
194-
}
195-
function onProgress() {
196-
stream.pipe(process.stdout);
197-
}
198-
});
199-
} else {
200-
const executionRequests = [];
201-
const executionRequestTemplate = {
202-
pipelineName,
203-
options: {
204-
noCache,
205-
resetVolume,
206-
branch,
207-
sha,
208-
enableNotifications,
209-
userYamlDescriptor,
210-
},
211-
};
212-
213-
if (variablesFromFile) {
214-
_.forEach(variablesFromFile, (variables) => {
215-
const request = _.cloneDeep(executionRequestTemplate);
216-
request.options.variables = variables;
217-
executionRequests.push(request);
218-
});
219-
} else {
220-
const variables = prepareKeyValueFromCLIEnvOption(argv.variable);
221-
const request = _.cloneDeep(executionRequestTemplate);
222-
request.options.variables = variables;
223-
request.options.contexts = contexts;
224-
executionRequests.push(request);
225-
}
226-
227-
_.forEach(executionRequests, async ({ pipelineName, options }) => {
228-
let workflowId;
229-
workflowId = await pipeline.runPipelineByName(pipelineName, options);
230-
231-
if (executionRequests.length === 1) {
232-
if (argv.detach) {
233-
console.log(workflowId);
234-
} else {
235-
await log.showWorkflowLogs(workflowId, true);
236-
const workflowInstance = await workflow.getWorkflowById(workflowId);
237-
switch (workflowInstance.getStatus()) {
238-
case 'success':
239-
process.exit(0);
240-
break;
241-
case 'error':
242-
process.exit(1);
243-
break;
244-
case 'terminated':
245-
process.exit(2);
246-
break;
247-
default:
248-
process.exit(100);
249-
break;
250-
}
251-
}
252-
} else {
253-
console.log(workflowId);
254-
}
255-
});
256-
}
102+
const flavor = getCommandFlavor(argv);
103+
await flavor.preRunAll();
104+
const exitCode = await flavor.run();
105+
await flavor.postRunAll();
106+
process.exit(exitCode);
257107
},
258108
});
259109

0 commit comments

Comments
 (0)