From c348720891334b460eca98e7a60e1c6d50912aa2 Mon Sep 17 00:00:00 2001 From: Clint Zirker Date: Tue, 19 Dec 2017 10:27:40 -0700 Subject: [PATCH] Version 1.0.4 --- leo-cli-build.js | 273 +++++++++++++++++++++++++++++++++++++++++++++ leo-cli-publish.js | 118 ++++++++++++++++++++ leo-cli-test.js | 197 ++++++++++++++++++++++++++++++++ leo-cli.js | 10 ++ 4 files changed, 598 insertions(+) create mode 100644 leo-cli-build.js create mode 100644 leo-cli-publish.js create mode 100644 leo-cli-test.js create mode 100644 leo-cli.js diff --git a/leo-cli-build.js b/leo-cli-build.js new file mode 100644 index 0000000..685aa7b --- /dev/null +++ b/leo-cli-build.js @@ -0,0 +1,273 @@ +#!/usr/bin/env node + +var path = require('path'); +// var fs = require('fs'); +var program = require('commander'); +var colors = require('colors'); +// var archiver = require('archiver'); +// var aws = require("aws-sdk"); +// var glob = require("glob"); +// var CopyWebpackPlugin = require('copy-webpack-plugin'); + +// var babelify = require("babelify"); + +// var spawn = require('child_process').spawn; +// var execsync = require('child_process').execSync; +// var through = require('through2'); + +// var async = require("async"); +// var moment = require("moment"); +// var leo = require("leo-sdk"); + +// var configure; + +// //Build Stuff +// var browserify = require('browserify'); +// var gulp = require("gulp"); +// var source = require('vinyl-source-stream'); +// var buffer = require('vinyl-buffer'); +// var gutil = require('gulp-util'); +// var uglify = require('gulp-uglify'); +// var rename = require('gulp-rename'); +// var ejs = require("gulp-ejs"); +// var buildConfig = require("./lib/build-config").build; + +// //webpack +// var webpack = require('webpack'); +// var ExtractTextPlugin = require("extract-text-webpack-plugin"); +var cmds = require("./lib/build.js") +program + .version('0.0.1') + .option("-e, --env [env]", "Environment") + .option("-r, --region [region]", "AWS Region") + .option("-p, --profile [profile]", "AWS Profile") + .arguments(' [alias] [region]') + .usage(' [alias] [region] [options]') + .action(function (dir, alias, region) { + + var rootDir = path.resolve(process.cwd(), "./" + dir); + cmds.build(program, rootDir, { + alias, + region + }, (err) => { + + }); + // alias = program.env || alias; + // region = program.region || region + // console.log("Build Alias:", alias || "dev"); + // process.env.LEO_ENV = alias || "dev"; + // process.env.LEO_REGION = region; + + // var rootDir = path.resolve(process.cwd(), "./" + dir); + // configure = buildConfig(rootDir); + + // configure.aws = configure.aws || {}; + // region = configure._meta.region; + // if (program.profile) { + // console.log("Using cli profile", program.profile) + // configure.aws.profile = program.profile; + // } + // if (configure.aws.profile) { + // console.log("Setting profile to", configure.aws.profile); + // var credentials = new aws.SharedIniFileCredentials({ + // profile: configure.aws.profile + // }); + // aws.config.credentials = credentials; + // process.env.AWS_DEFAULT_PROFILE = configure.aws.profile; + // } + + // console.log("Setting region to", region); + + // var deployAlias = alias ? alias.toLowerCase() : 'dev'; + + // var pkg = require(path.resolve(rootDir, "package.json")); + + + // var config = configure; + + // if (config.type == "bot" || config.type == "cron" || config.type == "resource" || config.type == "apigateway") { + // buildLambdaDirectory(rootDir, {}, function (err, data) { + // console.log(err, data); + // }); + // } else if (config.type == "package") { + // let cloudformation = fs.readFileSync(path.resolve(rootDir, "cloudformation.json"), { + // encoding: "utf-8" + // }); + // let lambdas = config.lambdas || []; + // let dir = `${config.name}-${moment.now()}`; + // async.mapLimit(lambdas, 5, (lambdaDir, done) => { + // buildLambdaDirectory(path.resolve(rootDir, lambdaDir), { + // dir: dir + // }, (err, data) => { + // //setTimeout(() => done(err, data), 500) + // done(err, data) + // }); + // }, err => { + // console.log("All lambda zip files completed", err || undefined); + // console.log("Create cloudformation"); + // let cfPath = path.resolve(rootDir, "cloudformation.json"); + // if (fs.existsSync(cfPath)) { + // let cf = require(cfPath); + // console.log(config) + // cf.Resources.LEO_VARS = { + // Type: "Custom::LeoVariables", + // Version: "1.0", + // Properties: { + // "ServiceToken": { + // "Fn::Join": [ + // "", ["arn:aws:lambda:", { + // "Ref": "AWS::Region" + // }, ":", { + // "Ref": "AWS::AccountId" + // }, ":leo:vars"] + // ] + // }, + // bucket: leo.configuration.bus.s3, + // version: moment.now().toString() + // } + // }; + + // fs.writeFileSync(`/tmp/${dir}/cloudformation.js`, JSON.stringify(cf, null, 2)); + + // //console.log(`Uploading static files ${staticDir} to ${config.staticS3}/${newVersion}`); + // //spawn('aws', ['s3', 'sync', staticDir, `${config.staticS3}/${newVersion}`]); + // } + + // }); + // } else { + // console.log("Unknown config.leo.type, not in (bot, cron, resource, microservice)"); + // } + }) + .parse(process.argv); + +if (!process.argv.slice(2).length) { + program.outputHelp(colors.red); +} + + +// function buildLambdaDirectory(rootDir, config, callback) { +// var folder = config.dir ? `tmp/${config.dir}` : "tmp"; +// if (!fs.existsSync(`/${folder}`)) { +// fs.mkdirSync(`/${folder}`); +// } +// var config = buildConfig(rootDir); +// console.log("Run build on", rootDir) +// execsync("npm install", { +// cwd: rootDir +// }); +// console.log(`Zipping Lambda Function ${config.name}`); +// var archive = archiver('zip'); +// var zipFilename = `/${folder}/${config.name}.zip`; +// var indexFilename = `${config.name}-index-${moment.now()}.js`; +// var zip = fs.createWriteStream(zipFilename); +// archive.pipe(zip); + +// var b = browserify({ +// standalone: 'lambda', +// bare: true, +// entries: [__dirname + "/lib/leowrap.js"], +// browserField: false, +// builtins: false, +// commondir: false, +// detectGlobals: true, +// insertGlobalVars: { +// process: function () { +// return; +// } +// }, +// debug: true +// }); +// b.transform(babelify, { +// presets: ["es2015"], +// sourceMaps: false +// }); +// b.external("aws-sdk"); + +// if (config.build && config.build.include) { +// for (var i = 0; i < config.build.include.length; i++) { +// var inc = config.build.include[i]; +// var src = inc.src || inc; +// var dest = inc.dest || "node_modules/"; + +// src = path.resolve(rootDir, src); +// b.external(path.basename(src)); + +// if (fs.lstatSync(src).isDirectory()) { +// execsync("npm install", { +// cwd: src +// }); +// } +// archive.directory(path.normalize(src), path.join(dest, path.basename(src))); + +// } +// } +// b.transform(function (file) { +// if (file.match("leowrap")) { +// return through(function (buf, enc, next) { +// next(null, ""); +// }, function (cb) { +// var type = config.type; +// if (config.cron && typeof config.cron == "object" && config.cron.autoType !== false) { +// type = "cron"; +// } +// var wrapperFile = __dirname + "/lib/wrappers/" + type + ".js"; +// if (!fs.existsSync(wrapperFile)) { +// wrapperFile = __dirname + "/lib/wrappers/base.js"; +// } +// var contents = fs.readFileSync(wrapperFile, 'utf-8') +// .replace("____FILE____", path.normalize(path.resolve(rootDir, "index.js")).replace(/\\/g, "\\\\")); +// this.push(contents); +// cb(); +// }); +// } else if (file.match("leoConfigure.js")) { +// return through(function (buf, enc, next) { +// next(null, ""); +// }, function (cb) { +// this.push("module.exports = " + JSON.stringify(config)); +// cb(); +// }); +// } else if (file.match("leo-sdk-config.js")) { + +// return through(function (buf, enc, next) { +// next(null, ""); +// }, function (cb) { +// var sdkConfigData = "{}"; +// var sdkConfigPath = path.resolve(`${require('os').homedir()}/.leo`, "config.json") +// if (fs.existsSync(sdkConfigPath)) { +// sdkConfigData = JSON.parse(fs.readFileSync(sdkConfigPath) || sdkConfigData); +// } +// // Can't Change aws profile in lambda so remove the profile key +// Object.keys(sdkConfigData).map(k => delete sdkConfigData[k].profile); +// this.push("module.exports = " + JSON.stringify(sdkConfigData)); +// cb(); +// }); +// } else { +// return through(); +// } +// }, { +// global: true +// }); +// b.bundle().pipe(source(indexFilename)).pipe(buffer()) +// .pipe(gulp.dest(`/${folder}/`)).on("end", () => { +// console.log("done building"); +// archive.file(`/${folder}/${indexFilename}`, { +// name: "index.js" +// }); +// if (config.files) { +// for (var file in config.files) { +// archive.file(config.files[file], { +// name: file +// }); +// } +// } +// zip.on("close", function () { +// console.log("Created zip"); +// fs.unlinkSync(`/${folder}/${indexFilename}`); +// callback(null, { +// config: config, +// path: zipFilename +// }); +// }); +// archive.finalize(); +// }); +// } \ No newline at end of file diff --git a/leo-cli-publish.js b/leo-cli-publish.js new file mode 100644 index 0000000..95a7f97 --- /dev/null +++ b/leo-cli-publish.js @@ -0,0 +1,118 @@ +#!/usr/bin/env node + +var path = require('path'); +var program = require('commander'); +var colors = require('colors'); +var buildConfig = require("./lib/build-config").build; +var cloudformation = require("./lib/cloud-formation.js"); +var createCloudFormation = cloudformation.createCloudFormation; +var aws = require("aws-sdk"); + +program + .version('0.0.1') + .option("-e, --env [env]", "Environment") + .option("--build", "Only build") + .option("--public", "Make published version public") + .option("--run [stack]", "Run the published cloudformation") + .option("--patch [stack]", "Stack to get original cloudformation") + .option("--region [region]", "Region to run cloudformation") + .option("--force [bots]", "Force bots to publish") + .option("--filter [bots]", "Filter bots to publish") + .usage(' [options]') + .action(function (dir) { + let env = program.env || "dev"; + // console.log(env) + let rootDir = path.resolve(process.cwd(), dir); + + let configure = buildConfig(rootDir); + + let filter = program.filter; + let force = program.force; + if (configure.type !== "microservice") { + filter = rootDir.replace(/^.*?(bots|api)[\\/]/, "") + force = filter; + rootDir = configure._meta.microserviceDir; + configure = buildConfig(rootDir); + } + + if (configure.aws.profile) { + console.log("Setting aws profile to", configure.aws.profile); + var credentials = new aws.SharedIniFileCredentials({ + profile: configure.aws.profile + }); + aws.config.credentials = credentials; + process.env.AWS_DEFAULT_PROFILE = configure.aws.profile; + } + + let nameLower = configure.name.toLowerCase(); + let folder = nameLower; + + program.region = program.region || (configure.regions || [])[0] || "us-west-2"; + + process.env.LEO_ENV = env; + process.env.LEO_REGION = program.region; + + let start = Promise.resolve(); + + if (program.patch) { + if (program.run) { + program.run = program.patch; + } + + // Get current CloudFormation for patch + start = cloudformation.get(program.patch, program.region).then(data => { + return undefined; + }); + } + + start.then(cf => createCloudFormation(configure._meta.microserviceDir, { + config: configure, + filter: filter, + publish: program.run || !program.build, + force: force, + regions: program.region ? [].concat(program.region) : configure.regions, + public: program.public, + cloudformation: cf, + overrideCloudFormationFile: !cf, + alias: process.env.LEO_ENV, + region: process.env.LEO_REGION + }).then((data) => { + + if (program.run || !program.build) { + console.log("\n---------------Publish Complete---------------"); + } else { + console.log("\n---------------Build Complete---------------"); + } + if (program.run && typeof program.run === "string") { + let bucket = data.filter(d => d.region == program.region)[0]; + let url = bucket.url + "cloudformation.json" + let updateStart = Date.now(); + console.log(`\n---------------Updating stack "${program.run}"---------------`); + console.log(`url: ${url}`); + let progress = setInterval(() => { + process.stdout.write(".") + }, 2000); + cloudformation.run(program.run, program.region, url, { + Parameters: Object.keys(bucket.cloudFormation.Parameters || {}).map(key => { + return { + ParameterKey: key, + UsePreviousValue: true + } + }) + }).then(data => { + clearInterval(progress); + console.log(` Update Complete ${Date.now() - updateStart}`); + }).catch(err => { + clearInterval(progress); + console.log(" Update Error:", err); + }); + } + })).catch(err => { + console.log(err); + }); + }) + .parse(process.argv); + +if (!process.argv.slice(2).length) { + program.outputHelp(colors.red); +} \ No newline at end of file diff --git a/leo-cli-test.js b/leo-cli-test.js new file mode 100644 index 0000000..5face68 --- /dev/null +++ b/leo-cli-test.js @@ -0,0 +1,197 @@ +#!/usr/bin/env node + +var path = require('path'); +var fs = require('fs'); +var program = require('commander'); +var colors = require('colors'); +var moment = require("moment"); +var aws = require("aws-sdk"); +let modulejs = require("module"); +let runInThisContext = require('vm').runInThisContext; + +program + .version('0.0.1') + .option("-e, --env [env]", "Environment") + .option("--region [region]", "Region to run cloudformation") + .usage(' [options]') + .action(function (dir) { + let env = program.env || "dev"; + let rootDir = path.resolve(process.cwd(), dir); + var buildConfig = require("./lib/build-config").build; + + process.env.LEO_ENV = env; + process.env.LEO_REGION = program.region; + + let config = buildConfig(rootDir); + var pkg = require(path.resolve(rootDir, "package.json")); + var type = config.type; + const packageName = pkg.name.replace(/[^a-zA-Z0-9]/g, ''); + const ID = pkg.logicalResource || packageName; + + let event = pkg.config && pkg.config.leo && pkg.config.leo.cron && pkg.config.leo.cron.settings || {}; + let eventjson = path.resolve(rootDir, "test/event.json"); + if (fs.existsSync(eventjson)) { + event = Object.assign(event, require(eventjson)); + } + + var cloudformation = new aws.CloudFormation({ + region: config.aws.region + }); + var lambda = new aws.Lambda({ + region: config.aws.region + }); + + + // cloudformation.describeStackResources({ + // StackName: "BotmonTest", + // LogicalResourceId: ID + // }, (err, data) => { + // console.log(err, data.StackResources) + // lambda.getFunctionConfiguration({ + // FunctionName: data.StackResources[0].PhysicalResourceId + // }, (err, data) => { + // event.id = data.FunctionName + // let p = data.Environment && data.Environment.Variables || {}; + // Object.keys(p).map(k => { + // let v = p[k]; + // if (typeof v !== "string") { + // v = JSON.stringify(v); + // } + // process.env[k] = v; + // }) + // console.log(err, data) + // console.log(event) + + // }) + // }); + + // return; + + // Load process info + let processjson = path.resolve(rootDir, "test/process.json"); + if (fs.existsSync(processjson)) { + let p = require(processjson); + p.env && Object.keys(p.env).map(k => { + let v = p.env[k]; + if (typeof v !== "string") { + v = JSON.stringify(v); + } + process.env[k] = v; + }) + } + + + if (config.aliases) { + let defaultAlias = Object.keys(config.aliases).map(k => config.aliases[k]).filter(a => a.default) || {}; + + + + + //let out = {}; + + // function flattenVariables(obj, out, separator, prefix) { + // prefix = prefix || ""; + // separator = separator || ":"; + // Object.keys(obj).forEach((k) => { + // var v = obj[k]; + // if (typeof v === "object" && !(Array.isArray(v)) && v !== null) { + // flattenVariables(v, out, separator, prefix + k.toLowerCase() + separator); + // } else { + // out[prefix + k.toLowerCase()] = v; + // } + // }); + // } + // flattenVariables(config.aliases, out, ".") + // config.variables = Object.assign(config.variables, out); + + + } + + //console.log(JSON.stringify(config, null, 2)); + + // setup handler + var wrapperFile = __dirname + "/lib/wrappers/" + type + ".js"; + if (!fs.existsSync(wrapperFile)) { + wrapperFile = __dirname + "/lib/wrappers/base.js"; + } + + var contents = fs.readFileSync(wrapperFile, 'utf-8') + .replace("____FILE____", path.normalize(path.resolve(rootDir, pkg.main || "index.js")).replace(/\\/g, "\\\\")) + .replace("____HANDLER____", config.handler || "handler"); + + // Compile the module to run + let r = path.normalize(path.resolve(rootDir, "__leo-cli-test-runner.js")).replace(/\\/g, "\\\\"); + let m = new modulejs(r, module); + + contents = stripBOM(contents).replace(/^\#\!.*/, ''); // remove shebang + runInThisContext(modulejs.wrap(contents), { + filename: r + }).apply(m.exports, [m.exports, require, m, r, path.dirname(r)]); + + let handler = m.exports.handler; + + var runner = { + event: event => event, + callback: (err, data, callback) => callback(err, data) + }; + var runnerFile = __dirname + "/lib/test/" + type + ".js"; + if (fs.existsSync(runnerFile)) { + runner = require(runnerFile); + } + + if (type === "bot" || type === "cron" && !event.botId) { + event.botId = event.id || (pkg.config && pkg.config.leo && pkg.config.leo.cron && pkg.config.leo.cron.id) || (pkg.config && pkg.config.leo && pkg.config.leo.name) || pkg.name; + } + + // TODO: This isn't just cron types but any bot invoked by leo + if (type === "cron" || type === "bot" || type === "raw") { + // TODO: build payload + } + + handler(runner.event(event), createContext(pkg, config), (err, data) => { + runner.callback(err, data, (err, data) => { + console.log("\n\n\n--------------------------Results--------------------------\n") + if (err) { + console.log("Error:", err) + } else { + if (typeof data === "object") { + data = JSON.stringify(data, null, 2); + } + if (data !== undefined) { + console.log(data); + } + } + }) + }); + }) + .parse(process.argv); + +if (!process.argv.slice(2).length) { + program.outputHelp(colors.red); +} + +function createContext(pkg, config) { + var start = new Date(); + var maxTime = (config.timeout || 300) * 1000; + return { + awsRequestId: "requestid-local" + moment.now().toString(), + getRemainingTimeInMillis: function () { + var timeSpent = new Date() - start; + if (timeSpent < maxTime) { + return maxTime - timeSpent; + } else { + return 0; + } + } + }; +} + +function stripBOM(content) { + // Remove byte order marker. This catches EF BB BF (the UTF-8 BOM) + // because the buffer-to-string conversion in `fs.readFileSync()` + // translates it to FEFF, the UTF-16 BOM. + if (content.charCodeAt(0) === 0xFEFF) { + content = content.slice(1); + } + return content; +} \ No newline at end of file diff --git a/leo-cli.js b/leo-cli.js new file mode 100644 index 0000000..d87a7e7 --- /dev/null +++ b/leo-cli.js @@ -0,0 +1,10 @@ +#!/usr/bin/env node + +var program = require('commander'); +var configure = require("./package.json"); + +program + .version(configure.version) + .command('publish [directory] [alias] [region]', "Publish your project to S3") + .command('test [directory] [alias] [region]', "Test your lambda") + .parse(process.argv); \ No newline at end of file