From 322743598a61d1fd0312fbd36445de26359c411e Mon Sep 17 00:00:00 2001 From: Ingo Richter Date: Mon, 9 Jun 2014 16:15:45 -0700 Subject: [PATCH 1/6] - download CEF builds from S3 - added task(s) to download the symbol files for CEF --- Gruntfile.js | 19 +++-- tasks/setup.js | 218 +++++++++++++++++++++++++++++++++---------------- 2 files changed, 163 insertions(+), 74 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index f205a5f17..e63b6b4f7 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -29,7 +29,7 @@ module.exports = function (grunt) { resolve = common.resolve, platform = common.platform(), staging; - + if (platform === "mac") { staging = "installer/mac/staging/<%= build.name %>.app/Contents"; } else if (platform === "win") { @@ -46,11 +46,11 @@ module.exports = function (grunt) { /* linux */ "cef-linux32": { "dest" : "<%= downloads %>", - "src" : "http://dev.brackets.io/cef/cef_binary_<%= cef.version %>_linux32_release.zip" + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_linux32_release.zip" }, "cef-linux64": { "dest" : "<%= downloads %>", - "src" : "http://dev.brackets.io/cef/cef_binary_<%= cef.version %>_linux64_release.zip" + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_linux64_release.zip" }, "node-linux32": { "dest" : "<%= downloads %>", @@ -63,7 +63,11 @@ module.exports = function (grunt) { /* mac */ "cef-mac": { "dest" : "<%= downloads %>", - "src" : "http://dev.brackets.io/cef/cef_binary_<%= cef.version %>_macosx.zip" + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_macosx.zip" + }, + "cef-mac-symbols": { + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_macosx32_release_symbols.7z", + "dest" : "<%= downloads %>/cefsymbols" }, "node-mac": { "dest" : "<%= downloads %>", @@ -72,7 +76,11 @@ module.exports = function (grunt) { /* win */ "cef-win": { "dest" : "<%= downloads %>", - "src" : "http://dev.brackets.io/cef/cef_binary_<%= cef.version %>_windows.zip" + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_windows.zip" + }, + "cef-win-symbols": { + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>windows32_debug_symbols.7z", + "dest" : "<%= downloads %>/cefsymbols" }, "node-win": { "dest" : "<%= downloads %>", @@ -201,6 +209,7 @@ module.exports = function (grunt) { } }, "cef": { + "url" : "http://s3.amazonaws.com/files.brackets.io/cef", "version" : "3.1547.1459" }, "node": { diff --git a/tasks/setup.js b/tasks/setup.js index 1bacf1f5f..52d41c275 100644 --- a/tasks/setup.js +++ b/tasks/setup.js @@ -1,24 +1,24 @@ /* * Copyright (c) 2013 Adobe Systems Incorporated. All rights reserved. - * + * * Permission is hereby granted, free of charge, to any person obtaining a - * copy of this software and associated documentation files (the "Software"), - * to deal in the Software without restriction, including without limitation - * the rights to use, copy, modify, merge, publish, distribute, sublicense, - * and/or sell copies of the Software, and to permit persons to whom the + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the * Software is furnished to do so, subject to the following conditions: - * + * * The above copyright notice and this permission notice shall be included in * all copies or substantial portions of the Software. - * + * * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING - * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER * DEALINGS IN THE SOFTWARE. - * + * */ /*jslint vars:true*/ /*global module, require, process*/ @@ -45,23 +45,23 @@ module.exports = function (grunt) { rename = common.rename, exec = common.exec, platform = common.platform(); - + // cross-platform symbolic link link = (function () { var typeArg, symlink; - + if (process.platform === "win32") { typeArg = "junction"; } - + symlink = q.denodeify(fs.symlink); - + return function (srcpath, destpath) { return symlink(srcpath, destpath, typeArg); }; }()); - + function unzip(src, dest) { grunt.verbose.writeln("Extracting " + src); return exec("unzip -q " + src + " -d " + dest); @@ -69,80 +69,160 @@ module.exports = function (grunt) { // task: cef grunt.registerTask("cef", "Download and setup CEF", function () { - var config = "cef-" + platform + common.arch(), - zipSrc = grunt.config("curl-dir." + config + ".src"), - zipName = path.basename(zipSrc), - zipDest = path.resolve(process.cwd(), path.join(grunt.config("curl-dir." + config + ".dest"), zipName)), + var config = "cef-" + platform + common.arch(), + zipSrc = grunt.config("curl-dir." + config + ".src"), + zipName = path.basename(zipSrc), + zipDest = path.resolve(process.cwd(), path.join(grunt.config("curl-dir." + config + ".dest"), zipName)), txtName; - + // extract zip file name and set config property grunt.config("cefZipDest", zipDest); grunt.config("cefZipSrc", zipSrc); grunt.config("cefZipName", zipName); grunt.config("cefConfig", config); - + // remove .zip extension txtName = zipName.substr(0, zipName.lastIndexOf(".")) + ".txt"; - + // optionally download if CEF is not found if (!grunt.file.exists("deps/cef/" + txtName)) { var cefTasks = ["cef-clean", "cef-extract", "cef-symlinks"]; - + if (grunt.file.exists(zipDest)) { grunt.verbose.writeln("Found CEF download " + zipDest); } else { cefTasks.unshift("cef-download"); } - + grunt.task.run(cefTasks); } else { grunt.verbose.writeln("Skipping CEF download. Found deps/cef/" + txtName); } }); - + // task: cef-clean grunt.registerTask("cef-clean", "Removes CEF binaries and linked folders", function () { var path; - + // delete dev symlinks from "setup_for_hacking" common.deleteFile("Release/dev", { force: true }); common.deleteFile("Debug/dev", { force: true }); - + // delete symlinks to cef Object.keys(CEF_MAPPING).forEach(function (key, index) { common.deleteFile(CEF_MAPPING[key]); }); - + // finally delete CEF binary\ common.deleteFile("deps/cef", { force: true }); }); - + // task: cef-download grunt.registerTask("cef-download", "Download CEF, see curl-dir config in Gruntfile.js", function () { // requires download-cef to set "cefZipName" in config grunt.task.requires(["cef"]); - + // run curl grunt.log.writeln("Downloading " + grunt.config("cefZipSrc") + ". This may take a while..."); grunt.task.run("curl-dir:" + grunt.config("cefConfig")); }); - + + grunt.registerTask("cef-symbols", "Download and unpack the CEF symbols", function () { + var config = "cef-" + platform + common.arch() + "-symbols", + zipSymbols = grunt.config("curl-dir." + config + ".src"); + + if (zipSymbols) { + var zipName = path.basename(zipSymbols), + zipDest = path.resolve(process.cwd(), path.join(grunt.config("curl-dir." + config + ".dest"), zipName)), + txtName; + + // extract zip file name and set config property + grunt.config("cefSymbolsZipDest", zipDest); + grunt.config("cefSymbolsZipSrc", zipSymbols); + grunt.config("cefSymbolsZipName", zipName); + grunt.config("cefConfig", config); + + // remove .zip extension + txtName = zipName.substr(0, zipName.lastIndexOf(".")) + ".txt"; + + // optionally download if CEF is not found + if (!grunt.file.exists("symbols/cef/" + txtName)) { + var cefTasks = ["cef-symbols-extract"]; + + if (grunt.file.exists(zipDest)) { + grunt.verbose.writeln("Found CEF symbols download " + zipDest); + } else { + cefTasks.unshift("cef-symbols-download"); + } + + grunt.task.run(cefTasks); + } else { + grunt.verbose.writeln("Skipping CEF symbols download. Found symbols/cef/" + txtName); + } + } + }); + + // task: cef-extract-symbols + grunt.registerTask("cef-symbols-extract", "Extract CEF symbols zip", function () { + // requires cef to set "cefZipName" in config + grunt.task.requires(["cef-symbols"]); + +// var done = this.async(), +// zipDest = grunt.config("cefSymbolsZipDest"), +// zipName = grunt.config("cefSymbolsZipName"), +// unzipPromise; +// +// // unzip to symbols/ +// unzipPromise = unzip(zipDest, "symbols"); +// +// // remove .zip ext +// zipName = path.basename(zipName, ".7z"); +// +// unzipPromise.then(function () { +// // rename version stamped name to cef +// return rename("deps/" + zipName, "deps/cef"); +// }).then(function () { +// var memo = path.resolve(process.cwd(), "symbols/cef/" + zipName + ".txt"), +// permissionsPromise, +// defer = q.defer(); +// +// // write empty file with zip file +// grunt.file.write(memo, ""); +// done(); +// }, function (err) { +// grunt.log.writeln(err); +// done(false); +// }); + }); + + // task: cef-download-symbols + grunt.registerTask("cef-symbols-download", "Download and extract the CEF debug/release symbols", function () { + // requires -cef-download-symbols to set "cefSymbolZipName" in config + grunt.task.requires(["cef-symbols"]); + + // run curl + grunt.log.writeln("Downloading " + grunt.config("cefSymbolZipSrc") + ". This may take a while..."); + // curl doesn't give me the option to handle download errors on my own. If the requested file can't + // be found, curl will log an error to the console. + grunt.task.run("curl-dir:" + grunt.config("cefConfig")); + }); + // task: cef-extract grunt.registerTask("cef-extract", "Extract CEF zip", function () { // requires cef to set "cefZipName" in config grunt.task.requires(["cef"]); - + var done = this.async(), zipDest = grunt.config("cefZipDest"), zipName = grunt.config("cefZipName"), unzipPromise; - + // unzip to deps/ unzipPromise = unzip(zipDest, "deps"); - + // remove .zip ext zipName = path.basename(zipName, ".zip"); - + unzipPromise.then(function () { // rename version stamped name to cef return rename("deps/" + zipName, "deps/cef"); @@ -150,18 +230,18 @@ module.exports = function (grunt) { var memo = path.resolve(process.cwd(), "deps/cef/" + zipName + ".txt"), permissionsPromise, defer = q.defer(); - + if (platform === "mac") { // FIXME figure out how to use fs.chmod to only do additive mode u+x permissionsPromise = exec("chmod u+x deps/cef/tools/*"); } else { permissionsPromise = q.resolve(); } - + return permissionsPromise.then(function () { - // write empty file with zip file + // write empty file with zip file grunt.file.write(memo, ""); - + return q.resolve(); }); }).then(function () { @@ -171,23 +251,23 @@ module.exports = function (grunt) { done(false); }); }); - + // task: cef-symlinks grunt.registerTask("cef-symlinks", "Create symlinks for CEF", function () { var done = this.async(), path, links = []; - + // create symlinks Object.keys(CEF_MAPPING).forEach(function (key, index) { path = CEF_MAPPING[key]; - + // some paths to deps/cef/* are platform specific and may not exist if (grunt.file.exists(key)) { links.push(link(key, path)); } }); - + // wait for all symlinks to complete q.all(links).then(function () { done(); @@ -196,7 +276,7 @@ module.exports = function (grunt) { done(false); }); }); - + // task: node-download grunt.registerTask("node", "Download Node.js binaries and setup dependencies", function () { var config = "node-" + platform + common.arch(), @@ -209,10 +289,10 @@ module.exports = function (grunt) { npmVersion = grunt.config("npm.version"), txtName = "version-" + nodeVersion + ".txt", missingDest = false; - + // extract file name and set config property nodeSrc = Array.isArray(nodeSrc) ? nodeSrc : [nodeSrc]; - + // curl-dir:node-win32 defines multiple downloads, account for this array nodeSrc.forEach(function (value, index) { nodeSrc[index] = value.substr(value.lastIndexOf("/") + 1); @@ -222,7 +302,7 @@ module.exports = function (grunt) { }); grunt.config("nodeSrc", nodeSrc); grunt.config("nodeDest", nodeDest); - + // optionally download if node is not found if (!grunt.file.exists("deps/node/" + txtName)) { if (!missingDest) { @@ -236,27 +316,27 @@ module.exports = function (grunt) { grunt.verbose.writeln("Skipping Node.js download. Found deps/node/" + txtName); } }); - + function nodeWriteVersion() { // write empty file with node-version name grunt.file.write("deps/node/version-" + grunt.config("node.version") + ".txt", ""); } - + // task: node-win32 grunt.registerTask("node-win", "Setup Node.js for Windows", function () { // requires node to set "nodeSrc" in config grunt.task.requires(["node"]); - + var done = this.async(), nodeDest = grunt.config("nodeDest"), exeFile = nodeDest[0], npmFile = nodeDest[1]; - + grunt.file.mkdir("deps/node"); - + // copy node.exe to Brackets-node grunt.file.copy(exeFile, "deps/node/node.exe"); - + // unzip NPM unzip(npmFile, "deps/node").then(function () { nodeWriteVersion(); @@ -266,24 +346,24 @@ module.exports = function (grunt) { done(false); }); }); - + // task: node-mac grunt.registerTask("node-mac", "Setup Node.js for Mac OSX and extract", function () { // requires node to set "nodeSrc" in config grunt.task.requires(["node"]); - + var done = this.async(), tarFile = grunt.config("nodeDest")[0], tarExec, tarName; - + grunt.verbose.writeln("Extracting " + tarFile); tarExec = exec("tar -xzf " + tarFile); - + // remove downloads/, remove .tar.gz extension tarName = tarFile.substr(tarFile.lastIndexOf("/") + 1); tarName = tarName.substr(0, tarName.lastIndexOf(".tar.gz")); - + tarExec.then(function () { return rename(tarName, "deps/node"); }).then(function () { @@ -300,39 +380,39 @@ module.exports = function (grunt) { done(false); }); }); - + // task: node-linux grunt.registerTask("node-linux", ["node-mac"]); - + // task: node-clean grunt.registerTask("node-clean", "Removes Node.js binaries", function () { common.deleteFile("deps/node", { force: true }); }); - + // task: create-project grunt.registerTask("create-project", "Create Xcode/VisualStudio/Makefile project", function () { var done = this.async(), promise, gypCommand; - + if (platform === "linux") { gypCommand = "bash -c 'python2 gyp/gyp --depth=.'"; } else { gypCommand = "bash -c 'gyp/gyp appshell.gyp -I common.gypi --depth=.'"; } - + grunt.log.writeln("Building project files"); - + // Run gyp promise = exec(gypCommand); - + if (platform === "mac") { promise = promise.then(function () { // FIXME port to JavaScript? return exec("bash scripts/fix-xcode.sh"); }); } - + promise.then(function () { done(); }, function (err) { @@ -340,7 +420,7 @@ module.exports = function (grunt) { done(false); }); }); - + // task: setup grunt.registerTask("setup", ["cef", "node", "create-project"]); }; From 43258ef8844b00ae03406cdb8862c444f22bfcfb Mon Sep 17 00:00:00 2001 From: Ingo Richter Date: Thu, 12 Jun 2014 01:08:12 -0700 Subject: [PATCH 2/6] - small changes based on review feedback --- Gruntfile.js | 4 +- tasks/setup.js | 105 +++++++++++++++++++++++++------------------------ 2 files changed, 56 insertions(+), 53 deletions(-) diff --git a/Gruntfile.js b/Gruntfile.js index e63b6b4f7..5fe4b59e9 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -66,7 +66,7 @@ module.exports = function (grunt) { "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_macosx.zip" }, "cef-mac-symbols": { - "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_macosx32_release_symbols.7z", + "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_macosx32_release_symbols.zip", "dest" : "<%= downloads %>/cefsymbols" }, "node-mac": { @@ -79,7 +79,7 @@ module.exports = function (grunt) { "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>_windows.zip" }, "cef-win-symbols": { - "src" : "<%= cef.url %>/cef_binary_<%= cef.version %>windows32_debug_symbols.7z", + "src" : ["<%= cef.url %>/cef_binary_<%= cef.version %>_windows32_debug_symbols.zip", "<%= cef.url %>/cef_binary_<%= cef.version %>_windows32_release_symbols.zip"], "dest" : "<%= downloads %>/cefsymbols" }, "node-win": { diff --git a/tasks/setup.js b/tasks/setup.js index 52d41c275..0a2ec446f 100644 --- a/tasks/setup.js +++ b/tasks/setup.js @@ -127,84 +127,87 @@ module.exports = function (grunt) { grunt.task.run("curl-dir:" + grunt.config("cefConfig")); }); + function symbolFileLocation() { + return path.resolve(process.cwd(), "deps/cef/symbols/"); + } + grunt.registerTask("cef-symbols", "Download and unpack the CEF symbols", function () { var config = "cef-" + platform + common.arch() + "-symbols", zipSymbols = grunt.config("curl-dir." + config + ".src"); if (zipSymbols) { - var zipName = path.basename(zipSymbols), - zipDest = path.resolve(process.cwd(), path.join(grunt.config("curl-dir." + config + ".dest"), zipName)), - txtName; + var symbolSrcUrls = zipSymbols; + + if (!Array.isArray(zipSymbols)) { + symbolSrcUrls = []; + symbolSrcUrls.push(zipSymbols); + } + + symbolSrcUrls.forEach(function (srcUrl) { + var zipName = path.basename(srcUrl), + zipDest = path.resolve(process.cwd(), path.join(grunt.config("curl-dir." + config + ".dest"), zipName)), + txtName; + + // extract zip file name and set config property + grunt.config("cefConfig", config); - // extract zip file name and set config property - grunt.config("cefSymbolsZipDest", zipDest); - grunt.config("cefSymbolsZipSrc", zipSymbols); - grunt.config("cefSymbolsZipName", zipName); - grunt.config("cefConfig", config); + // remove .zip extension + txtName = path.basename(zipName, ".zip") + ".txt"; - // remove .zip extension - txtName = zipName.substr(0, zipName.lastIndexOf(".")) + ".txt"; + // optionally download if CEF is not found + if (!grunt.file.exists(path.resolve(path.join(symbolFileLocation(), txtName)))) { + // pass the name of the zip file + var cefTasks = ["cef-symbols-extract" + ":" + zipDest]; - // optionally download if CEF is not found - if (!grunt.file.exists("symbols/cef/" + txtName)) { - var cefTasks = ["cef-symbols-extract"]; + if (grunt.file.exists(zipDest)) { + grunt.verbose.writeln("Found CEF symbols download " + zipDest); + } else { + cefTasks.unshift("cef-symbols-download" + ":" + config); + } - if (grunt.file.exists(zipDest)) { - grunt.verbose.writeln("Found CEF symbols download " + zipDest); + grunt.task.run(cefTasks); } else { - cefTasks.unshift("cef-symbols-download"); + grunt.verbose.writeln("Skipping CEF symbols download. Found deps/cef/symbols/" + txtName); } - - grunt.task.run(cefTasks); - } else { - grunt.verbose.writeln("Skipping CEF symbols download. Found symbols/cef/" + txtName); - } + }); } }); // task: cef-extract-symbols grunt.registerTask("cef-symbols-extract", "Extract CEF symbols zip", function () { - // requires cef to set "cefZipName" in config grunt.task.requires(["cef-symbols"]); -// var done = this.async(), -// zipDest = grunt.config("cefSymbolsZipDest"), -// zipName = grunt.config("cefSymbolsZipName"), -// unzipPromise; -// -// // unzip to symbols/ -// unzipPromise = unzip(zipDest, "symbols"); -// -// // remove .zip ext -// zipName = path.basename(zipName, ".7z"); -// -// unzipPromise.then(function () { -// // rename version stamped name to cef -// return rename("deps/" + zipName, "deps/cef"); -// }).then(function () { -// var memo = path.resolve(process.cwd(), "symbols/cef/" + zipName + ".txt"), -// permissionsPromise, -// defer = q.defer(); -// -// // write empty file with zip file -// grunt.file.write(memo, ""); -// done(); -// }, function (err) { -// grunt.log.writeln(err); -// done(false); -// }); + var done = this.async(), + zipDest = arguments[0], + zipName = path.basename(zipDest, '.zip'), + unzipPromise; + + // unzip to deps/cef/symbols/ + unzipPromise = unzip(zipDest, symbolFileLocation()); + + unzipPromise.then(function () { + var memo = path.resolve(path.join(symbolFileLocation(), zipName + ".txt")); + + // write empty file with zip file + grunt.file.write(memo, ""); + done(); + }, function (err) { + grunt.log.writeln(err); + done(false); + }); }); // task: cef-download-symbols grunt.registerTask("cef-symbols-download", "Download and extract the CEF debug/release symbols", function () { - // requires -cef-download-symbols to set "cefSymbolZipName" in config grunt.task.requires(["cef-symbols"]); + var downloadConfig = arguments[0]; + // run curl - grunt.log.writeln("Downloading " + grunt.config("cefSymbolZipSrc") + ". This may take a while..."); + grunt.log.writeln("Downloading " + downloadConfig + ". This may take a while..."); // curl doesn't give me the option to handle download errors on my own. If the requested file can't // be found, curl will log an error to the console. - grunt.task.run("curl-dir:" + grunt.config("cefConfig")); + grunt.task.run("curl-dir:" + downloadConfig); }); // task: cef-extract From 9630a91391f2f9a3026cf3bfb8d7f26806e23a9c Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 12 Jun 2014 14:57:23 -0700 Subject: [PATCH 3/6] - encode windows path to get rid of ':' before passing it as argument to the grunt task --- tasks/setup.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/setup.js b/tasks/setup.js index 0a2ec446f..79a54ae18 100644 --- a/tasks/setup.js +++ b/tasks/setup.js @@ -157,7 +157,8 @@ module.exports = function (grunt) { // optionally download if CEF is not found if (!grunt.file.exists(path.resolve(path.join(symbolFileLocation(), txtName)))) { // pass the name of the zip file - var cefTasks = ["cef-symbols-extract" + ":" + zipDest]; + var zipDestSafe = zipDest.replace(":", "|"); + var cefTasks = ["cef-symbols-extract" + ":" + zipDestSafe]; if (grunt.file.exists(zipDest)) { grunt.verbose.writeln("Found CEF symbols download " + zipDest); @@ -178,7 +179,7 @@ module.exports = function (grunt) { grunt.task.requires(["cef-symbols"]); var done = this.async(), - zipDest = arguments[0], + zipDest = arguments[0].replace("|", ":"), zipName = path.basename(zipDest, '.zip'), unzipPromise; From cbabe4d78567b3a3d7c3efdd3120fb3729f7b3c7 Mon Sep 17 00:00:00 2001 From: Ingo Richter Date: Fri, 13 Jun 2014 15:31:16 -0700 Subject: [PATCH 4/6] - use system curl instead of grunt-curl to prevent out of memory error downloading the OSX symbols - fix the download issues with Windows --- tasks/setup.js | 95 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 85 insertions(+), 10 deletions(-) diff --git a/tasks/setup.js b/tasks/setup.js index 79a54ae18..2690d7055 100644 --- a/tasks/setup.js +++ b/tasks/setup.js @@ -67,6 +67,11 @@ module.exports = function (grunt) { return exec("unzip -q " + src + " -d " + dest); } + function curl(src, dest) { + grunt.verbose.writeln("Downloading " + src); + return exec("curl -o " + dest + " " + src); + } + // task: cef grunt.registerTask("cef", "Download and setup CEF", function () { var config = "cef-" + platform + common.arch(), @@ -127,8 +132,50 @@ module.exports = function (grunt) { grunt.task.run("curl-dir:" + grunt.config("cefConfig")); }); - function symbolFileLocation() { - return path.resolve(process.cwd(), "deps/cef/symbols/"); + grunt.registerTask("curl-download", function () { + var downloadConfig = arguments[0], + downloadSource = grunt.config("curl-dir." + downloadConfig + ".src"), + downloadDest = path.join(process.cwd(), grunt.config("curl-dir." + downloadConfig + ".dest")), + downloadSourceURLS = downloadSource, + done = this.async(); + + if (!Array.isArray(downloadSource)) { + downloadSourceURLS = []; + downloadSourceURLS.push(downloadSource); + } + + var promises = downloadSourceURLS.map(function (srcUrl) { + var filename = path.basename(srcUrl), + dest = path.join(downloadDest, filename); + + grunt.verbose.writeln("Download " + srcUrl + " to " + dest); + return curl(srcUrl, dest); + }); + + q.all(promises).then(done).catch(function (err) { + done(err); + }); + }); + + function cefFileLocation() { + return path.resolve(process.cwd(), "deps/cef"); + } + + // Deduct the configuration (debug|release) from the zip file name + function symbolCompileConfiguration(zipFileName) { + if (zipFileName) { + var re = /\w+?[\d+\.\d+\.\d+]+_\w+?_(\w+?)_\w+\.zip/; + var match = zipFileName.match(re); + + if (!match) { + grunt.log.error("File name doesn't match the pattern for cef symbols:", zipFileName); + return zipFileName; + } + + return match[1]; + } else { + grunt.log.error("Please provide a zip file name"); + } } grunt.registerTask("cef-symbols", "Download and unpack the CEF symbols", function () { @@ -146,7 +193,8 @@ module.exports = function (grunt) { symbolSrcUrls.forEach(function (srcUrl) { var zipName = path.basename(srcUrl), zipDest = path.resolve(process.cwd(), path.join(grunt.config("curl-dir." + config + ".dest"), zipName)), - txtName; + txtName, + _symbolFileLocation; // extract zip file name and set config property grunt.config("cefConfig", config); @@ -155,9 +203,11 @@ module.exports = function (grunt) { txtName = path.basename(zipName, ".zip") + ".txt"; // optionally download if CEF is not found - if (!grunt.file.exists(path.resolve(path.join(symbolFileLocation(), txtName)))) { + _symbolFileLocation = path.join(cefFileLocation(), symbolCompileConfiguration(zipName)); + + if (!grunt.file.exists(path.resolve(path.join(_symbolFileLocation, txtName)))) { // pass the name of the zip file - var zipDestSafe = zipDest.replace(":", "|"); + var zipDestSafe = zipDest.replace(":", "|"); var cefTasks = ["cef-symbols-extract" + ":" + zipDestSafe]; if (grunt.file.exists(zipDest)) { @@ -168,7 +218,7 @@ module.exports = function (grunt) { grunt.task.run(cefTasks); } else { - grunt.verbose.writeln("Skipping CEF symbols download. Found deps/cef/symbols/" + txtName); + grunt.verbose.writeln("Skipping CEF symbols download. Found " + _symbolFileLocation + "/" + txtName); } }); } @@ -183,11 +233,35 @@ module.exports = function (grunt) { zipName = path.basename(zipDest, '.zip'), unzipPromise; - // unzip to deps/cef/symbols/ - unzipPromise = unzip(zipDest, symbolFileLocation()); + var symbolFileLocation = path.join(cefFileLocation(), symbolCompileConfiguration(path.basename(zipDest))); + + // unzip to deps/cef + unzipPromise = unzip(zipDest, symbolFileLocation); + + var symbolDir = path.join(symbolFileLocation, zipName); unzipPromise.then(function () { - var memo = path.resolve(path.join(symbolFileLocation(), zipName + ".txt")); + var rename = q.denodeify(fs.rename), + readdir = q.denodeify(fs.readdir); + + return readdir(symbolDir).then(function (files) { + if (files.length) { + var promises = files.map(function (file) { + return rename(path.join(symbolDir, file), path.join(path.dirname(symbolDir), file)); + }); + + return q.all(promises); + } else { + return; + } + }, function (err) { + return err; + }); + }).then(function () { + var rmdir = q.denodeify(fs.rmdir); + return rmdir(symbolDir); + }).then(function () { + var memo = path.resolve(path.join(symbolFileLocation, zipName + ".txt")); // write empty file with zip file grunt.file.write(memo, ""); @@ -208,7 +282,8 @@ module.exports = function (grunt) { grunt.log.writeln("Downloading " + downloadConfig + ". This may take a while..."); // curl doesn't give me the option to handle download errors on my own. If the requested file can't // be found, curl will log an error to the console. - grunt.task.run("curl-dir:" + downloadConfig); + //grunt.task.run("curl-dir:" + downloadConfig); + grunt.task.run("curl-download:" + downloadConfig); }); // task: cef-extract From f3cf619c77318caa509b1d61b87978aa6f5a28b2 Mon Sep 17 00:00:00 2001 From: unknown Date: Fri, 13 Jun 2014 17:50:16 -0700 Subject: [PATCH 5/6] - resolved issue with windows path containing spaces --- tasks/setup.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/tasks/setup.js b/tasks/setup.js index 2690d7055..0e4b81795 100644 --- a/tasks/setup.js +++ b/tasks/setup.js @@ -64,12 +64,17 @@ module.exports = function (grunt) { function unzip(src, dest) { grunt.verbose.writeln("Extracting " + src); - return exec("unzip -q " + src + " -d " + dest); + return exec("unzip -q \"" + src + "\" -d \"" + dest + "\""); } function curl(src, dest) { - grunt.verbose.writeln("Downloading " + src); - return exec("curl -o " + dest + " " + src); + if (process.platform === "win32") { + var dest = dest.replace(/\\/g, "/"); + } + + grunt.verbose.writeln("Downloading " + src + " to " + dest); + var cmd = "curl -o \"" + dest + "\" \"" + src + "\""; + return exec(cmd); } // task: cef @@ -144,6 +149,9 @@ module.exports = function (grunt) { downloadSourceURLS.push(downloadSource); } + // ensure the download directory exists + fs.mkdirSync(downloadDest); + var promises = downloadSourceURLS.map(function (srcUrl) { var filename = path.basename(srcUrl), dest = path.join(downloadDest, filename); From 44506185cd92c9ce28a5b202af828e6d98c80818 Mon Sep 17 00:00:00 2001 From: Ingo Richter Date: Sun, 15 Jun 2014 16:15:21 -0700 Subject: [PATCH 6/6] - update to grunt-curl 2.0.2 which resolved the out of memory with node. This makes my interim solution obsolete. --- package.json | 4 ++-- tasks/setup.js | 41 +---------------------------------------- 2 files changed, 3 insertions(+), 42 deletions(-) diff --git a/package.json b/package.json index ccbd8fa80..f82e583a1 100644 --- a/package.json +++ b/package.json @@ -18,7 +18,7 @@ "grunt-contrib-copy": "0.4.0", "grunt-contrib-clean": "0.4.0", "guid": "0.0.10", - "grunt-curl": "0.5.0", + "grunt-curl": "2.0.2", "grunt-shell": "0.2.1", "q": "0.9.2", "semver": "2.0.11" @@ -28,4 +28,4 @@ "install": "", "postinstall": "bash -c 'mv .appshell.gyp appshell.gyp;'" } -} \ No newline at end of file +} diff --git a/tasks/setup.js b/tasks/setup.js index 0e4b81795..d22b14921 100644 --- a/tasks/setup.js +++ b/tasks/setup.js @@ -67,16 +67,6 @@ module.exports = function (grunt) { return exec("unzip -q \"" + src + "\" -d \"" + dest + "\""); } - function curl(src, dest) { - if (process.platform === "win32") { - var dest = dest.replace(/\\/g, "/"); - } - - grunt.verbose.writeln("Downloading " + src + " to " + dest); - var cmd = "curl -o \"" + dest + "\" \"" + src + "\""; - return exec(cmd); - } - // task: cef grunt.registerTask("cef", "Download and setup CEF", function () { var config = "cef-" + platform + common.arch(), @@ -137,34 +127,6 @@ module.exports = function (grunt) { grunt.task.run("curl-dir:" + grunt.config("cefConfig")); }); - grunt.registerTask("curl-download", function () { - var downloadConfig = arguments[0], - downloadSource = grunt.config("curl-dir." + downloadConfig + ".src"), - downloadDest = path.join(process.cwd(), grunt.config("curl-dir." + downloadConfig + ".dest")), - downloadSourceURLS = downloadSource, - done = this.async(); - - if (!Array.isArray(downloadSource)) { - downloadSourceURLS = []; - downloadSourceURLS.push(downloadSource); - } - - // ensure the download directory exists - fs.mkdirSync(downloadDest); - - var promises = downloadSourceURLS.map(function (srcUrl) { - var filename = path.basename(srcUrl), - dest = path.join(downloadDest, filename); - - grunt.verbose.writeln("Download " + srcUrl + " to " + dest); - return curl(srcUrl, dest); - }); - - q.all(promises).then(done).catch(function (err) { - done(err); - }); - }); - function cefFileLocation() { return path.resolve(process.cwd(), "deps/cef"); } @@ -290,8 +252,7 @@ module.exports = function (grunt) { grunt.log.writeln("Downloading " + downloadConfig + ". This may take a while..."); // curl doesn't give me the option to handle download errors on my own. If the requested file can't // be found, curl will log an error to the console. - //grunt.task.run("curl-dir:" + downloadConfig); - grunt.task.run("curl-download:" + downloadConfig); + grunt.task.run("curl-dir:" + downloadConfig); }); // task: cef-extract