From e0d49ebb4e0cf282fa0ea79d6087830373887e2f Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Tue, 8 Aug 2017 11:43:13 +0000 Subject: [PATCH 1/2] Add support for tasks --- app.js | 3 ++- join.js | 3 ++- lib/boot.js | 8 ++++++++ lib/tasks.js | 53 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 lib/tasks.js diff --git a/app.js b/app.js index d3e58c74..d0aebd5b 100644 --- a/app.js +++ b/app.js @@ -18,7 +18,8 @@ const users = require('./lib/users'); boot.executeInParallel([ boot.forwardHttp, boot.ensureHttpsCertificates, - boot.ensureDockerTlsCertificates + boot.ensureDockerTlsCertificates, + boot.loadTasks, ], () => { // You can customize these values in './db.json'. const hostname = db.get('hostname', 'localhost'); diff --git a/join.js b/join.js index 8cead4c5..e5b1358a 100644 --- a/join.js +++ b/join.js @@ -27,7 +27,8 @@ boot.executeInParallel([ boot.forwardHttp, boot.ensureHttpsCertificates, boot.ensureDockerTlsCertificates, - boot.verifyJanitorOAuth2Access + boot.verifyJanitorOAuth2Access, + boot.loadTasks, ], () => { boot.registerDockerClient(() => { log('[ok] joined cluster as [hostname = ' + hostname + ']'); diff --git a/lib/boot.js b/lib/boot.js index 255e2bb1..e13930a7 100644 --- a/lib/boot.js +++ b/lib/boot.js @@ -8,6 +8,7 @@ const certificates = require('./certificates'); const db = require('./db'); const log = require('./log'); const oauth2 = require('./oauth2'); +const tasks = require('./tasks'); const hostname = db.get('hostname', 'localhost'); @@ -345,3 +346,10 @@ exports.registerDockerClient = function (next) { next(); }); }; + +exports.loadTasks = function (callback) { + setInterval(() => { + tasks.check(); + }, 60000); + callback(); +}; diff --git a/lib/tasks.js b/lib/tasks.js new file mode 100644 index 00000000..017b8973 --- /dev/null +++ b/lib/tasks.js @@ -0,0 +1,53 @@ +const db = require('./db'); +const log = require('./log'); +let taskCounter = 0; // Used to generate unique task IDs + +exports.tasks = db.get('tasks'); +exports.taskTypes = new Map(); + +exports.check = function () { + const now = Date.now(); + for (const task of Object.values(exports.tasks)) { + if (now > Number(task.date)) { + exports.execute(task); + } + } +}; + +exports.addType = function (type, task) { + if (exports.taskTypes.has(type)) { + throw new Error('[fail] task', task, 'already exists'); + } + + exports.taskTypes.set(type, task); +}; + +exports.add = function (date, type, data) { + taskCounter++; + + const msSince1970 = date.getTime(); + const taskId = `${type}-${msSince1970}-${taskCounter}`; + + const task = { + id: taskId, + date: msSince1970, + type, + data, + }; + + exports.tasks[taskId] = task; + db.save(); + return task; +}; + +exports.remove = function (id) { + delete exports.tasks[id]; + db.save(); +}; + +exports.execute = function ({ type, data, id }) { + const task = exports.taskTypes.get(type); + log('[ok] Task', id, 'executed', data); + task(data); + exports.remove(id); +}; From d049b17d1dda1290189c53eb6d91d0b504cc7f90 Mon Sep 17 00:00:00 2001 From: Tim Nguyen Date: Tue, 8 Aug 2017 11:43:48 +0000 Subject: [PATCH 2/2] Add temporary machines support --- api/hosts-api.js | 26 +++++-- lib/boot.js | 4 ++ lib/machines.js | 148 +++++++++++++++++++++++++++++++++------- templates/projects.html | 7 +- 4 files changed, 148 insertions(+), 37 deletions(-) diff --git a/api/hosts-api.js b/api/hosts-api.js index c801813f..cabc54be 100644 --- a/api/hosts-api.js +++ b/api/hosts-api.js @@ -409,13 +409,6 @@ containersAPI.put({ title: 'Create a container', handler (request, response) { - const { user } = request; - if (!user) { - response.statusCode = 403; // Forbidden - response.json({ error: 'Unauthorized' }, null, 2); - return; - } - const projectId = request.query.project; if (!(projectId in db.get('projects'))) { response.statusCode = 404; // Not Found @@ -423,6 +416,25 @@ containersAPI.put({ return; } + const { user, session } = request; + if (!user) { + if (!session) { + return; + } + + machines.spawnTemporary(session.id, projectId, (error, machine) => { + if (error) { + response.statusCode = 500; // Internal Server Error + response.json({ error: 'Could not create container' }, null, 2); + return; + } + response.json({ + container: machine.docker.container + }, null, 2); + }); + return; + } + machines.spawn(user, projectId, (error, machine) => { if (error) { log('[fail] could not spawn machine', error); diff --git a/lib/boot.js b/lib/boot.js index e13930a7..7db96ed4 100644 --- a/lib/boot.js +++ b/lib/boot.js @@ -348,6 +348,10 @@ exports.registerDockerClient = function (next) { }; exports.loadTasks = function (callback) { + tasks.addType('destroy-temporary', ({ session, container }) => { + const machines = require('./machines'); + machines.destroyTemporary(session, container, log); + }); setInterval(() => { tasks.check(); }, 60000); diff --git a/lib/machines.js b/lib/machines.js index 829a786f..eb97a7a2 100644 --- a/lib/machines.js +++ b/lib/machines.js @@ -8,6 +8,7 @@ const docker = require('./docker'); const log = require('./log'); const metrics = require('./metrics'); const streams = require('./streams'); +const tasks = require('./tasks'); // Get an existing user machine with the given project and machine ID. exports.getMachineById = function (user, projectId, machineId) { @@ -185,6 +186,56 @@ exports.spawn = function (user, projectId, callback) { const machine = getOrCreateNewMachine(user, projectId); + _spawn(machine, project, function (error, machine) { + if (error) { + callback(error); + return; + } + + const containerId = machine.docker.container; + // Quickly authorize the user's public SSH keys to access this container. + deploySSHAuthorizedKeys(user, machine, error => { + log('spawn-sshkeys', containerId.slice(0, 16), error || 'success'); + db.save(); + }); + + // Install all non-empty user configuration files into this container. + Object.keys(user.configurations).forEach(file => { + if (!user.configurations[file]) { + return; + } + exports.deployConfiguration(user, machine, file).then(() => { + log('spawn-config', file, containerId.slice(0, 16), 'success'); + }).catch(error => { + log('spawn-config', file, containerId.slice(0, 16), error); + }); + }); + callback(null, machine); + }); +}; + +// Instantiate a new temporary machine for a project. (Fast!) +exports.spawnTemporary = function (sessionId, projectId, callback) { + const machine = createNewTemporaryMachine(sessionId, projectId); + + _spawn(machine, getProject(projectId), (error, machine) => { + if (error) { + callback(error); + return; + } + + const destroyDate = new Date(Date.now()); + destroyDate.setHours(destroyDate.getHours() + 8); + tasks.add(destroyDate, 'destroy-temporary', { + session: sessionId, + container: machine.docker.container + }); + + callback(null, machine); + }); +}; + +function _spawn (machine, project, callback) { // Keep track of the last project update this machine will be based on. metrics.set(machine, 'updated', project.data.updated); @@ -207,7 +258,7 @@ exports.spawn = function (user, projectId, callback) { log('spawn', image, error); machine.status = 'start-failed'; db.save(); - callback(new Error('Unable to start machine for project: ' + projectId)); + callback(new Error('Unable to start machine for project: ' + project.id)); return; } @@ -220,27 +271,9 @@ exports.spawn = function (user, projectId, callback) { metrics.push(project, 'spawn-time', [ now, now - time ]); db.save(); - // Quickly authorize the user's public SSH keys to access this container. - deploySSHAuthorizedKeys(user, machine, error => { - log('spawn-sshkeys', container.id.slice(0, 16), error || 'success'); - db.save(); - }); - - // Install all non-empty user configuration files into this container. - Object.keys(user.configurations).forEach(file => { - if (!user.configurations[file]) { - return; - } - exports.deployConfiguration(user, machine, file).then(() => { - log('spawn-config', file, container.id.slice(0, 16), 'success'); - }).catch(error => { - log('spawn-config', file, container.id.slice(0, 16), error); - }); - }); - callback(null, machine); }); -}; +} // Destroy a given user machine and recycle its ports. exports.destroy = function (user, projectId, machineId, callback) { @@ -257,7 +290,7 @@ exports.destroy = function (user, projectId, machineId, callback) { return; } - const { container: containerId, host } = machine.docker; + const containerId = machine.docker.container; if (!containerId) { // This machine has no associated container, just recycle it as is. machine.status = 'new'; @@ -266,12 +299,9 @@ exports.destroy = function (user, projectId, machineId, callback) { return; } - log('destroy', containerId.slice(0, 16), 'started'); - docker.removeContainer({ host, container: containerId }, error => { + _destroy(machine, (error) => { if (error) { - log('destroy', containerId.slice(0, 16), error); callback(error); - return; } // Recycle the machine's name and ports. @@ -279,6 +309,45 @@ exports.destroy = function (user, projectId, machineId, callback) { machine.docker.container = ''; db.save(); callback(); + }); +}; + +exports.destroyTemporary = function (sessionId, containerId, callback) { + const machines = db.get('temporaryMachines')[sessionId]; + if (!machines) { + callback(new Error('No machines for session ' + sessionId)); + } + + const machineIndex = machines.findIndex(machine => + machine.docker.container === containerId); + + if (machineIndex === -1) { + callback(new Error('Wrong container ID')); + } + + _destroy(machines[machineIndex], (error) => { + if (error) { + callback(error); + } + + machines.splice(machineIndex, 1); + db.save(); + callback(); + }); +}; + +function _destroy (machine, callback) { + const { container: containerId, host } = machine.docker; + + log('destroy', containerId.slice(0, 16), 'started'); + docker.removeContainer({ host, container: containerId }, error => { + if (error) { + log('destroy', containerId.slice(0, 16), error); + callback(error); + return; + } + + callback(); if (!machine.docker.image) { log('destroy', containerId.slice(0, 16), 'success'); @@ -297,7 +366,7 @@ exports.destroy = function (user, projectId, machineId, callback) { db.save(); }); }); -}; +} // Install or overwrite a configuration file in all the user's containers. exports.deployConfigurationInAllContainers = function (user, file) { @@ -445,6 +514,33 @@ function getOrCreateNewMachine (user, projectId) { return machine; } +function createNewTemporaryMachine (sessionId, projectId) { + const project = getProject(projectId); + const temporaryMachines = db.get('temporaryMachines'); + if (!(sessionId in temporaryMachines)) { + temporaryMachines[sessionId] = []; + } + + const machines = temporaryMachines[sessionId]; + + const machine = { + properties: { + name: project.name + ' #' + machines.length, + }, + status: 'new', + docker: { + host: '', + container: '', + ports: {}, + logs: '' + }, + data: {} + }; + machines.push(machine); + + return machine; +} + // Get a unique available port starting from 42000. function getPort () { const ports = db.get('ports'); diff --git a/templates/projects.html b/templates/projects.html index 4d2f7460..749c3f0d 100644 --- a/templates/projects.html +++ b/templates/projects.html @@ -9,11 +9,10 @@

In just a single click, enter a fully-functional development environment. Ye
{{= project.name in xmlattr}} Logo

{{= project.name in html}}

-
{{if user then {{ -
+
+ - }} else {{ - New Container}}}} +