Merge pull request #196 from kubernetes-simulator/scoring

Scoring

attack/Makefile

@@ -4,6 +4,8 @@ GITHUB_ORG := controlplaneio
 DOCKER_HUB_ORG := controlplane
 VERSION := 0.1-dev
 
+LOCAL_DEV_SIMULATION := $(abspath ../simulation-scripts/scenario/etcd-inverted-wedge)
+
 include ../prelude.mk
 
 .DEFAULT_GOAL := help
@@ -15,9 +17,10 @@ all: test
 
 .PHONY: docker-run
 docker-run: docker-test ## Runs the attack container
-	docker run                                  \
-		-h attack                                 \
-		--rm --init -it $(CONTAINER_NAME_LATEST)  \
+	docker run \
+		-h attack \
+		-v $(LOCAL_DEV_SIMULATION)/tasks.yaml:/tasks.yaml \
+		--rm --init -it $(CONTAINER_NAME_LATEST) \
 		bash
 
 .PHONY: docker-build
@@ -26,7 +29,7 @@ docker-build: ## Builds the attack container
 
 .PHONY: docker-test
 docker-test: docker-build ## Run the attack container tests
-	@docker run                        \
+	@docker run \
 		--rm -t $(CONTAINER_NAME_LATEST) \
 		goss validate
 

attack/scripts/bashrc

@@ -236,6 +236,16 @@ welcome() {
     envsubst </challenge.txt
   fi
   info ' '
+  info 'There is a rudimentary beta scoring mechanism. You will be scored based'
+  info 'on how many hints you have seen but it currently does not validate that'
+  info 'you correctly completed the task. This is in the works :)'
+  info ' '
+  info 'Remember, when you start a task you will only have one opportunity to '
+  info 'be scored on that task. Once it is ended or if you switch to another '
+  info 'task, you will either be scored or marked as "skipping scoring". You '
+  info 'can still return to the task but scoring will be fixed to the result of'
+  info 'your first attempt.'
+  info ' '
   info 'To start the first task:'
   info ' '
   info "  \$ start_task 1"
@@ -249,6 +259,11 @@ welcome() {
   info "  \$ next_hint"
   info "  \$ show_hints"
   info ' '
+  info 'To end a task, you can start a new task or:'
+  info ' '
+  info "  \$ end_task"
+  info ' '
+  info ' '
   info "SSH to master(s), node(s), and an internal node with network access to the cluster:"
   info ' '
   info "  \$ ssh_master 0"

tools/scenario-tools/bin/start_task.js → tools/scenario-tools/bin/process_task.js

@@ -2,7 +2,7 @@
 
 const { createLogger } = require('../lib/logger')
 const { cloneArray } = require('../lib/helpers')
-const { startTask } = require('../lib/tasks.js')
+const { processTask } = require('../lib/tasks.js')
 
 const logger = createLogger({})
 
@@ -13,14 +13,16 @@ const args = cloneArray(process.argv)
 args.shift() // remove `node` from argv
 args.shift() // remove `scenario.js` from argv
 
-if (args.length !== 1) {
+if (process.argv0 === 'start_task' && args.length !== 1) {
   logger.error('Please provide the task you wish to start. These are listed in the instructions')
   process.exit(1)
 }
 
-const success = startTask(args[0])
-if (success) {
+// ignore the result - it will be true/false depending on whether the user
+// actually switched task
+processTask(args[0]).then(_ => {
   process.exit(0)
-} else {
+}, reason => {
+  logger.error(reason.message)
   process.exit(1)
-}
+})

tools/scenario-tools/lib/constants.js

@@ -1,4 +1,5 @@
 module.exports = {
   TASKS_FILE_PATH: '/tasks.yaml',
-  PROGRESS_FILE_PATH: '/progress.json'
+  PROGRESS_FILE_PATH: '/progress.json',
+  MAX_SCORE: 100
 }

tools/scenario-tools/lib/hints.js

@@ -21,6 +21,7 @@ function showHint (task, index, taskspath = TASKS_FILE_PATH, log = logger) {
   const hint = tasks[task].hints[index]
 
   log.info(hint.text)
+  log.info(`This hint incurred a penalty of ${hint.penalty} to your score`)
 }
 
 function showHints (task, taskspath = TASKS_FILE_PATH,
@@ -36,7 +37,7 @@ function showHints (task, taskspath = TASKS_FILE_PATH,
     return logger.info(`You have not seen any hints for ${task}`)
   }
 
-  const lastSeenHintIndex = progress[task]
+  const lastSeenHintIndex = progress[task].lastHintIndex
   const hintcount = tasks[task].hints.length
 
   for (let i = 0; i <= lastSeenHintIndex && i < hintcount; i++) {
@@ -58,15 +59,21 @@ function nextHint (task, taskspath = TASKS_FILE_PATH,
   let hintIndex
 
   if (progress[task] === undefined) {
-    hintIndex = progress[task] = 0
+    progress[task] = {}
+  }
+
+  if (progress[task].lastHintIndex === undefined) {
+    hintIndex = progress[task].lastHintIndex = 0
   } else {
-    const { tasks } = loadYamlFile(taskspath)
+    hintIndex = progress[task].lastHintIndex = progress[task].lastHintIndex + 1
+  }
 
-    hintIndex = progress[task] = ++progress[task]
-    if (hintIndex >= tasks[task].hints.length) {
-      return logger.info('You have seen all the hints for this task')
-    }
+  const { tasks } = loadYamlFile(taskspath)
+
+  if (hintIndex >= tasks[task].hints.length) {
+    return logger.info('You have seen all the hints for this task')
   }
+
   saveProgress(progress, progresspath)
 
   showHint(task, hintIndex, taskspath, log)

tools/scenario-tools/lib/scoring.js

@@ -0,0 +1,23 @@
+const { MAX_SCORE } = require('./constants')
+
+function calculateScore (progress, tasks) {
+  const currentTask = progress.current_task
+  const lastSeenHintIndex = progress[currentTask].lastHintIndex
+  const hintCount = tasks[currentTask].hints.length
+
+  if (lastSeenHintIndex === undefined) {
+    return MAX_SCORE
+  }
+
+  let score = MAX_SCORE
+
+  for (let i = 0; i <= lastSeenHintIndex && i < hintCount; i++) {
+    score -= tasks[currentTask].hints[i].penalty
+  }
+
+  return score
+}
+
+module.exports = {
+  calculateScore
+}

tools/scenario-tools/lib/tasks.js

@@ -1,25 +1,147 @@
 const { loadYamlFile, getProgress, saveProgress } = require('./io')
+const { calculateScore } = require('./scoring')
 const { createLogger } = require('./logger')
 const { TASKS_FILE_PATH, PROGRESS_FILE_PATH } = require('./constants')
+const { prompt } = require('inquirer')
 
 const logger = createLogger({})
 
-function startTask (task, taskspath = TASKS_FILE_PATH,
+async function askToBeScored (currentTask) {
+  logger.info(`Would you like to be scored for task ${currentTask}? (If you choose no you cannot be scored on this task in the future)`)
+  return prompt({
+    type: 'expand',
+    name: 'answer',
+    choices: [{
+      key: 'y',
+      name: 'Yes',
+      value: 'yes'
+    }, {
+      key: 'n',
+      name: 'No',
+      value: 'no'
+    }, {
+      key: 'c',
+      name: 'Cancel',
+      value: 'cancel'
+
+    }]
+  })
+}
+
+function updateProgressWithNewTask (progress, newTask) {
+  progress.current_task = newTask
+
+  if (progress[newTask] === undefined) {
+    progress[newTask] = {
+      lastHintIndex: undefined,
+      score: undefined
+    }
+  }
+
+  return progress
+}
+
+function processResponse (answer, progress, tasks, log = logger) {
+  if (answer === undefined) {
+    throw new Error(
+      'No changes made - expected an answer from the scoring prompt')
+  }
+
+  const currentTask = progress.current_task
+
+  if (answer === 'cancel') {
+    log.info(`You cancelled... leaving you on ${currentTask}`)
+    return false
+  }
+
+  if (answer === 'yes') {
+    const score = calculateScore(progress, tasks)
+    log.info(`Your score for task ${currentTask} was ${score}`)
+    progress[currentTask].score = score
+  } else if (answer === 'no') {
+    log.info(`You chose not to be scored on task ${currentTask}`)
+    progress[currentTask].score = 'skip'
+  } else {
+    throw new Error(
+      `No changes made - unrecognised answer from scoring prompt: ${answer}`)
+  }
+
+  return progress
+}
+
+// This is the entrypoint for both `start_task` and `end_task` if no `newTask`
+// argument was supplied we assume this was invoked as `end_task`
+async function processTask (newTask, taskspath = TASKS_FILE_PATH,
   progresspath = PROGRESS_FILE_PATH, log = logger) {
   const { tasks } = loadYamlFile(taskspath)
 
-  if (!tasks[task]) {
+  if (newTask !== undefined && !tasks[newTask]) {
     log.warn('Cannot find task')
     return false
   }
 
   const progress = getProgress(progresspath)
-  progress.current_task = task
-  saveProgress(progress, progresspath)
+  let newProgress
 
-  log.info(`You are now on ${task}`)
+  if (newTask !== undefined) {
+    newProgress = await startTask(newTask, tasks, progress, log, askToBeScored)
+  } else {
+    newProgress = await endTask(tasks, progress, log, askToBeScored)
+  }
 
-  return true
+  if (newProgress !== false) {
+    saveProgress(newProgress, progresspath)
+  }
+}
+
+async function endTask (tasks, progress, log, prompter) {
+  const currentTask = progress.current_task
+
+  if (currentTask === undefined) {
+    log.warn('Cannot end task - you have not started one')
+    return false
+  }
+
+  const { answer } = await prompter(currentTask)
+  const newProgress = processResponse(answer, progress, tasks, log)
+  if (newProgress === false) return false
+
+  progress.current_task = undefined
+  log.info(`You have ended task ${currentTask}`)
+  return progress
+}
+
+async function startTask (newTask, tasks, progress, log, prompter) {
+  const currentTask = progress.current_task
+
+  // User is trying to switch to the task they are already on
+  if (newTask === currentTask) {
+    log.warn(`You are already on task ${currentTask}`)
+    return false
+  }
+
+  // User hasnt started a task yet
+  if (currentTask === undefined) {
+    log.info(`You are now on task ${newTask}`)
+    return updateProgressWithNewTask(progress, newTask)
+  }
+
+  // user has started a task and previously either asked not to be scored or
+  // was already scored
+  if (progress[currentTask].score !== undefined) {
+    log.info(`You are now on task ${newTask}`)
+    return updateProgressWithNewTask(progress, newTask)
+  }
+
+  // user must have started a task and hasn't yet been scored or skipped scoring
+  const { answer } = await prompter(currentTask)
+  const newProgress = processResponse(answer, progress, tasks, log)
+  if (newProgress === false) return false
+
+  if (newTask !== undefined) {
+    log.info(`You are now on task ${newTask}`)
+    return updateProgressWithNewTask(newProgress, newTask)
+  }
 }
 
 function getCurrentTask (progresspath = PROGRESS_FILE_PATH) {
@@ -28,6 +150,10 @@ function getCurrentTask (progresspath = PROGRESS_FILE_PATH) {
 }
 
 module.exports = {
+  endTask,
+  getCurrentTask,
+  processTask,
+  processResponse,
   startTask,
-  getCurrentTask
+  updateProgressWithNewTask
 }