Merge pull request #310 from CESSProject/hotfix/handover_storage_files

Hotfix/handover storage files
CESSProject · Feb 20, 2024 · 4b118c5 · 4b118c5
2 parents c63513c + 44c65be
commit 4b118c5
Show file tree

Hide file tree

Showing 3 changed files with 116 additions and 73 deletions.
diff --git a/scripts/docker/ceseal/gramine/handover.ts b/scripts/docker/ceseal/gramine/handover.ts
@@ -1,14 +1,22 @@
 import * as path from "https://deno.land/std/path/mod.ts";
 import { readStringDelim } from "https://deno.land/std/io/mod.ts";
 import { copySync } from "https://deno.land/std/fs/copy.ts";
+import { sortBy } from "https://deno.land/[email protected]/collections/sort_by.ts";
+import { ensureDir, exists } from "https://deno.land/[email protected]/fs/mod.ts";
 // import { sleep } from "https://deno.land/x/sleep/mod.ts";
 
+const LOG_PREFIX = "[Handover🤝]"
+
+function log(...args: any[]) {
+  args.unshift(LOG_PREFIX + " ");
+  console.log(...args);
+}
+
 async function startCeseal(basePath: string, port: string | number, tmpPath = "/tmp", extra_args = []) {
   const logPath = path.join(tmpPath, "ceseal.log");
 
   const args = [
-      '--cores=0',  // Disable benchmark
-      '--port', port.toString(),
+    '--port', port.toString(),
   ];
   args.push(...extra_args);
 
@@ -24,7 +32,7 @@ async function startCeseal(basePath: string, port: string | number, tmpPath = "/
   const child = bin.spawn();
 
   child.stdout.pipeTo(Deno.openSync(logPath, { read: true, write: true, create: true }).writable);
-  
+
   return child;
 }
 
@@ -36,7 +44,7 @@ async function waitCesealStarted(logFile: string) {
     if (event.kind !== "modify") continue;
     for await (const line of readStringDelim(fileReader, "\n")) {
       if (!line) break;
-      console.log(line);
+      log(line);
       if (line.includes("Ceseal internal server will listening on")) {
         return true;
       }
@@ -46,53 +54,64 @@ async function waitCesealStarted(logFile: string) {
   return true
 }
 
-const exists = async (filename: string): Promise<boolean> => {
-  try {
-    await Deno.stat(filename);
-    // successful, file or directory must exist
-    return true;
-  } catch (error) {
-    if (error instanceof Deno.errors.NotFound) {
-      // file or directory does not exist
-      return false;
-    } else {
-      // unexpected error, maybe permissions, pass it along
-      throw error;
+async function confirmPreviousVersion(): Promise<number | undefined> {
+  let versions: number[] = [];
+  for await (const dirEntry of Deno.readDir('/opt/ceseal/backups')) {
+    versions.push(parseInt(dirEntry.name));
+  }
+  const sortedByDesc = sortBy(versions, (it) => it, { order: "desc" });
+  let previousVersion: number | undefined = undefined;
+  for (const version of sortedByDesc) {
+    if (!previousVersion || previousVersion < version) {
+      if (version >= currentVersion) {
+        continue;
+      } else if (!await exists(`/opt/ceseal/backups/${version}/data/protected_files/runtime-data.seal`)) {
+        console.log(`no runtime-data.seal found in ${version}, skip`)
+        continue
+      }
+      previousVersion = version;
+      break;
     }
   }
-};
+  return previousVersion;
+}
+
+async function killPreviousCeseal(version: number) {
+  const cmd = ["bash", "-c", `ps -eaf | grep "backups/${version}/cruntime/sgx/loader" | grep -v "grep" | awk '{print $2}'`];
+  const p = Deno.run({ cmd, stdout: "piped", stderr: "piped" });
+
+  // Reading the outputs closes their pipes
+  const [{ code }, rawOutput, rawError] = await Promise.all([
+    p.status(),
+    p.output(),
+    p.stderrOutput(),
+  ]);
+
+  if (code === 0) {
+    const pid = new TextDecoder().decode(rawOutput);
+    log(`the previous version ${version} ceseal pid: ${pid}`);
+    const p = Deno.run({ cmd: ["bash", "-c", `kill -9 ${pid}`] });
+    await p.status();
+  } else {
+    const errorString = new TextDecoder().decode(rawError);
+    log(errorString);
+  }
+}
 
 const currentPath = await Deno.realPath("/opt/ceseal/releases/current");
 const currentVersion = currentPath.split("/").pop();
-console.log(`Current ${currentPath}`)
+log(`Current ${currentPath}`)
 
 // Check current (the image contains) has initialized
 if (await exists(path.join(currentPath, "data/protected_files/runtime-data.seal"))) {
-  console.log("runtime-data.seal exists, no need to handover")
+  log("runtime-data.seal exists, no need to handover")
   Deno.exit(0);
 }
 
-// TODO: descending sort folders and find the latest handoverable ceseal
-let previousVersion: number | undefined = undefined;
-for await (const dirEntry of Deno.readDir('/opt/ceseal/backups')) {
-  // console.log(dirEntry);
-
-  // TODO: check handoverable (initialized && synced && version). Q: how to deal with not synced?
-  const version = parseInt(dirEntry.name);
-  if (!previousVersion || previousVersion < version) {
-    if (version >= currentVersion) {
-      continue;
-    } else if (!await exists(`/opt/ceseal/backups/${version}/data/protected_files/runtime-data.seal`)) {
-      console.log(`no runtime-data.seal found in ${version}, skip`)
-      continue
-    }
-
-    previousVersion = version;
-  }
-}
+let previousVersion: number | undefined = await confirmPreviousVersion();
 
 if (previousVersion === undefined) {
-  console.log("No previous version, no need to handover!");
+  log("No previous version, no need to handover!");
 
   // Copy current to backups
   try { copySync(currentPath, `/opt/ceseal/backups/${currentVersion}`) } catch (err) { console.error(err.message) }
@@ -101,49 +120,71 @@ if (previousVersion === undefined) {
 }
 
 if (currentVersion == previousVersion) {
-  console.log("same version, no need to handover")
+  log("same version, no need to handover")
   Deno.exit(0);
 }
 
 const previousPath = `/opt/ceseal/backups/${previousVersion}`;
-console.log(`Previous ${previousPath}`);
-
-console.log("starting");
-try { Deno.removeSync("/tmp/ceseal.log") } catch (_err) {}
+log(`Previous ${previousPath}`);
+
+const previousStoragePath = path.join(previousPath, "data/storage_files");
+const currentProtectedPath = path.join(currentPath, "data/protected_files");
+const currentStoragePath = path.join(currentPath, "data/storage_files");
+
+log("starting");
+try { Deno.removeSync("/tmp/ceseal.log") } catch (_err) { }
+if (previousVersion == 24013112) {
+  log(`skip the storage files of version ${previousVersion}`);
+  const cmd = ["bash", "-c", `rm -rf ${previousStoragePath}/*`];
+  const p = Deno.run({ cmd });
+  await p.status();
+}
 let oldProcess = await startCeseal(previousPath, "1888");
-await waitCesealStarted("/tmp/ceseal.log");
-console.log("started");
-
-// Waiting old bin start, I'm thinking it's good to not get from api but just dump a file then pass to the new one?
-// await sleep(30)
-
-const command = new Deno.Command(`/opt/ceseal/releases/current/gramine-sgx`, {
-  args: [
-    "ceseal",
-    "--request-handover-from=http://localhost:1888",
-  ],
-  cwd: "/opt/ceseal/releases/current"
-});
-const { code, stdout, stderr } = command.outputSync();
-
-console.log(code);
-console.log(new TextDecoder().decode(stdout));
-console.log(new TextDecoder().decode(stderr));
-
-// oldProcess.kill("SIGKILL");
-
-if (code != 0) {
-  console.log("Handover failed");
-  Deno.exit(1);
+try {
+  await waitCesealStarted("/tmp/ceseal.log");
+  log("started");
+
+  // Waiting old bin start, I'm thinking it's good to not get from api but just dump a file then pass to the new one?
+  // await sleep(30)  
+  try {
+    await ensureDir(currentProtectedPath);
+  } catch (err) {
+    console.error(err.message)
+  }
+  try {
+    await ensureDir(currentStoragePath);
+  } catch (err) {
+    console.error(err.message)
+  }
+
+  const command = new Deno.Command(`/opt/ceseal/releases/current/gramine-sgx`, {
+    args: [
+      "ceseal",
+      "--request-handover-from=http://localhost:1888",
+    ],
+    cwd: "/opt/ceseal/releases/current"
+  });
+  const { code, stdout, stderr } = command.outputSync();
+
+  log(code);
+  log(new TextDecoder().decode(stdout));
+  log(new TextDecoder().decode(stderr));
+
+  if (code != 0) {
+    log("Handover failed");
+    Deno.exit(1);
+  }
+
+} finally {
+  oldProcess.kill();  //the kill() method not to kill the ceseal process due to it's running use exec
+  await killPreviousCeseal(previousVersion as number);
 }
 
-console.log("Handover completed");
+log("Handover completed");
 
 // Copy checkpoint from previous
-const previousStoragePath = path.join(previousPath, "data/storage_files")
-const storagePath = path.join(currentPath, "data/storage_files")
-try { Deno.removeSync(storagePath) } catch (err) { console.error(err.message) }
-try { copySync(previousStoragePath, storagePath) } catch (err) { console.error(err.message) }
+try { Deno.removeSync(currentStoragePath) } catch (err) { console.error(err.message) }
+try { copySync(previousStoragePath, currentStoragePath) } catch (err) { console.error(err.message) }
 
 // Copy current to backups
 try { copySync(currentPath, `/opt/ceseal/backups/${currentVersion}`) } catch (err) { console.error(err.message) }

diff --git a/standalone/teeworker/ceseal/Cargo.toml b/standalone/teeworker/ceseal/Cargo.toml
@@ -1,7 +1,7 @@
 [package]
 edition = "2021"
 name = "ceseal"
-version = "0.1.0"
+version = "0.1.1"
 build = "build.rs"
 
 [profile.release]

diff --git a/standalone/teeworker/ceseal/gramine-build/Makefile b/standalone/teeworker/ceseal/gramine-build/Makefile
@@ -51,7 +51,9 @@ signed-sgx-artifacts: ${BIN_NAME}.manifest ${BIN_NAME}.manifest.sgx ${BIN_NAME}.
 
 .PHONY: ${BIN_FILE}
 ${BIN_FILE}:
+	@echo "IAS_ENV: ${IAS_ENV}"
 	@echo "IAS_SPID: ${IAS_SPID}"
+	@echo "IAS_API_KEY: ${IAS_API_KEY}"
 	make BUILD=${BUILD} -C ..
 
 ${BIN_NAME}: ${BIN_FILE}