Implement Deno CLI for schema validator uploads

cli/README.md

@@ -0,0 +1,3 @@
+# OpenNeuro CLI for Deno
+
+Simplified CLI for OpenNeuro implemented in Deno. Deno eliminates the need to install the CLI and allows for more code reuse with OpenNeuro's web frontend.

cli/openneuro.ts

@@ -0,0 +1,10 @@
+/**
+ * Entrypoint for OpenNeuro CLI
+ */
+import { commandLine } from "./src/options.ts"
+
+export async function main() {
+  await commandLine(Deno.args)
+}
+
+await main()

cli/src/commands/download.ts

@@ -0,0 +1,6 @@
+import { Command } from "../deps.ts"
+
+export const download = new Command()
+  .name("download")
+  .description("Download a dataset from OpenNeuro")
+  .arguments("<accession_number> <dataset_directory>")

cli/src/commands/git-credential.test.ts

@@ -0,0 +1,17 @@
+import { assertEquals } from "../deps.ts"
+import { gitCredentialAction } from "./git-credential.ts"
+
+Deno.test("git-credential parses stdin correctly", async () => {
+  const stdin = new ReadableStream<Uint8Array>({
+    start(controller) {
+      controller.enqueue(
+        new TextEncoder().encode(
+          "host=staging.openneuro.org\nprotocol=https\npath=/datasets/ds000001\n",
+        ),
+      )
+      controller.close()
+    },
+  })
+  const output = await gitCredentialAction(stdin, () => "token")
+  assertEquals(output, "username=@openneuro/cli\npassword=token\n")
+})

cli/src/commands/git-credential.ts

@@ -0,0 +1,68 @@
+import { Command, TextLineStream } from "../deps.ts"
+
+const prepareRepoAccess = `
+  mutation prepareRepoAccess($datasetId: ID!) {
+    prepareRepoAccess(datasetId: $datasetId) {
+      token
+      endpoint
+    }
+  }
+`
+
+export function getRepoToken(datasetId?: string) {
+  /*
+  return client
+    .mutate({
+      mutation: prepareRepoAccess,
+      variables: {
+        datasetId,
+      },
+    })
+    .then(({ data }) => data.prepareRepoAccess.token)
+    */
+  return "token"
+}
+
+/**
+ * Provide a git-credential helper for OpenNeuro
+ */
+export async function gitCredentialAction(
+  stdinReadable: ReadableStream<Uint8Array> = Deno.stdin.readable,
+  tokenGetter = getRepoToken,
+) {
+  let pipeOutput = ""
+  const credential: Record<string, string | undefined> = {}
+  // Create a stream of lines from stdin
+  const lineStream = stdinReadable
+    .pipeThrough(new TextDecoderStream())
+    .pipeThrough(new TextLineStream())
+  for await (const line of lineStream) {
+    const [key, value] = line.split("=", 2)
+    credential[key] = value
+  }
+  if ("path" in credential && credential.path) {
+    const datasetId = credential.path.split("/").pop()
+    const token = await tokenGetter(datasetId)
+    const output: Record<string, string> = {
+      username: "@openneuro/cli",
+      password: token,
+    }
+    for (const key in output) {
+      pipeOutput += `${key}=${output[key]}\n`
+    }
+  } else {
+    throw new Error(
+      "Invalid input from git, check the credential helper is configured correctly",
+    )
+  }
+  return pipeOutput
+}
+
+export const gitCredential = new Command()
+  .name("git-credential")
+  .description(
+    "A git credentials helper for easier datalad or git-annex access to datasets.",
+  )
+  .action(() => {
+    console.log(gitCredentialAction())
+  })

cli/src/commands/login.test.ts

@@ -0,0 +1,33 @@
+import { assertEquals, assertSpyCalls, Select, stub } from "../deps.ts"
+import { loginAction } from "./login.ts"
+
+Deno.test("login action supports non-interactive mode if all options are provided", async () => {
+  const SelectStub = stub(Select, "prompt", () => {
+    return new Promise<void>(() => {})
+  })
+  await loginAction({
+    url: "https://example.com",
+    token: "1234",
+    errorReporting: false,
+  })
+  // Test to make sure we get here before the timeout
+  assertSpyCalls(SelectStub, 0)
+  SelectStub.restore()
+  localStorage.clear()
+})
+
+Deno.test("login action sets values in localStorage", async () => {
+  const loginOptions = {
+    url: "https://example.com",
+    token: "1234",
+    errorReporting: true,
+  }
+  await loginAction(loginOptions)
+  assertEquals(localStorage.getItem("url"), loginOptions.url)
+  assertEquals(localStorage.getItem("token"), loginOptions.token)
+  assertEquals(
+    localStorage.getItem("errorReporting"),
+    loginOptions.errorReporting.toString(),
+  )
+  localStorage.clear()
+})

cli/src/commands/login.ts

@@ -0,0 +1,69 @@
+/**
+ * Configure credentials and other persistent settings for OpenNeuro
+ */
+import { Command, Confirm, Secret, Select } from "../deps.ts"
+import type { CommandOptions } from "../deps.ts"
+import { LoginError } from "../error.ts"
+
+export interface ClientConfig {
+  url: string
+  token: string
+  errorReporting: boolean
+}
+
+const messages = {
+  url:
+    "URL for OpenNeuro instance to upload to (e.g. `https://openneuro.org`).",
+  token: "API key for OpenNeuro. See https://openneuro.org/keygen",
+  errorReporting:
+    "Enable error reporting. Errors and performance metrics are sent to the configured OpenNeuro instance.",
+}
+
+/**
+ * Get credentials from local storage
+ */
+export function getConfig(): ClientConfig {
+  const url = localStorage.getItem("url")
+  const token = localStorage.getItem("token")
+  const errorReporting = localStorage.getItem("errorReporting") === "true"
+  if (url && token && errorReporting) {
+    const config: ClientConfig = {
+      url,
+      token,
+      errorReporting,
+    }
+    return config
+  } else {
+    throw new LoginError("Run `openneuro login` before upload.")
+  }
+}
+
+export async function loginAction(options: CommandOptions) {
+  const url = options.url ? options.url : await Select.prompt({
+    message: "Choose an OpenNeuro instance to use.",
+    options: [
+      "https://openneuro.org",
+      "https://staging.openneuro.org",
+      "http://localhost:9876",
+    ],
+  })
+  localStorage.setItem("url", url)
+  const token = options.token ? options.token : await Secret.prompt(
+    `Enter your API key for OpenNeuro (get an API key from ${url}/keygen).`,
+  )
+  localStorage.setItem("token", token)
+  const errorReporting = options.hasOwnProperty("errorReporting")
+    ? options.errorReporting
+    : await Confirm.prompt(messages.errorReporting)
+  localStorage.setItem("errorReporting", errorReporting.toString())
+}
+
+export const login = new Command()
+  .name("login")
+  .description(
+    "Setup credentials for OpenNeuro. Set -u, -t, and -e flags to skip interactive prompts.",
+  )
+  .option("-u, --url <url>", messages.url)
+  .option("-t, --token <token>", messages.token)
+  .option("-e, --error-reporting <boolean>", messages.errorReporting)
+  .action(loginAction)

cli/src/commands/upload.ts

@@ -0,0 +1,120 @@
+import { FetchHttpStack } from "../fetchHttpStack.ts"
+import { validateCommand } from "./validate.ts"
+import { ClientConfig, getConfig } from "./login.ts"
+import { logger } from "../logger.ts"
+import {
+  Confirm,
+  ProgressBar,
+  relative,
+  resolve,
+  Tus,
+  Uppy,
+  walk,
+} from "../deps.ts"
+import type { CommandOptions } from "../deps.ts"
+
+export function readConfig(): ClientConfig {
+  const config = getConfig()
+  logger.info(
+    `configured with URL "${config.url}" and token "${
+      config.token.slice(
+        0,
+        3,
+      )
+    }...${config.token.slice(-3)}`,
+  )
+  return config
+}
+
+export async function uploadAction(
+  options: CommandOptions,
+  dataset_directory: string,
+) {
+  const clientConfig = readConfig()
+  const dataset_directory_abs = resolve(dataset_directory)
+  logger.info(
+    `upload ${dataset_directory} resolved to ${dataset_directory_abs}`,
+  )
+
+  // TODO - call the validator here
+
+  let datasetId = "ds001001"
+  if (options.dataset) {
+    datasetId = options.dataset
+  } else {
+    if (!options.create) {
+      const confirmation = await new Confirm(
+        "Confirm creation of a new dataset?",
+      )
+      if (!confirmation) {
+        console.log("Specify --dataset to upload to an existing dataset.")
+        return
+      }
+    }
+    // TODO Create dataset here
+    datasetId = "ds001001"
+  }
+  // Setup upload
+  const uppy = new Uppy({
+    id: "@openneuro/cli",
+    autoProceed: true,
+    debug: true,
+  }).use(Tus, {
+    endpoint: "http://localhost:9876/tusd/files/",
+    chunkSize: 64000000, // ~64MB
+    uploadLengthDeferred: true,
+    headers: {
+      Authorization: `Bearer ${clientConfig.token}`,
+    },
+    httpStack: new FetchHttpStack(),
+  })
+
+  const progressBar = new ProgressBar({
+    title: "Upload",
+    total: 100,
+  })
+  progressBar.render(0)
+  uppy.on("progress", (progress) => {
+    progressBar.render(progress)
+  })
+
+  // Upload all files
+  for await (
+    const walkEntry of walk(dataset_directory, {
+      includeDirs: false,
+      includeSymlinks: false,
+    })
+  ) {
+    const file = await Deno.open(walkEntry.path)
+    const relativePath = relative(dataset_directory_abs, walkEntry.path)
+    const uppyFile = {
+      name: walkEntry.name,
+      data: file.readable.getReader(),
+      meta: {
+        datasetId,
+        relativePath,
+      },
+    }
+    logger.debug(JSON.stringify({ name: uppyFile.name, meta: uppyFile.meta }))
+    uppy.addFile(uppyFile)
+  }
+}
+
+/**
+ * Upload is validate extended with upload features
+ */
+export const upload = validateCommand
+  .name("upload")
+  .description("Upload a dataset to OpenNeuro")
+  .option("--json", "Hidden for upload usage", { hidden: true, override: true })
+  .option("--filenameMode", "Hidden for upload usage", {
+    hidden: true,
+    override: true,
+  })
+  .option("-d, --dataset", "Specify an existing dataset to update.", {
+    conflicts: ["create"],
+  })
+  .option("-c, --create", "Skip confirmation to create a new dataset.", {
+    conflicts: ["dataset"],
+  })
+  .action(uploadAction)

cli/src/commands/validate.ts

@@ -0,0 +1,28 @@
+import { Command } from "../deps.ts"
+
+export const validateCommand = new Command()
+  .name("bids-validator")
+  .description(
+    "This tool checks if a dataset in a given directory is compatible with the Brain Imaging Data Structure specification. To learn more about Brain Imaging Data Structure visit http://bids.neuroimaging.io",
+  )
+  .arguments("<dataset_directory>")
+  .version("alpha")
+  .option("--json", "Output machine readable JSON")
+  .option(
+    "-s, --schema <type:string>",
+    "Specify a schema version to use for validation",
+    {
+      default: "latest",
+    },
+  )
+  .option("-v, --verbose", "Log more extensive information about issues")
+  .option(
+    "--ignoreNiftiHeaders",
+    "Disregard NIfTI header content during validation",
+  )
+  .option(
+    "--filenameMode",
+    "Enable filename checks for newline separated filenames read from stdin",
+  )
+
+export const validate = validateCommand