diff --git a/packages/openneuro-app/package.json b/packages/openneuro-app/package.json index 62019ce3a..0fa910536 100644 --- a/packages/openneuro-app/package.json +++ b/packages/openneuro-app/package.json @@ -19,7 +19,6 @@ "@emotion/react": "11.11.1", "@emotion/styled": "11.11.0", "@niivue/niivue": "0.45.1", - "@openneuro/client": "^4.28.3", "@openneuro/components": "^4.28.3", "@sentry/react": "^8.25.0", "@tanstack/react-table": "^8.9.3", diff --git a/packages/openneuro-app/src/client.jsx b/packages/openneuro-app/src/client.jsx index 74989cde2..9229f8b32 100644 --- a/packages/openneuro-app/src/client.jsx +++ b/packages/openneuro-app/src/client.jsx @@ -3,14 +3,12 @@ */ import "./scripts/utils/global-polyfill" import "./scripts/sentry" -import { ApolloProvider, InMemoryCache } from "@apollo/client" -import { createClient } from "@openneuro/client" +import { ApolloClient, ApolloProvider, InMemoryCache } from "@apollo/client" import React from "react" import { createRoot } from "react-dom/client" import { BrowserRouter, Route, Routes } from "react-router-dom" import App from "./scripts/app" import Index from "./scripts/index" -import { version } from "./lerna.json" import { config } from "./scripts/config" import * as gtag from "./scripts/utils/gtag" import { relayStylePagination } from "@apollo/client/utilities" @@ -20,22 +18,22 @@ gtag.initialize(config.analytics.trackingIds) const mainElement = document.getElementById("main") const container = createRoot(mainElement) +const client = new ApolloClient({ + uri: `${config.url}/crn/graphql`, + cache: new InMemoryCache({ + typePolicies: { + Query: { + fields: { + advancedSearch: relayStylePagination(), + }, + }, + }, + }), +}) + container.render( - + } /> diff --git a/packages/openneuro-app/src/scripts/dataset/download/download-query.js b/packages/openneuro-app/src/scripts/dataset/download/download-query.js index 2d4190a7d..b4e37b03a 100644 --- a/packages/openneuro-app/src/scripts/dataset/download/download-query.js +++ b/packages/openneuro-app/src/scripts/dataset/download/download-query.js @@ -1,10 +1,43 @@ -import { datasets } from "@openneuro/client" +import { gql } from "@apollo/client" + +export const DOWNLOAD_DATASET = gql` +query downloadDraft($datasetId: ID!, $tree: String) { + dataset(id: $datasetId) { + id + draft { + id + files(tree: $tree) { + id + directory + filename + size + urls + } + } + } +} +` + +export const DOWNLOAD_SNAPSHOT = gql` + query downloadSnapshot($datasetId: ID!, $tag: String!, $tree: String) { + snapshot(datasetId: $datasetId, tag: $tag) { + id + files(tree: $tree) { + id + directory + filename + size + urls + } + } + } +` export const downloadDataset = (client) => async ({ datasetId, snapshotTag, tree = null }) => { if (snapshotTag) { const { data } = await client.query({ - query: datasets.downloadSnapshot, + query: DOWNLOAD_SNAPSHOT, variables: { datasetId, tag: snapshotTag, @@ -14,7 +47,7 @@ export const downloadDataset = return data.snapshot.files } else { const { data } = await client.query({ - query: datasets.downloadDataset, + query: DOWNLOAD_DATASET, variables: { datasetId, tree, diff --git a/packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts b/packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts new file mode 100644 index 000000000..dc4d70ddc --- /dev/null +++ b/packages/openneuro-app/src/scripts/uploader/file-upload-parallel.ts @@ -0,0 +1,122 @@ +/** + * Convert from a URL compatible path + * @param {String} path + */ +export const decodeFilePath = (path) => { + return path.replace(new RegExp(":", "g"), "/") +} + +/** + * Determine parallelism based on Request list + * @param {Array} requests + * @param {number} bytes expected total size of all requests + * @returns {number} + */ +export function uploadParallelism(requests, bytes) { + const averageSize = bytes / requests.length + const parallelism = averageSize / 524288 // 512KB + if (parallelism > 8) { + return 8 + } else if (parallelism < 2) { + return 2 + } else { + return Math.round(parallelism) + } +} + +/** + * Extract filename from Request URL + * @param {string} url .../a:path:to:a:file + */ +export function parseFilename(url) { + const filePath = url.substring(url.lastIndexOf("/") + 1) + return decodeFilePath(filePath) +} + +/** + * Control retry delay for upload file requests + * @param {number} step Attempt number + * @param {Request} request Active request + */ +export async function retryDelay(step, request) { + if (step <= 4) { + await new Promise((r) => setTimeout(r, step ** 2 * 1000)) + } else { + throw new Error( + `Failed to upload file after ${step} attempts - "${request.url}"`, + ) + } +} + +/** + * Repeatable function for single file upload fetch request + * @param {object} uploadProgress Progress controller instance + * @param {typeof fetch} fetch Fetch implementation to use - useful for environments without a native fetch + * @returns {function (Request, number): Promise} + */ +export const uploadFile = + (uploadProgress, fetch) => async (request, attempt = 1) => { + // Create a retry function with attempts incremented + const filename = parseFilename(request.url) + const handleFailure = async (failure) => { + const retryClone = request.clone() + // eslint-disable-next-line no-console + console.warn(`\nRetrying upload for ${filename}: ${failure}`) + try { + await retryDelay(attempt, request) + return uploadFile(uploadProgress, fetch)(retryClone, attempt + 1) + } catch (err) { + if ("failUpload" in uploadProgress) { + uploadProgress.failUpload(filename) + } + throw err + } + } + // This is needed to cancel the request in case of client errors + if ("startUpload" in uploadProgress) { + uploadProgress.startUpload(filename) + } + try { + // Clone before using the request to allow retries to reuse the body + const response = await fetch(request) + if (response.status === 200) { + // We need to wait for the response body or fetch-h2 may leave the connection open + await response.json() + if ("finishUpload" in uploadProgress) { + uploadProgress.finishUpload(filename) + } + uploadProgress.increment() + } else { + await handleFailure(response.statusText) + } + } catch (err) { + await handleFailure(err) + } + } + +/** + * @param {Request[]} requests + * @param {number} totalSize + * @param {object} uploadProgress + * @param {typeof fetch} fetch + */ +export async function uploadParallel( + requests, + totalSize, + uploadProgress, + fetch, +) { + // Array stride of parallel requests + const parallelism = uploadParallelism(requests, totalSize) + for ( + let rIndex = 0; + rIndex < requests.length; + rIndex = rIndex + parallelism + ) { + await Promise.allSettled( + requests + .slice(rIndex, rIndex + parallelism) + .map(uploadFile(uploadProgress, fetch)), + ) + } +} diff --git a/packages/openneuro-app/src/scripts/uploader/file-upload.js b/packages/openneuro-app/src/scripts/uploader/file-upload.js index 33eabbfac..9df2093ca 100644 --- a/packages/openneuro-app/src/scripts/uploader/file-upload.js +++ b/packages/openneuro-app/src/scripts/uploader/file-upload.js @@ -1,5 +1,5 @@ import { config } from "../config" -import { uploads } from "@openneuro/client" +import { uploadParallel } from "./file-upload-parallel" /** * Trim the webkitRelativePath value to only include the dataset relative path @@ -38,7 +38,7 @@ export const getRelativePath = ( */ export const encodeFilePath = (file, options = { stripRelativePath: false }) => file.webkitRelativePath - ? uploads.encodeFilePath(getRelativePath(file, options)) + ? getRelativePath(file, options).replace(new RegExp("/", "g"), ":") : file.name /** @@ -85,5 +85,5 @@ export async function uploadFiles({ // No background fetch // Parallelism is handled by the client in this case - return uploads.uploadParallel(requests, totalSize, uploadProgress, fetch) + return uploadParallel(requests, totalSize, uploadProgress, fetch) } diff --git a/packages/openneuro-app/src/scripts/uploader/hash-file-list.ts b/packages/openneuro-app/src/scripts/uploader/hash-file-list.ts new file mode 100644 index 000000000..4e5f09ea2 --- /dev/null +++ b/packages/openneuro-app/src/scripts/uploader/hash-file-list.ts @@ -0,0 +1,36 @@ +/** + * Java hashcode implementation for browser and Node.js + * @param {string} str + */ +function hashCode(str) { + return str + .split("") + .reduce( + (prevHash, currVal) => + ((prevHash << 5) - prevHash + currVal.charCodeAt(0)) | 0, + 0, + ) +} + +/** + * Calculate a hash from a list of files to upload + * @param {string} datasetId Dataset namespace for this hash + * @param {Array} files Files being uploaded + * @returns {string} Hex string identity hash + */ +export function hashFileList(datasetId, files) { + return Math.abs( + hashCode( + datasetId + + files + .map( + (f) => + `${ + "webkitRelativePath" in f ? f.webkitRelativePath : f.filename + }:${f.size}`, + ) + .sort() + .join(":"), + ), + ).toString(16) +} diff --git a/packages/openneuro-app/src/scripts/uploader/upload-mutation.js b/packages/openneuro-app/src/scripts/uploader/upload-mutation.js index d53314d2b..f4d947e52 100644 --- a/packages/openneuro-app/src/scripts/uploader/upload-mutation.js +++ b/packages/openneuro-app/src/scripts/uploader/upload-mutation.js @@ -1,6 +1,34 @@ -import { datasets, uploads } from "@openneuro/client" +import { gql } from "@apollo/client" import { SUBMIT_METADATA } from "../dataset/mutations/submit-metadata.jsx" +export const CREATE_DATASET = gql` + mutation createDataset($affirmedDefaced: Boolean, $affirmedConsent: Boolean) { + createDataset( + affirmedDefaced: $affirmedDefaced + affirmedConsent: $affirmedConsent + ) { + id + } + } +` + +export const PREPARE_UPLOAD = gql` + mutation prepareUpload($datasetId: ID!, $uploadId: ID!) { + prepareUpload(datasetId: $datasetId, uploadId: $uploadId) { + id + datasetId + token + endpoint + } + } +` + +export const FINISH_UPLOAD = gql` + mutation finishUpload($uploadId: ID!) { + finishUpload(uploadId: $uploadId) + } +` + /** * Create a dataset and update the label * @param {object} client Apollo client @@ -9,7 +37,7 @@ export const createDataset = (client) => ({ affirmedDefaced, affirmedConsent }) => { return client .mutate({ - mutation: datasets.createDataset, + mutation: CREATE_DATASET, variables: { affirmedDefaced, affirmedConsent }, errorPolicy: "all", }) @@ -22,7 +50,7 @@ export const createDataset = */ export const prepareUpload = (client) => ({ datasetId, uploadId }) => { return client.mutate({ - mutation: uploads.prepareUpload, + mutation: PREPARE_UPLOAD, variables: { datasetId, uploadId }, }) } @@ -33,7 +61,7 @@ export const prepareUpload = (client) => ({ datasetId, uploadId }) => { */ export const finishUpload = (client) => (uploadId) => { return client.mutate({ - mutation: uploads.finishUpload, + mutation: FINISH_UPLOAD, variables: { uploadId }, }) } diff --git a/packages/openneuro-app/src/scripts/uploader/uploader.jsx b/packages/openneuro-app/src/scripts/uploader/uploader.jsx index 8601e165c..a42a05226 100644 --- a/packages/openneuro-app/src/scripts/uploader/uploader.jsx +++ b/packages/openneuro-app/src/scripts/uploader/uploader.jsx @@ -3,17 +3,37 @@ import { toast } from "react-toastify" import ToastContent from "../common/partials/toast-content.jsx" import React from "react" import PropTypes from "prop-types" -import { ApolloConsumer } from "@apollo/client" +import { ApolloConsumer, gql } from "@apollo/client" import * as gtag from "../utils/gtag" import UploaderContext from "./uploader-context.js" import FileSelect from "./file-select" import { locationFactory } from "./uploader-location.js" import * as mutation from "./upload-mutation.js" -import { datasets, uploads } from "@openneuro/client" import { useNavigate } from "react-router-dom" import { uploadFiles } from "./file-upload.js" import { UploadProgress } from "./upload-progress-class" import { addPathToFiles } from "./add-path-to-files.js" +import { hashFileList } from "./hash-file-list" + +// Get only working tree files +const GET_DRAFT_FILES = gql` + query dataset($id: ID!, $tree: String) { + dataset(id: $id) { + id + draft { + id + files(tree: $tree) { + filename + size + } + } + metadata { + affirmedDefaced + affirmedConsent + } + } + } +` /** * Stateful uploader workflow and status @@ -101,7 +121,7 @@ export class UploadClient extends React.Component { return ({ files }) => { this.props.client .query({ - query: datasets.getDraftFiles, + query: GET_DRAFT_FILES, variables: { id: datasetId }, }) .then( @@ -279,7 +299,7 @@ export class UploadClient extends React.Component { }, } = await mutation.prepareUpload(this.props.client)({ datasetId: this.state.datasetId, - uploadId: uploads.hashFileList(this.state.datasetId, filesToUpload), + uploadId: hashFileList(this.state.datasetId, filesToUpload), }) try {