diff --git a/CHANGELOG.unreleased.md b/CHANGELOG.unreleased.md index 22acd7221e3..34b1207cef2 100644 --- a/CHANGELOG.unreleased.md +++ b/CHANGELOG.unreleased.md @@ -21,6 +21,7 @@ For upgrade instructions, please check the [migration guide](MIGRATIONS.released - Added support for 64-bit segmentations (uint64). Note that the actual support only covers IDs up to 2^53 - 1 (matches the JavaScript number type). Also note that JSON mappings are only compatible with segment ids which only use the lower 32 bits. [#6317](https://github.com/scalableminds/webknossos/pull/6317) - Added a "duplicate" button for annotations. [#6386](https://github.com/scalableminds/webknossos/pull/6386) - Added new filter options for the dataset list api at `api/datasets`: `organizationName: String`, `onlyMyOrganization: Boolean`, `uploaderId: String` [#6377](https://github.com/scalableminds/webknossos/pull/6377) +- The Add Dataset view now hosts an Add Zarr Dataset tab to explore, configure and import remote Zarr datasets. [#6383](https://github.com/scalableminds/webknossos/pull/6383) ### Changed - webKnossos uses WebGL 2 instead of WebGL 1 now. In case your browser/hardware does not support this, webKnossos will alert you and you need to upgrade your system. [#6350](https://github.com/scalableminds/webknossos/pull/6350) diff --git a/frontend/javascripts/admin/admin_rest_api.ts b/frontend/javascripts/admin/admin_rest_api.ts index c3f4479c5c2..32ed1329d54 100644 --- a/frontend/javascripts/admin/admin_rest_api.ts +++ b/frontend/javascripts/admin/admin_rest_api.ts @@ -84,6 +84,7 @@ import * as Utils from "libs/utils"; import messages from "messages"; import window, { location } from "libs/window"; import { SaveQueueType } from "oxalis/model/actions/save_actions"; +import { DatasourceConfiguration } from "types/schemas/datasource.types"; const MAX_SERVER_ITEMS_PER_RESPONSE = 1000; @@ -1432,6 +1433,45 @@ export async function addForeignDataSet( }); return result; } + +type ExplorationResult = { + dataSource: DatasourceConfiguration | undefined; + report: string; +}; + +export async function exploreRemoteDataset( + remoteUris: string[], + credentials?: { username: string; pass: string }, +): Promise { + const { dataSource, report } = await Request.sendJSONReceiveJSON("/api/datasets/exploreRemote", { + data: credentials + ? remoteUris.map((uri) => ({ + remoteUri: uri, + user: credentials.username, + password: credentials.pass, + })) + : remoteUris.map((uri) => ({ remoteUri: uri })), + }); + if (report.indexOf("403 Forbidden") !== -1 || report.indexOf("401 Unauthorized") !== -1) { + Toast.error("The data could not be accessed. Please verify the credentials!"); + } + return { dataSource, report }; +} + +export async function storeRemoteDataset( + datasetName: string, + organizationName: string, + datasource: string, +): Promise { + return doWithToken((token) => + fetch(`/data/datasets/${organizationName}/${datasetName}?token=${token}`, { + method: "PUT", + headers: { "Content-Type": "application/json" }, + body: datasource, + }), + ); +} + // Returns void if the name is valid. Otherwise, a string is returned which denotes the reason. export async function isDatasetNameValid( datasetId: APIDatasetId, diff --git a/frontend/javascripts/admin/dataset/dataset_add_view.tsx b/frontend/javascripts/admin/dataset/dataset_add_view.tsx index 46efa6da899..06fad68102b 100644 --- a/frontend/javascripts/admin/dataset/dataset_add_view.tsx +++ b/frontend/javascripts/admin/dataset/dataset_add_view.tsx @@ -10,6 +10,7 @@ import { enforceActiveUser } from "oxalis/model/accessors/user_accessor"; import DatasetAddForeignView from "admin/dataset/dataset_add_foreign_view"; import DatasetAddNeuroglancerView from "admin/dataset/dataset_add_neuroglancer_view"; import DatasetAddBossView from "admin/dataset/dataset_add_boss_view"; +import DatasetAddZarrView from "admin/dataset/dataset_add_zarr_view"; import DatasetUploadView from "admin/dataset/dataset_upload_view"; import features from "features"; import { getDatastores } from "admin/admin_rest_api"; @@ -128,6 +129,21 @@ function DatasetAddView({ history }: RouteComponentProps) { > + + + Add Remote Zarr Dataset + + } + key="2" + > + + {datastores.wkConnect.length > 0 && ( } - key="2" + key="3" > } - key="3" + key="4" > } - key="4" + key="5" > history.push("/dashboard")} /> diff --git a/frontend/javascripts/admin/dataset/dataset_add_zarr_view.tsx b/frontend/javascripts/admin/dataset/dataset_add_zarr_view.tsx new file mode 100644 index 00000000000..cf919ee6cca --- /dev/null +++ b/frontend/javascripts/admin/dataset/dataset_add_zarr_view.tsx @@ -0,0 +1,319 @@ +import { Form, Input, Button, Col, Radio, Row, Collapse } from "antd"; +import { connect } from "react-redux"; +import React, { useState } from "react"; +import type { APIUser } from "types/api_flow_types"; +import type { OxalisState } from "oxalis/store"; +import { exploreRemoteDataset, isDatasetNameValid, storeRemoteDataset } from "admin/admin_rest_api"; +import messages from "messages"; +import { jsonStringify } from "libs/utils"; +import { CardContainer } from "admin/dataset/dataset_components"; +import Password from "antd/lib/input/Password"; +import TextArea from "antd/lib/input/TextArea"; +import { AsyncButton } from "components/async_clickables"; +import Toast from "libs/toast"; +import _ from "lodash"; +import { Hint } from "oxalis/view/action-bar/download_modal_view"; +import { formatScale } from "libs/format_utils"; +import { DataLayer, DatasourceConfiguration } from "types/schemas/datasource.types"; +const { Panel } = Collapse; +const FormItem = Form.Item; +const RadioGroup = Radio.Group; + +type OwnProps = { + onAdded: (arg0: string, arg1: string) => Promise; +}; +type StateProps = { + activeUser: APIUser | null | undefined; +}; +type Props = OwnProps & StateProps; + +function ensureLargestSegmentIdsInPlace(datasource: DatasourceConfiguration) { + for (const layer of datasource.dataLayers) { + if (layer.category === "color" || layer.largestSegmentId == null) { + continue; + } + layer.largestSegmentId = 1; + Toast.warning(`Please adapt the largestSegmentID for layer ${layer.name}.`); + } +} + +function mergeNewLayers( + loadedDatasource: DatasourceConfiguration, + datasourceToMerge: DatasourceConfiguration, +): DatasourceConfiguration { + const allLayers = datasourceToMerge.dataLayers.concat(loadedDatasource.dataLayers); + const groupedLayers = _.groupBy(allLayers, (layer: DataLayer) => layer.name) as unknown as Record< + string, + DataLayer[] + >; + const uniqueLayers: DataLayer[] = []; + for (const entry of _.entries(groupedLayers)) { + const [name, layerGroup] = entry; + if (layerGroup.length === 1) { + uniqueLayers.push(layerGroup[0]); + } else { + let idx = 1; + for (const layer of layerGroup) { + if (idx === 1) { + uniqueLayers.push(layer); + } else { + uniqueLayers.push({ ...layer, name: `${name}_${idx}` }); + } + idx++; + } + } + } + return { + ...loadedDatasource, + dataLayers: uniqueLayers, + id: { + ...loadedDatasource.id, + name: `${loadedDatasource.id.name}_and_${datasourceToMerge.id.name}`, + }, + }; +} + +function DatasetAddZarrView(props: Props) { + const { activeUser, onAdded } = props; + const [datasourceConfig, setDatasourceConfig] = useState(""); + const [exploreLog, setExploreLog] = useState(""); + const [datasourceUrl, setDatasourceUrl] = useState(""); + const [showCredentialsFields, setShowCredentialsFields] = useState(false); + const [usernameOrAccessKey, setUsernameOrAccessKey] = useState(""); + const [passwordOrSecretKey, setPasswordOrSecretKey] = useState(""); + const [selectedProtocol, setSelectedProtocol] = useState<"s3" | "https">("https"); + + function validateUrls(userInput: string) { + if ( + (userInput.indexOf("https://") === 0 && userInput.indexOf("s3://") !== 0) || + (userInput.indexOf("https://") !== 0 && userInput.indexOf("s3://") === 0) + ) { + setSelectedProtocol(userInput.indexOf("https://") === 0 ? "https" : "s3"); + setShowCredentialsFields(userInput.indexOf("s3://") === 0); + } else { + throw new Error("Dataset URL must employ either the https:// or s3:// protocol."); + } + } + + async function handleExplore() { + if (!datasourceUrl) { + Toast.error("Please provide a valid URL for exploration."); + return; + } + const { dataSource, report } = + !usernameOrAccessKey || !passwordOrSecretKey + ? await exploreRemoteDataset([datasourceUrl]) + : await exploreRemoteDataset([datasourceUrl], { + username: usernameOrAccessKey, + pass: passwordOrSecretKey, + }); + setExploreLog(report); + if (!dataSource) { + Toast.error("Exploring this remote dataset did not return a datasource."); + return; + } + ensureLargestSegmentIdsInPlace(dataSource); + if (!datasourceConfig) { + setDatasourceConfig(jsonStringify(dataSource)); + return; + } + let loadedDatasource; + try { + loadedDatasource = JSON.parse(datasourceConfig); + } catch (e) { + Toast.error( + "The current datasource config contains invalid JSON. Cannot add the new Zarr data.", + ); + return; + } + if (!_.isEqual(loadedDatasource.scale, dataSource.scale)) { + Toast.warning( + `${messages["dataset.add_zarr_different_scale_warning"]}\n${formatScale(dataSource.scale)}`, + { timeout: 10000 }, + ); + } + setDatasourceConfig(jsonStringify(mergeNewLayers(loadedDatasource, dataSource))); + } + + async function handleStoreDataset() { + if (datasourceConfig && activeUser) { + let configJSON; + try { + configJSON = JSON.parse(datasourceConfig); + const nameValidationResult = await isDatasetNameValid({ + name: configJSON.id.name, + owningOrganization: activeUser.organization, + }); + if (nameValidationResult) { + throw new Error(nameValidationResult); + } + const response = await storeRemoteDataset( + configJSON.id.name, + activeUser.organization, + datasourceConfig, + ); + if (response.status !== 200) { + const errorJSONString = JSON.stringify(await response.json()); + throw new Error(`${response.status} ${response.statusText} ${errorJSONString}`); + } + } catch (e) { + Toast.error(`The datasource config could not be stored. ${e}`); + return; + } + onAdded(activeUser.organization, configJSON.id.name); + } + } + + return ( + // Using Forms here only to validate fields and for easy layout +
+ + Please enter a URL that points to the Zarr data you would like to import. If necessary, + specify the credentials for the dataset. More layers can be added to the datasource + specification below using the Add button. Once you have approved of the resulting datasource + you can import it. +
+ { + try { + validateUrls(value); + return Promise.resolve(); + } catch (e) { + return Promise.reject(e); + } + }, + }, + ]} + validateFirst + > + setDatasourceUrl(e.target.value)} + /> + + + setShowCredentialsFields(e.target.value === "show")} + > + + {selectedProtocol === "https" ? "None" : "Anonymous"} + + + {selectedProtocol === "https" ? "Basic authentication" : "With credentials"} + + + + {showCredentialsFields ? ( + + + + setUsernameOrAccessKey(e.target.value)} + /> + + + + + setPasswordOrSecretKey(e.target.value)} + /> + + + + ) : null} + + + + + + Add Layer + + + + + + + +
{exploreLog}
+
+
+
+ +