Add frontend for zarr import

frontend/javascripts/admin/admin_rest_api.ts

@@ -84,6 +84,7 @@ import * as Utils from "libs/utils";
 import messages from "messages";
 import window, { location } from "libs/window";
 import { SaveQueueType } from "oxalis/model/actions/save_actions";
+import { DatasourceConfiguration } from "types/schemas/datasource.types";
 
 const MAX_SERVER_ITEMS_PER_RESPONSE = 1000;
 
@@ -1431,6 +1432,42 @@ export async function addForeignDataSet(
   });
   return result;
 }
+
+export async function exploreRemoteDataset(
+  remoteUris: string[],
+  credentials?: { username: string; pass: string },
+): Promise<DatasourceConfiguration> {
+  const { dataSource, report } = await Request.sendJSONReceiveJSON("/api/datasets/exploreRemote", {
+    data: credentials
+      ? remoteUris.map((uri) => ({
+          remoteUri: uri,
+          user: credentials.username,
+          password: credentials.pass,
+        }))
+      : remoteUris.map((uri) => ({ remoteUri: uri })),
+  });
+  console.log(report);
+  if (report[1].indexOf("403 Forbidden") !== -1 || report[1].indexOf("401 Unauthorized") !== -1) {
+    Toast.error("The data could not be accessed. Please verify the credentials!");
+    return dataSource;
+  }
+  return dataSource;
+}
+
+export async function storeRemoteDataset(
+  datasetName: string,
+  organizationName: string,
+  datasource: string,
+): Promise<Response> {
+  return doWithToken((token) =>
+    fetch(`/data/datasets/${organizationName}/${datasetName}?token=${token}`, {
+      method: "PUT",
+      headers: { "Content-Type": "application/json" },
+      body: datasource,
+    }),
+  );
+}
+
 // Returns void if the name is valid. Otherwise, a string is returned which denotes the reason.
 export async function isDatasetNameValid(
   datasetId: APIDatasetId,

frontend/javascripts/admin/dataset/dataset_add_view.tsx

@@ -10,6 +10,7 @@ import { enforceActiveUser } from "oxalis/model/accessors/user_accessor";
 import DatasetAddForeignView from "admin/dataset/dataset_add_foreign_view";
 import DatasetAddNeuroglancerView from "admin/dataset/dataset_add_neuroglancer_view";
 import DatasetAddBossView from "admin/dataset/dataset_add_boss_view";
+import DatasetAddZarrView from "admin/dataset/dataset_add_zarr_view";
 import DatasetUploadView from "admin/dataset/dataset_upload_view";
 import features from "features";
 import { getDatastores } from "admin/admin_rest_api";
@@ -128,6 +129,21 @@ function DatasetAddView({ history }: RouteComponentProps) {
             >
               <DatasetUploadView datastores={datastores.own} onUploaded={handleDatasetAdded} />
             </TabPane>
+            <TabPane
+              tab={
+                <span>
+                  <DatabaseOutlined />
+                  Add Zarr Dataset
+                </span>
+              }
+              key="2"
+            >
+              <DatasetAddZarrView
+                datastores={datastores.own}
+                // @ts-expect-error ts-migrate(2322) FIXME: Type '(datasetOrganization: string, uploadedDatase... Remove this comment to see the full error message
+                onAdded={handleDatasetAdded}
+              />
+            </TabPane>
             {datastores.wkConnect.length > 0 && (
               <TabPane
                 tab={
@@ -136,7 +152,7 @@ function DatasetAddView({ history }: RouteComponentProps) {
                     Add Neuroglancer Dataset
                   </span>
                 }
-                key="2"
+                key="3"
               >
                 <DatasetAddNeuroglancerView
                   datastores={datastores.wkConnect}
@@ -153,7 +169,7 @@ function DatasetAddView({ history }: RouteComponentProps) {
                     Add BossDB Dataset
                   </span>
                 }
-                key="3"
+                key="4"
               >
                 <DatasetAddBossView
                   datastores={datastores.wkConnect}
@@ -170,7 +186,7 @@ function DatasetAddView({ history }: RouteComponentProps) {
                     Add Foreign Dataset
                   </span>
                 }
-                key="4"
+                key="5"
               >
                 <DatasetAddForeignView onAdded={() => history.push("/dashboard")} />
               </TabPane>

frontend/javascripts/admin/dataset/dataset_add_zarr_view.tsx

@@ -0,0 +1,217 @@
+import { Form, Input, Button, Col, Row, Divider } from "antd";
+import { connect } from "react-redux";
+import React, { useState } from "react";
+import type { APIUser } from "types/api_flow_types";
+import type { OxalisState } from "oxalis/store";
+import { exploreRemoteDataset, storeRemoteDataset } from "admin/admin_rest_api";
+import messages from "messages";
+import { jsonStringify } from "libs/utils";
+import { CardContainer } from "admin/dataset/dataset_components";
+import Password from "antd/lib/input/Password";
+import TextArea from "antd/lib/input/TextArea";
+import { AsyncButton } from "components/async_clickables";
+import Toast from "libs/toast";
+import DataLayer from "oxalis/model/data_layer";
+import _ from "lodash";
+import { Hint } from "oxalis/view/action-bar/download_modal_view";
+const FormItem = Form.Item;
+
+type OwnProps = {
+  onAdded: (arg0: string, arg1: string) => Promise<void>;
+};
+type StateProps = {
+  activeUser: APIUser | null | undefined;
+};
+type Props = OwnProps & StateProps;
+
+function DatasetAddZarrView(props: Props) {
+  const { activeUser, onAdded } = props;
+  const [datasourceConfig, setDatasourceConfig] = useState<string>();
+  // const [exploreLog, setExploreLog] = useState<string>();
+  const [datasourceUrl, setDatasourceUrl] = useState<string>("");
+  const [usernameOrAccessKey, setUsernameOrAccessKey] = useState<string>("");
+  const [passwordOrSecretKey, setPasswordOrSecretKey] = useState<string>("");
+  const [selectedProtocol, setSelectedProtocol] = useState<string>("https://");
+
+  function validateUrls(userInput: string) {
+    if (
+      (userInput.indexOf("https://") === 0 && userInput.indexOf("s3://") !== 0) ||
+      (userInput.indexOf("https://") !== 0 && userInput.indexOf("s3://") === 0)
+    ) {
+      throw new Error("Dataset URL must employ either the https:// or s3:// protocol.");
+    } else {
+      setSelectedProtocol(userInput.indexOf("https://") === 0 ? "https://" : "s3://");
+    }
+  }
+
+  async function handleExplore() {
+    if (datasourceUrl) {
+      let datasourceToMerge;
+      if (!usernameOrAccessKey || !passwordOrSecretKey) {
+        datasourceToMerge = await exploreRemoteDataset([datasourceUrl]);
+      } else {
+        datasourceToMerge = await exploreRemoteDataset([datasourceUrl], {
+          username: usernameOrAccessKey,
+          pass: passwordOrSecretKey,
+        });
+      }
+      if (datasourceToMerge) {
+        if (datasourceConfig) {
+          // TODO: check that both datasources have same voxel size else warning
+          let currentDatasource;
+          try {
+            currentDatasource = JSON.parse(datasourceConfig);
+          } catch (e) {
+            Toast.error("The loaded datasource config contains invalid JSON.");
+            return;
+          }
+          const layers = currentDatasource.dataLayers.concat(datasourceToMerge.dataLayers);
+          const uniqueLayers = _.uniqBy(layers, (layer: DataLayer) => layer.name);
+          currentDatasource.dataLayers = uniqueLayers;
+          currentDatasource.id.name = `merge_${currentDatasource.id.name}_${datasourceToMerge.id.name}`;
+          setDatasourceConfig(jsonStringify(currentDatasource));
+        } else {
+          setDatasourceConfig(jsonStringify(datasourceToMerge));
+        }
+      } else {
+        Toast.error("Exploring this remote dataset did not return a datasource.");
+      }
+    } else {
+      Toast.error("Please provide a valid URL for exploration.");
+    }
+  }
+
+  async function handleStoreDataset() {
+    if (datasourceConfig && activeUser) {
+      let configJSON;
+      try {
+        configJSON = JSON.parse(datasourceConfig);
+      } catch (e) {
+        Toast.error("The loaded datasource config contains invalid JSON.");
+        return;
+      }
+      const result = await storeRemoteDataset(
+        configJSON.id.name,
+        activeUser.organization,
+        datasourceConfig,
+      );
+      console.log(result);
+      if (result) {
+        onAdded(activeUser.organization, configJSON.id.name);
+      }
+    }
+  }
+
+  return (
+    // Using Forms here only to validate fields and for easy layout
+    <div style={{ padding: 5 }}>
+      <CardContainer title="Add Zarr Dataset">
+        Please enter a URL that points to the Zarr data you would like to import. If necessary,
+        specify the credentials for the dataset. More layers can be added to the loaded datasource
+        using the Add button. Once you have approved of the resulting datasource you can import it.
+        <Form style={{ marginTop: 20 }} layout="vertical">
+          <FormItem
+            name="url"
+            label="Dataset URL"
+            hasFeedback
+            rules={[
+              {
+                required: true,
+                message: messages["dataset.import.required.url"],
+              },
+              {
+                validator: (_rule, value) => {
+                  try {
+                    validateUrls(value);
+                    return Promise.resolve();
+                  } catch (e) {
+                    return Promise.reject(e);
+                  }
+                },
+              },
+            ]}
+            validateFirst
+          >
+            <Input
+              defaultValue={datasourceUrl}
+              onChange={(e) => setDatasourceUrl(e.target.value)}
+            />
+          </FormItem>
+          <Row gutter={8}>
+            <Col span={12}>
+              <FormItem label={selectedProtocol === "https://" ? "Username" : "Access Key"}>
+                <Input
+                  value={usernameOrAccessKey}
+                  onChange={(e) => setUsernameOrAccessKey(e.target.value)}
+                />
+              </FormItem>
+            </Col>
+            <Col span={12}>
+              <FormItem label={selectedProtocol === "https://" ? "Password" : "Secret Key"}>
+                <Password
+                  value={passwordOrSecretKey}
+                  onChange={(e) => setPasswordOrSecretKey(e.target.value)}
+                />
+              </FormItem>
+            </Col>
+          </Row>
+          <FormItem style={{ marginBottom: 0 }}>
+            <AsyncButton
+              size="large"
+              type="default"
+              style={{ width: "100%" }}
+              onClick={handleExplore}
+            >
+              Add
+            </AsyncButton>
+          </FormItem>
+          <Divider />
+          <FormItem label="Datasource">
+            <TextArea
+              rows={4}
+              autoSize={{ minRows: 3, maxRows: 15 }}
+              style={{
+                fontFamily: 'Monaco, Consolas, "Lucida Console", "Courier New", monospace',
+              }}
+              placeholder="No datasource loaded yet"
+              value={datasourceConfig}
+              onChange={(e) => setDatasourceConfig(e.target.value)}
+            />
+          </FormItem>
+          <Row gutter={8}>
+            <Col span={12}>
+              <FormItem>
+                <Button
+                  size="large"
+                  type="default"
+                  style={{ width: "100%" }}
+                  onClick={() => setDatasourceConfig("")}
+                >
+                  Reset
+                </Button>
+              </FormItem>
+            </Col>
+            <Col span={12}>
+              <Button
+                size="large"
+                type="primary"
+                style={{ width: "100%" }}
+                onClick={handleStoreDataset}
+                disabled={!datasourceConfig}
+              >
+                Import
+              </Button>
+            </Col>
+          </Row>
+        </Form>
+      </CardContainer>
+    </div>
+  );
+}
+
+const mapStateToProps = (state: OxalisState): StateProps => ({
+  activeUser: state.activeUser,
+});
+
+const connector = connect(mapStateToProps);
+export default connector(DatasetAddZarrView);

frontend/javascripts/oxalis/view/right-border-tabs/starting_job_modals.tsx

@@ -336,7 +336,7 @@ function StartingJobModal(props: StartingJobModalProps) {
 
       Toast.info(
         <>
-          The {jobName} job has been started. You can look in the{" "}
+          The {jobName} job has been started. See the{" "}
           <a target="_blank" href="/jobs" rel="noopener noreferrer">
             Processing Jobs
           </a>{" "}