From 656e9449bf869cf49ff761a600cb13f4efdcb168 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Wed, 27 Feb 2019 13:30:41 +0100 Subject: [PATCH 01/13] add first version of add remote dataset view --- .../admin/dataset/dataset_add_remote_view.js | 185 ++++++++++++++++++ .../admin/dataset/dataset_add_view.js | 23 ++- 2 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 frontend/javascripts/admin/dataset/dataset_add_remote_view.js diff --git a/frontend/javascripts/admin/dataset/dataset_add_remote_view.js b/frontend/javascripts/admin/dataset/dataset_add_remote_view.js new file mode 100644 index 00000000000..4cefbadb0c7 --- /dev/null +++ b/frontend/javascripts/admin/dataset/dataset_add_remote_view.js @@ -0,0 +1,185 @@ +// @flow +import { Form, Input, Select, Button, Card, Col, Row } from "antd"; +import { connect } from "react-redux"; +import React from "react"; + +import type { APIDataStore, APIUser } from "admin/api_flow_types"; +import type { OxalisState } from "oxalis/store"; +import { getDatastores, isDatasetNameValid } from "admin/admin_rest_api"; +import messages from "messages"; +import { trackAction } from "oxalis/model/helpers/analytics"; + +const FormItem = Form.Item; +const { Option } = Select; + +type OwnProps = {| + withoutCard?: boolean, + onAdded: (string, string) => void, +|}; +type StateProps = {| + activeUser: ?APIUser, +|}; +type Props = {| ...OwnProps, ...StateProps |}; +type PropsWithForm = {| + ...Props, + form: Object, +|}; + +type State = { + datastores: Array, +}; + +class DatasetAddRemoteView extends React.PureComponent { + state = { + datastores: [], + }; + + componentDidMount() { + this.fetchData(); + } + + async fetchData() { + const datastores = await getDatastores(); + + this.setState({ + datastores, + }); + + if (datastores.length > 0) { + this.props.form.setFieldsValue({ datastore: datastores[0].url }); + } + } + + validateUrl(url: string) { + const delimiterIndex = url.indexOf("#!"); + if (delimiterIndex < 0) { + return "The URL doesn't contain the #! delimiter. Please insert the full URL."; + } + + const jsonConfig = url.slice(delimiterIndex + 2); + try { + const config = JSON.parse(decodeURIComponent(jsonConfig)); + console.log(config.layers); + return null; + } catch (error) { + return error; + } + } + + handleSubmit() { + console.log("Submit"); + trackAction("Added remote dataset"); + } + + render() { + const { getFieldDecorator } = this.props.form; + + const Container = ({ children }) => { + if (this.props.withoutCard) { + return {children}; + } else { + return ( + Add Remote Dataset} + > + {children} + + ); + } + }; + return ( +
+ +
+ + + + {getFieldDecorator("name", { + rules: [ + { required: true, message: messages["dataset.import.required.name"] }, + { min: 3 }, + { pattern: /[0-9a-zA-Z_-]+$/ }, + { + validator: async (_rule, value, callback) => { + if (!this.props.activeUser) + throw new Error("Can't do operation if no user is logged in."); + const reasons = await isDatasetNameValid({ + name: value, + owningOrganization: this.props.activeUser.organization, + }); + if (reasons != null) { + callback(reasons); + } else { + callback(); + } + }, + }, + ], + validateFirst: true, + })()} + + + + + {getFieldDecorator("datastore", { + rules: [ + { required: true, message: messages["dataset.import.required.datastore"] }, + ], + })( + , + )} + + + + + {getFieldDecorator("url", { + rules: [ + { required: true, message: messages["dataset.import.required.name"] }, + { min: 3 }, + { pattern: /[0-9a-zA-Z_-]+$/ }, + { + validator: async (_rule, value, callback) => { + const reasons = this.validateUrl(value); + if (reasons != null) { + callback(reasons); + } else { + callback(); + } + }, + }, + ], + validateFirst: true, + })()} + + + + +
+ +
+ ); + } +} + +const mapStateToProps = (state: OxalisState): StateProps => ({ + activeUser: state.activeUser, +}); + +export default connect(mapStateToProps)( + Form.create()(DatasetAddRemoteView), +); diff --git a/frontend/javascripts/admin/dataset/dataset_add_view.js b/frontend/javascripts/admin/dataset/dataset_add_view.js index 14549206d6f..368925e6a48 100644 --- a/frontend/javascripts/admin/dataset/dataset_add_view.js +++ b/frontend/javascripts/admin/dataset/dataset_add_view.js @@ -4,6 +4,7 @@ import { Tabs, Icon } from "antd"; import React from "react"; import DatasetAddForeignView from "admin/dataset/dataset_add_foreign_view"; +import DatasetAddRemoteView from "admin/dataset/dataset_add_remote_view"; import DatasetUploadView from "admin/dataset/dataset_upload_view"; import features from "features"; @@ -31,15 +32,31 @@ const DatasetAddView = ({ history }: Props) => ( }} /> - {features().addForeignDataset ? ( + + + Add Remote Dataset + + } + key="2" + > + { + const url = `/datasets/${organization}/${datasetName}/import`; + history.push(url); + }} + /> + + {features().addForeignDataset || true ? ( - Add foreign Dataset + Add Foreign Dataset } - key="2" + key="3" > history.push("/dashboard")} /> From ca790e8df095ec35b5edef940f49f2ba63fdee6a Mon Sep 17 00:00:00 2001 From: Florian M Date: Wed, 27 Feb 2019 13:54:24 +0100 Subject: [PATCH 02/13] add isConnector property to datastores --- app/controllers/InitialDataController.scala | 4 ++-- app/models/binary/DataSetService.scala | 2 +- app/models/binary/DataStore.scala | 13 ++++++++----- conf/evolutions/041-add-datastore-isconnector.sql | 13 +++++++++++++ .../reversions/041-add-datastore-isconnector.sql | 11 +++++++++++ tools/postgres/schema.sql | 5 +++-- 6 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 conf/evolutions/041-add-datastore-isconnector.sql create mode 100644 conf/evolutions/reversions/041-add-datastore-isconnector.sql diff --git a/app/controllers/InitialDataController.scala b/app/controllers/InitialDataController.scala index a744bff6cad..cb672626c94 100644 --- a/app/controllers/InitialDataController.scala +++ b/app/controllers/InitialDataController.scala @@ -189,7 +189,7 @@ Samplecountry if (conf.Datastore.enabled) { dataStoreDAO.findOneByName("localhost").futureBox.map { maybeStore => if (maybeStore.isEmpty) { - logger.info("inserting local datastore"); + logger.info("inserting local datastore") dataStoreDAO.insertOne(DataStore("localhost", conf.Http.uri, conf.Datastore.key)) } } @@ -199,7 +199,7 @@ Samplecountry if (conf.Tracingstore.enabled) { tracingStoreDAO.findOneByName("localhost").futureBox.map { maybeStore => if (maybeStore.isEmpty) { - logger.info("inserting local tracingstore"); + logger.info("inserting local tracingstore") tracingStoreDAO.insertOne(TracingStore("localhost", conf.Http.uri, conf.Tracingstore.key)) } } diff --git a/app/models/binary/DataSetService.scala b/app/models/binary/DataSetService.scala index 50b837a9fda..ca33a0588cf 100644 --- a/app/models/binary/DataSetService.scala +++ b/app/models/binary/DataSetService.scala @@ -103,7 +103,7 @@ class DataSetService @Inject()(organizationDAO: OrganizationDAO, .getWithJsonResponse[InboxDataSource] def addForeignDataStore(name: String, url: String)(implicit ctx: DBAccessContext): Fox[Unit] = { - val dataStore = DataStore(name, url, "", isForeign = true) // the key can be "" because keys are only important for own DataStore. Own Datastores have a key that is not "" + val dataStore = DataStore(name, url, "", isForeign = true, isConnector = false) // the key can be "" because keys are only important for own DataStore. Own Datastores have a key that is not "" for { _ <- dataStoreDAO.insertOne(dataStore) } yield () diff --git a/app/models/binary/DataStore.scala b/app/models/binary/DataStore.scala index 33576809081..080fc041f93 100644 --- a/app/models/binary/DataStore.scala +++ b/app/models/binary/DataStore.scala @@ -19,7 +19,8 @@ case class DataStore( key: String, isScratch: Boolean = false, isDeleted: Boolean = false, - isForeign: Boolean = false + isForeign: Boolean = false, + isConnector: Boolean = false ) class DataStoreService @Inject()(dataStoreDAO: DataStoreDAO)(implicit ec: ExecutionContext) @@ -32,7 +33,8 @@ class DataStoreService @Inject()(dataStoreDAO: DataStoreDAO)(implicit ec: Execut "name" -> dataStore.name, "url" -> dataStore.url, "isForeign" -> dataStore.isForeign, - "isScratch" -> dataStore.isScratch + "isScratch" -> dataStore.isScratch, + "isConnector" -> dataStore.isConnector )) def validateAccess[A](name: String)(block: (DataStore) => Future[Result])(implicit request: Request[A], @@ -61,7 +63,8 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext r.key, r.isscratch, r.isdeleted, - r.isforeign + r.isforeign, + r.isconnector )) def findOneByKey(key: String)(implicit ctx: DBAccessContext): Fox[DataStore] = @@ -89,8 +92,8 @@ class DataStoreDAO @Inject()(sqlClient: SQLClient)(implicit ec: ExecutionContext def insertOne(d: DataStore): Fox[Unit] = for { - _ <- run(sqlu"""insert into webknossos.dataStores(name, url, key, isScratch, isDeleted, isForeign) - values(${d.name}, ${d.url}, ${d.key}, ${d.isScratch}, ${d.isDeleted}, ${d.isForeign})""") + _ <- run(sqlu"""insert into webknossos.dataStores(name, url, key, isScratch, isDeleted, isForeign, isConnector) + values(${d.name}, ${d.url}, ${d.key}, ${d.isScratch}, ${d.isDeleted}, ${d.isForeign}, ${d.isConnector})""") } yield () } diff --git a/conf/evolutions/041-add-datastore-isconnector.sql b/conf/evolutions/041-add-datastore-isconnector.sql new file mode 100644 index 00000000000..b48d0e11e3c --- /dev/null +++ b/conf/evolutions/041-add-datastore-isconnector.sql @@ -0,0 +1,13 @@ +-- https://github.com/scalableminds/webknossos/pull/TODO + +START TRANSACTION; + +DROP VIEW webknossos.dataStores_; + +ALTER TABLE webknossos.dataStores ADD COLUMN isConnector BOOLEAN NOT NULL DEFAULT false; + +CREATE VIEW webknossos.dataStores_ AS SELECT * FROM webknossos.dataStores WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation SET schemaVersion = 41; + +COMMIT TRANSACTION; diff --git a/conf/evolutions/reversions/041-add-datastore-isconnector.sql b/conf/evolutions/reversions/041-add-datastore-isconnector.sql new file mode 100644 index 00000000000..5643b072ce4 --- /dev/null +++ b/conf/evolutions/reversions/041-add-datastore-isconnector.sql @@ -0,0 +1,11 @@ +START TRANSACTION; + +DROP VIEW webknossos.dataStores_; + +ALTER TABLE webknossos.dataStores DROP COLUMN isConnector; + +CREATE VIEW webknossos.dataStores_ AS SELECT * FROM webknossos.dataStores WHERE NOT isDeleted; + +UPDATE webknossos.releaseInformation SET schemaVersion = 40; + +COMMIT TRANSACTION; diff --git a/tools/postgres/schema.sql b/tools/postgres/schema.sql index 1aa2a385efc..e0d8d2f7e5e 100644 --- a/tools/postgres/schema.sql +++ b/tools/postgres/schema.sql @@ -21,7 +21,7 @@ START TRANSACTION; CREATE TABLE webknossos.releaseInformation ( schemaVersion BIGINT NOT NULL ); -INSERT INTO webknossos.releaseInformation(schemaVersion) values(40); +INSERT INTO webknossos.releaseInformation(schemaVersion) values(41); COMMIT TRANSACTION; CREATE TABLE webknossos.analytics( @@ -139,7 +139,8 @@ CREATE TABLE webknossos.dataStores( key VARCHAR(1024) NOT NULL, isScratch BOOLEAN NOT NULL DEFAULT false, isDeleted BOOLEAN NOT NULL DEFAULT false, - isForeign BOOLEAN NOT NULL DEFAULT false + isForeign BOOLEAN NOT NULL DEFAULT false, + isConnector BOOLEAN NOT NULL DEFAULT false ); CREATE TABLE webknossos.tracingStores( From 2618efa5f9b2e11cac0edf7b5650e53249610256 Mon Sep 17 00:00:00 2001 From: Daniel Werner Date: Wed, 27 Feb 2019 20:27:11 +0100 Subject: [PATCH 03/13] add dataset tab to import wk-connect datasets, refactor add dataset state handling --- frontend/javascripts/admin/admin_rest_api.js | 18 +- frontend/javascripts/admin/api_flow_types.js | 18 +- .../admin/dataset/dataset_add_remote_view.js | 185 ------------------ .../admin/dataset/dataset_add_view.js | 112 ++++++----- .../dataset/dataset_add_wk_connect_view.js | 142 ++++++++++++++ .../admin/dataset/dataset_components.js | 95 +++++++++ .../admin/dataset/dataset_upload_view.js | 102 ++-------- frontend/javascripts/admin/onboarding.js | 18 +- .../dashboard/dataset/dataset_import_view.js | 11 +- .../dataset/simple_advanced_data_form.js | 22 +-- frontend/javascripts/messages.js | 7 +- frontend/javascripts/oxalis/store.js | 2 + .../skeletontracing_server_objects.js | 2 + .../fixtures/tasktracing_server_objects.js | 2 + .../fixtures/volumetracing_server_objects.js | 2 + 15 files changed, 402 insertions(+), 336 deletions(-) delete mode 100644 frontend/javascripts/admin/dataset/dataset_add_remote_view.js create mode 100644 frontend/javascripts/admin/dataset/dataset_add_wk_connect_view.js create mode 100644 frontend/javascripts/admin/dataset/dataset_components.js diff --git a/frontend/javascripts/admin/admin_rest_api.js b/frontend/javascripts/admin/admin_rest_api.js index a1b62903e46..39a5529bbc9 100644 --- a/frontend/javascripts/admin/admin_rest_api.js +++ b/frontend/javascripts/admin/admin_rest_api.js @@ -43,6 +43,7 @@ import { type ServerSkeletonTracing, type ServerTracing, type ServerVolumeTracing, + type wkConnectDatasetConfig, } from "admin/api_flow_types"; import type { DatasetConfiguration } from "oxalis/store"; import type { NewTask, TaskCreationResponse } from "admin/task/task_create_bulk_view"; @@ -766,8 +767,8 @@ export function getDatasetAccessList(datasetId: APIDatasetId): Promise { - await doWithToken(token => +export function addDataset(datasetConfig: DatasetConfig): Promise { + return doWithToken(token => Request.sendMultipartFormReceiveJSON(`/data/datasets?token=${token}`, { data: datasetConfig, host: datasetConfig.datastore, @@ -775,6 +776,19 @@ export async function addDataset(datasetConfig: DatasetConfig): Promise { ); } +export function addWkConnectDataset( + datastoreHost: string, + datasetConfig: wkConnectDatasetConfig, +): Promise { + return doWithToken(token => + Request.sendJSONReceiveJSON(`/data/datasets?token=${token}`, { + data: datasetConfig, + host: datastoreHost, + method: "POST", + }), + ); +} + export async function addForeignDataSet( dataStoreName: string, url: string, diff --git a/frontend/javascripts/admin/api_flow_types.js b/frontend/javascripts/admin/api_flow_types.js index fd78de79e4e..38dd03e6380 100644 --- a/frontend/javascripts/admin/api_flow_types.js +++ b/frontend/javascripts/admin/api_flow_types.js @@ -74,8 +74,9 @@ export type APIDataSource = APIDataSourceBase & { export type APIDataStore = { +name: string, +url: string, - +isForeign?: boolean, + +isForeign: boolean, +isScratch: boolean, + +isConnector: boolean, }; export type APITracingStore = { @@ -363,6 +364,21 @@ export type DatasetConfig = { +zipFile: File, }; +type wkConnectLayer = { + source: string, + type: "image" | "segmentation", +}; + +export type wkConnectDatasetConfig = { + neuroglancer: { + [string]: { + [string]: { + layers: { [string]: wkConnectLayer }, + }, + }, + }, +}; + export type APITimeTracking = { time: string, timestamp: number, diff --git a/frontend/javascripts/admin/dataset/dataset_add_remote_view.js b/frontend/javascripts/admin/dataset/dataset_add_remote_view.js deleted file mode 100644 index 4cefbadb0c7..00000000000 --- a/frontend/javascripts/admin/dataset/dataset_add_remote_view.js +++ /dev/null @@ -1,185 +0,0 @@ -// @flow -import { Form, Input, Select, Button, Card, Col, Row } from "antd"; -import { connect } from "react-redux"; -import React from "react"; - -import type { APIDataStore, APIUser } from "admin/api_flow_types"; -import type { OxalisState } from "oxalis/store"; -import { getDatastores, isDatasetNameValid } from "admin/admin_rest_api"; -import messages from "messages"; -import { trackAction } from "oxalis/model/helpers/analytics"; - -const FormItem = Form.Item; -const { Option } = Select; - -type OwnProps = {| - withoutCard?: boolean, - onAdded: (string, string) => void, -|}; -type StateProps = {| - activeUser: ?APIUser, -|}; -type Props = {| ...OwnProps, ...StateProps |}; -type PropsWithForm = {| - ...Props, - form: Object, -|}; - -type State = { - datastores: Array, -}; - -class DatasetAddRemoteView extends React.PureComponent { - state = { - datastores: [], - }; - - componentDidMount() { - this.fetchData(); - } - - async fetchData() { - const datastores = await getDatastores(); - - this.setState({ - datastores, - }); - - if (datastores.length > 0) { - this.props.form.setFieldsValue({ datastore: datastores[0].url }); - } - } - - validateUrl(url: string) { - const delimiterIndex = url.indexOf("#!"); - if (delimiterIndex < 0) { - return "The URL doesn't contain the #! delimiter. Please insert the full URL."; - } - - const jsonConfig = url.slice(delimiterIndex + 2); - try { - const config = JSON.parse(decodeURIComponent(jsonConfig)); - console.log(config.layers); - return null; - } catch (error) { - return error; - } - } - - handleSubmit() { - console.log("Submit"); - trackAction("Added remote dataset"); - } - - render() { - const { getFieldDecorator } = this.props.form; - - const Container = ({ children }) => { - if (this.props.withoutCard) { - return {children}; - } else { - return ( - Add Remote Dataset} - > - {children} - - ); - } - }; - return ( -
- -
- - - - {getFieldDecorator("name", { - rules: [ - { required: true, message: messages["dataset.import.required.name"] }, - { min: 3 }, - { pattern: /[0-9a-zA-Z_-]+$/ }, - { - validator: async (_rule, value, callback) => { - if (!this.props.activeUser) - throw new Error("Can't do operation if no user is logged in."); - const reasons = await isDatasetNameValid({ - name: value, - owningOrganization: this.props.activeUser.organization, - }); - if (reasons != null) { - callback(reasons); - } else { - callback(); - } - }, - }, - ], - validateFirst: true, - })()} - - - - - {getFieldDecorator("datastore", { - rules: [ - { required: true, message: messages["dataset.import.required.datastore"] }, - ], - })( - , - )} - - - - - {getFieldDecorator("url", { - rules: [ - { required: true, message: messages["dataset.import.required.name"] }, - { min: 3 }, - { pattern: /[0-9a-zA-Z_-]+$/ }, - { - validator: async (_rule, value, callback) => { - const reasons = this.validateUrl(value); - if (reasons != null) { - callback(reasons); - } else { - callback(); - } - }, - }, - ], - validateFirst: true, - })()} - - - - -
- -
- ); - } -} - -const mapStateToProps = (state: OxalisState): StateProps => ({ - activeUser: state.activeUser, -}); - -export default connect(mapStateToProps)( - Form.create()(DatasetAddRemoteView), -); diff --git a/frontend/javascripts/admin/dataset/dataset_add_view.js b/frontend/javascripts/admin/dataset/dataset_add_view.js index 368925e6a48..8cbc18fb202 100644 --- a/frontend/javascripts/admin/dataset/dataset_add_view.js +++ b/frontend/javascripts/admin/dataset/dataset_add_view.js @@ -1,12 +1,14 @@ // @flow import { type RouterHistory, withRouter } from "react-router-dom"; import { Tabs, Icon } from "antd"; -import React from "react"; +import React, { useState, useEffect } from "react"; import DatasetAddForeignView from "admin/dataset/dataset_add_foreign_view"; -import DatasetAddRemoteView from "admin/dataset/dataset_add_remote_view"; +import DatasetAddWkConnectView from "admin/dataset/dataset_add_wk_connect_view"; import DatasetUploadView from "admin/dataset/dataset_upload_view"; import features from "features"; +import { getDatastores } from "admin/admin_rest_api"; +import type { APIDataStore } from "admin/api_flow_types"; const { TabPane } = Tabs; @@ -14,54 +16,76 @@ type Props = { history: RouterHistory, }; -const DatasetAddView = ({ history }: Props) => ( - - - - Upload Dataset - - } - key="1" - > - { - const url = `/datasets/${organization}/${datasetName}/import`; - history.push(url); - }} - /> - - - - Add Remote Dataset - - } - key="2" - > - { - const url = `/datasets/${organization}/${datasetName}/import`; - history.push(url); - }} - /> - - {features().addForeignDataset || true ? ( +// TODO: Replace with useFetch once it is merged +function useDatastores(): { own: Array, wkConnect: Array } { + const [datastores, setDatastores] = useState({ own: [], wkConnect: [] }); + const fetchDatastores = async () => { + const fetchedDatastores = await getDatastores(); + const categorizedDatastores = { + own: fetchedDatastores.filter(ds => !ds.isForeign && !ds.isConnector), + wkConnect: fetchedDatastores.filter(ds => ds.isConnector), + }; + setDatastores(categorizedDatastores); + }; + useEffect(() => { + fetchDatastores(); + }, []); + return datastores; +} + +const DatasetAddView = ({ history }: Props) => { + const datastores = useDatastores(); + return ( + - - Add Foreign Dataset + + Upload Dataset } - key="3" + key="1" > - history.push("/dashboard")} /> + { + const url = `/datasets/${organization}/${datasetName}/import`; + history.push(url); + }} + /> - ) : null} - -); + + + Add wk-connect Dataset + + } + key="2" + > + { + const url = `/datasets/${organization}/${datasetName}/import`; + history.push(url); + }} + /> + + {features().addForeignDataset ? ( + + + Add Foreign Dataset + + } + key="3" + > + history.push("/dashboard")} /> + + ) : null} + + ); +}; export default withRouter(DatasetAddView); diff --git a/frontend/javascripts/admin/dataset/dataset_add_wk_connect_view.js b/frontend/javascripts/admin/dataset/dataset_add_wk_connect_view.js new file mode 100644 index 00000000000..dbda450d2e0 --- /dev/null +++ b/frontend/javascripts/admin/dataset/dataset_add_wk_connect_view.js @@ -0,0 +1,142 @@ +// @flow +import { Form, Input, Button, Col, Row } from "antd"; +import { connect } from "react-redux"; +import React from "react"; +import _ from "lodash"; + +import type { APIDataStore, APIUser } from "admin/api_flow_types"; +import type { OxalisState } from "oxalis/store"; +import { addWkConnectDataset } from "admin/admin_rest_api"; +import messages from "messages"; +import Toast from "libs/toast"; +import * as Utils from "libs/utils"; +import { trackAction } from "oxalis/model/helpers/analytics"; +import { + CardContainer, + DatasetNameFormItem, + DatastoreFormItem, +} from "admin/dataset/dataset_components"; + +const FormItem = Form.Item; + +type OwnProps = {| + datastores: Array, + withoutCard?: boolean, + onAdded: (string, string) => void, +|}; +type StateProps = {| + activeUser: ?APIUser, +|}; +type Props = {| ...OwnProps, ...StateProps |}; +type PropsWithForm = {| + ...Props, + form: Object, +|}; + +class DatasetAddWkConnectView extends React.PureComponent { + validateAndParseUrl(url: string) { + const delimiterIndex = url.indexOf("#!"); + if (delimiterIndex < 0) { + throw new Error("The URL doesn't contain the #! delimiter. Please insert the full URL."); + } + + const jsonConfig = url.slice(delimiterIndex + 2); + const config = JSON.parse(decodeURIComponent(jsonConfig)); + config.layers.forEach(layer => { + if (!layer.source.startsWith("precomputed://")) { + throw new Error( + "This dataset contains layers that are not supported by wk-connect. wk-connect supports only 'precomputed://' neuroglancer layers.", + ); + } + }); + return config; + } + + handleSubmit = evt => { + evt.preventDefault(); + const { activeUser } = this.props; + + this.props.form.validateFields(async (err, formValues) => { + if (!err && activeUser != null) { + const neuroglancerConfig = this.validateAndParseUrl(formValues.url); + const fullLayers = _.keyBy(neuroglancerConfig.layers, "name"); + const layers = _.mapValues(fullLayers, ({ source, type }) => ({ + type, + source: source.replace(/^(precomputed:\/\/)/, ""), + })); + + const datasetConfig = { + neuroglancer: { + [activeUser.organization]: { + [formValues.name]: { + layers, + }, + }, + }, + }; + + await addWkConnectDataset(formValues.datastore, datasetConfig); + + Toast.success(messages["dataset.add_success"]); + trackAction("Add remote dataset"); + await Utils.sleep(3000); // wait for 3 seconds so the server can catch up / do its thing + this.props.onAdded(activeUser.organization, formValues.name); + } + }); + }; + + render() { + const { form, activeUser, withoutCard, datastores } = this.props; + const { getFieldDecorator } = form; + + return ( +
+ + Currently wk-connect supports adding Neuroglancer datasets. Simply set a dataset name, + select the wk-connect datastore and paste the URL to the Neuroglancer dataset. +
+ + + + + + + + + + {getFieldDecorator("url", { + rules: [ + { required: true, message: messages["dataset.import.required.url"] }, + { + validator: async (_rule, value, callback) => { + try { + this.validateAndParseUrl(value); + callback(); + } catch (error) { + callback(error); + } + }, + }, + ], + validateFirst: true, + })()} + + + + +
+ +
+ ); + } +} + +const mapStateToProps = (state: OxalisState): StateProps => ({ + activeUser: state.activeUser, +}); + +export default connect(mapStateToProps)( + Form.create()(DatasetAddWkConnectView), +); diff --git a/frontend/javascripts/admin/dataset/dataset_components.js b/frontend/javascripts/admin/dataset/dataset_components.js new file mode 100644 index 00000000000..3f923d1a174 --- /dev/null +++ b/frontend/javascripts/admin/dataset/dataset_components.js @@ -0,0 +1,95 @@ +// @flow +import * as React from "react"; + +import { Form, Input, Select, Card } from "antd"; +import messages from "messages"; +import { isDatasetNameValid } from "admin/admin_rest_api"; +import type { APIDataStore, APIUser } from "admin/api_flow_types"; + +const FormItem = Form.Item; +const { Option } = Select; + +export function CardContainer({ + children, + withoutCard, + title, +}: { + children: React.Node, + withoutCard: ?boolean, + title: string, +}) { + if (withoutCard) { + return {children}; + } else { + return ( + {title}} + > + {children} + + ); + } +} + +export function DatasetNameFormItem({ form, activeUser }: { form: Object, activeUser: ?APIUser }) { + const { getFieldDecorator } = form; + return ( + + {getFieldDecorator("name", { + rules: [ + { required: true, message: messages["dataset.import.required.name"] }, + { min: 3 }, + { pattern: /[0-9a-zA-Z_-]+$/ }, + { + validator: async (_rule, value, callback) => { + if (!activeUser) throw new Error("Can't do operation if no user is logged in."); + const reasons = await isDatasetNameValid({ + name: value, + owningOrganization: activeUser.organization, + }); + if (reasons != null) { + callback(reasons); + } else { + callback(); + } + }, + }, + ], + validateFirst: true, + })()} + + ); +} + +export function DatastoreFormItem({ + form, + datastores, +}: { + form: Object, + datastores: Array, +}) { + const { getFieldDecorator } = form; + return ( + + {getFieldDecorator("datastore", { + rules: [{ required: true, message: messages["dataset.import.required.datastore"] }], + initialValue: datastores.length ? datastores[0].url : null, + })( + , + )} + + ); +} diff --git a/frontend/javascripts/admin/dataset/dataset_upload_view.js b/frontend/javascripts/admin/dataset/dataset_upload_view.js index b960508e7d3..116b1d877d1 100644 --- a/frontend/javascripts/admin/dataset/dataset_upload_view.js +++ b/frontend/javascripts/admin/dataset/dataset_upload_view.js @@ -1,20 +1,25 @@ // @flow -import { Form, Input, Select, Button, Card, Spin, Upload, Icon, Col, Row } from "antd"; +import { Form, Button, Spin, Upload, Icon, Col, Row } from "antd"; import { connect } from "react-redux"; import React from "react"; import type { APIDataStore, APIUser, DatasetConfig } from "admin/api_flow_types"; import type { OxalisState } from "oxalis/store"; -import { getDatastores, addDataset, isDatasetNameValid } from "admin/admin_rest_api"; +import { addDataset } from "admin/admin_rest_api"; import Toast from "libs/toast"; import * as Utils from "libs/utils"; import messages from "messages"; import { trackAction } from "oxalis/model/helpers/analytics"; +import { + CardContainer, + DatasetNameFormItem, + DatastoreFormItem, +} from "admin/dataset/dataset_components"; const FormItem = Form.Item; -const { Option } = Select; type OwnProps = {| + datastores: Array, withoutCard?: boolean, onUploaded: (string, string) => void, |}; @@ -28,32 +33,14 @@ type PropsWithForm = {| |}; type State = { - datastores: Array, isUploading: boolean, }; class DatasetUploadView extends React.PureComponent { state = { - datastores: [], isUploading: false, }; - componentDidMount() { - this.fetchData(); - } - - async fetchData() { - const datastores = await getDatastores(); - - this.setState({ - datastores, - }); - - if (datastores.length > 0) { - this.props.form.setFieldsValue({ datastore: datastores[0].url }); - } - } - normFile = e => { if (Array.isArray(e)) { return e; @@ -99,77 +86,20 @@ class DatasetUploadView extends React.PureComponent { }; render() { - const { getFieldDecorator } = this.props.form; - - const Container = ({ children }) => { - if (this.props.withoutCard) { - return {children}; - } else { - return ( - Upload Dataset} - > - {children} - - ); - } - }; + const { form, activeUser, withoutCard, datastores } = this.props; + const { getFieldDecorator } = form; + return (
- +
- - {getFieldDecorator("name", { - rules: [ - { required: true, message: messages["dataset.import.required.name"] }, - { min: 3 }, - { pattern: /[0-9a-zA-Z_-]+$/ }, - { - validator: async (_rule, value, callback) => { - if (!this.props.activeUser) - throw new Error("Can't do operation if no user is logged in."); - const reasons = await isDatasetNameValid({ - name: value, - owningOrganization: this.props.activeUser.organization, - }); - if (reasons != null) { - callback(reasons); - } else { - callback(); - } - }, - }, - ], - validateFirst: true, - })()} - + - - {getFieldDecorator("datastore", { - rules: [ - { required: true, message: messages["dataset.import.required.datastore"] }, - ], - })( - , - )} - + @@ -181,7 +111,7 @@ class DatasetUploadView extends React.PureComponent { { - this.props.form.setFieldsValue({ zipFile: [file] }); + form.setFieldsValue({ zipFile: [file] }); return false; }} > @@ -201,7 +131,7 @@ class DatasetUploadView extends React.PureComponent {
- +
); diff --git a/frontend/javascripts/admin/onboarding.js b/frontend/javascripts/admin/onboarding.js index e1d9b72e216..0a4ebc83288 100644 --- a/frontend/javascripts/admin/onboarding.js +++ b/frontend/javascripts/admin/onboarding.js @@ -18,14 +18,14 @@ import Clipboard from "clipboard-js"; import React, { type Node, useState, useEffect } from "react"; import { Link } from "react-router-dom"; -import type { APIUser } from "admin/api_flow_types"; +import type { APIUser, APIDataStore } from "admin/api_flow_types"; import type { OxalisState } from "oxalis/store"; import { location } from "libs/window"; import DatasetImportView from "dashboard/dataset/dataset_import_view"; import DatasetUploadView from "admin/dataset/dataset_upload_view"; import RegistrationForm from "admin/auth/registration_form"; import Toast from "libs/toast"; -import { getOrganizations } from "admin/admin_rest_api"; +import { getOrganizations, getDatastores } from "admin/admin_rest_api"; const { Step } = Steps; const FormItem = Form.Item; @@ -37,6 +37,7 @@ type Props = StateProps; type State = { currentStep: number, + datastores: Array, organizationName: string, datasetNameToImport: ?string, }; @@ -210,11 +211,21 @@ const OrganizationForm = Form.create()(({ form, onComplete }) => { class OnboardingView extends React.PureComponent { state = { - currentStep: 0, + currentStep: 2, + datastores: [], organizationName: "", datasetNameToImport: null, }; + componentDidMount() { + this.fetchData(); + } + + async fetchData() { + const datastores = (await getDatastores()).filter(ds => !ds.isForeign && !ds.isConnector); + this.setState({ datastores }); + } + advanceStep = () => { this.setState(prevState => ({ currentStep: prevState.currentStep + 1, @@ -290,6 +301,7 @@ class OnboardingView extends React.PureComponent { > {this.state.datasetNameToImport == null ? ( { this.setState({ datasetNameToImport: datasetName }); }} diff --git a/frontend/javascripts/dashboard/dataset/dataset_import_view.js b/frontend/javascripts/dashboard/dataset/dataset_import_view.js index b12262d2068..a9f9d6a32b0 100644 --- a/frontend/javascripts/dashboard/dataset/dataset_import_view.js +++ b/frontend/javascripts/dashboard/dataset/dataset_import_view.js @@ -235,7 +235,11 @@ class DatasetImportView extends React.PureComponent { await updateDatasetTeams(dataset, teamIds); const dataSource = JSON.parse(formValues.dataSourceJson); - if (this.state.dataset != null && !this.state.dataset.isForeign) { + if ( + this.state.dataset != null && + !this.state.dataset.isForeign && + !this.state.dataset.dataStore.isConnector + ) { await updateDatasetDatasource(this.props.datasetId.name, dataset.dataStore.url, dataSource); } @@ -389,7 +393,10 @@ class DatasetImportView extends React.PureComponent {