Feat/discovery load time spike

src/Discovery/Discovery.tsx

@@ -4,7 +4,7 @@ import React, {
 import * as JsSearch from 'js-search';
 import jsonpath from 'jsonpath';
 import {
-  Tag, Popover, Space, Collapse, Button, Dropdown, Pagination, Tooltip,
+  Tag, Popover, Space, Collapse, Button, Dropdown, Pagination, Tooltip, Spin,
 } from 'antd';
 import {
   LockOutlined,
@@ -215,6 +215,7 @@ export interface Props {
   onAccessSortDirectionSet: (accessSortDirection: AccessSortDirection) => any,
   onResourcesSelected: (resources: DiscoveryResource[]) => any,
   onPaginationSet: (pagination: { currentPage: number, resultsPerPage: number }) => any,
+  batchLoadingInfo: {allBatchesAreLoaded: boolean},
 }
 
 const Discovery: React.FunctionComponent<Props> = (props: Props) => {
@@ -237,6 +238,13 @@ const Discovery: React.FunctionComponent<Props> = (props: Props) => {
   );
   const [visibleResources, setVisibleResources] = useState([]);
 
+  const batchesAreLoading = props.batchLoadingInfo.allBatchesAreLoaded === false;
+  const BatchLoadingSpinner = () => (
+    <div style={{ textAlign: 'center' }}>
+      <Spin />
+    </div>
+  );
+
   const handleSearchChange = (ev) => {
     const { value } = ev.currentTarget;
     props.onSearchChange(value);
@@ -656,6 +664,7 @@ const Discovery: React.FunctionComponent<Props> = (props: Props) => {
       {/* Header with stats */}
       <div className='discovery-header'>
         <DiscoverySummary
+          batchLoadingInfo={props.batchLoadingInfo}
           visibleResources={visibleResources}
           config={config}
         />
@@ -696,12 +705,18 @@ const Discovery: React.FunctionComponent<Props> = (props: Props) => {
               <Collapse activeKey={(searchableTagCollapsed) ? '' : '1'} ghost>
                 <Panel className='discovery-header__dropdown-tags-display-panel' header='' key='1'>
                   <div className='discovery-header__dropdown-tags'>
-                    <DiscoveryDropdownTagViewer
-                      config={config}
-                      studies={props.studies}
-                      selectedTags={props.selectedTags}
-                      setSelectedTags={props.onTagsSelected}
-                    />
+                    { batchesAreLoading
+                      ? (
+                        <BatchLoadingSpinner />
+                      )
+                      : (
+                        <DiscoveryDropdownTagViewer
+                          config={config}
+                          studies={props.studies}
+                          selectedTags={props.selectedTags}
+                          setSelectedTags={props.onTagsSelected}
+                        />
+                      )}
                   </div>
                 </Panel>
               </Collapse>
@@ -751,17 +766,20 @@ const Discovery: React.FunctionComponent<Props> = (props: Props) => {
               <div
                 className='discovery-filters--visible'
               >
-                <DiscoveryAdvancedSearchPanel
-                  config={props.config}
-                  studies={props.studies}
-                  filterState={filterState}
-                  setFilterState={(event) => {
-                    props.onAdvancedSearch(event);
-                    setFilterState(event);
-                  }}
-                  filterMultiSelectionLogic={filterMultiSelectionLogic}
-                  setFilterMultiSelectionLogic={setFilterMultiSelectionLogic}
-                />
+                {batchesAreLoading ? <BatchLoadingSpinner />
+                  : (
+                    <DiscoveryAdvancedSearchPanel
+                      config={props.config}
+                      studies={props.studies}
+                      filterState={filterState}
+                      setFilterState={(event) => {
+                        props.onAdvancedSearch(event);
+                        setFilterState(event);
+                      }}
+                      filterMultiSelectionLogic={filterMultiSelectionLogic}
+                      setFilterMultiSelectionLogic={setFilterMultiSelectionLogic}
+                    />
+                  )}
               </div>
             ) : (<div className='discovery-filters--hide' />)}
 

src/Discovery/DiscoveryLoadingProgressMini.tsx

@@ -0,0 +1,69 @@
+import React, { useState, useEffect } from 'react';
+import { Progress } from 'antd';
+
+interface DiscoveryLoadingProps {
+ batchLoadingInfo: { allBatchesAreLoaded: boolean }
+}
+
+// this should probably be done in a CSS for production:
+const style = document.createElement('style');
+style.type = 'text/css';
+const css = '.discovery-header__dropdown-tags-container {margin-top: 15px;} .discovery-header{align-items:start;} ';
+style.innerHTML = css;
+document.head.appendChild(style);
+
+const DiscoveryLoadingProgressMini = ({
+  batchLoadingInfo,
+}: DiscoveryLoadingProps) => {
+  const [percent, setPercent] = useState(0);
+  const { allBatchesAreLoaded } = batchLoadingInfo;
+  const [displayProgressBar, setDisplayProgressBar] = useState(true);
+
+  // Fake loading UI
+  const percentUpdateInterval = 500;
+  const percentIncrementAmount = 5;
+  useEffect(() => {
+    const interval = setInterval(() => {
+      setPercent((prevPercent) => prevPercent + percentIncrementAmount);
+    }, percentUpdateInterval);
+    return () => clearInterval(interval);
+  }, [percent, allBatchesAreLoaded]);
+
+  // hide the bar with a transition delay after the batches are loaded,
+  // giving the browser some time to process the batch
+  const delayTimeBeforeHidingProgressBar = 2000;
+  useEffect(() => {
+    if (allBatchesAreLoaded) {
+      // Change displayProgressBar to false after delay
+      setTimeout(() => {
+        setDisplayProgressBar(false);
+      }, delayTimeBeforeHidingProgressBar);
+    }
+  }, [allBatchesAreLoaded]);
+
+  const progressContainerStyle = {
+    textAlign: 'center',
+    marginBottom: '5px',
+    display: displayProgressBar ? 'block' : 'none',
+  };
+
+  return (
+    <div style={progressContainerStyle}>
+      <Progress
+        width={80}
+        showInfo={false}
+        percent={allBatchesAreLoaded ? 100 : percent}
+        status='success'
+        strokeColor='#99286B'
+      />
+      <p
+        style={{ lineHeight: 'normal', textTransform: 'inherit' }}
+        className='discovery-header__stat-label'
+      >
+        Loading studies...
+      </p>
+    </div>
+  );
+};
+
+export default DiscoveryLoadingProgressMini;

src/Discovery/DiscoverySummary.tsx

@@ -3,6 +3,7 @@ import uniq from 'lodash/uniq';
 import sum from 'lodash/sum';
 import jsonpath from 'jsonpath';
 import { DiscoveryConfig } from './DiscoveryConfig';
+import DiscoveryLoadingProgressMini from './DiscoveryLoadingProgressMini';
 
 /**
  * Check for non-numeric items in an array and convert them to numbers.
@@ -55,6 +56,7 @@ const renderAggregation = (aggregation: AggregationConfig, studies: any[] | null
 interface Props {
   visibleResources: any[] | null;
   config: DiscoveryConfig;
+  batchLoadingInfo: { allBatchesAreLoaded: boolean },
 }
 
 const DiscoverySummary = (props: Props) => (
@@ -72,6 +74,10 @@ const DiscoverySummary = (props: Props) => (
                 {aggregation.name}
               </div>
             </div>
+            <DiscoveryLoadingProgressMini
+              batchLoadingInfo={props.batchLoadingInfo}
+            />
+
           </div>
         </React.Fragment>
       ))

src/Discovery/MDSUtils.jsx

@@ -3,7 +3,7 @@ import { mdsURL, studyRegistrationConfig } from '../localconf';
 const LIMIT = 2000; // required or else mds defaults to returning 10 records
 const STUDY_DATA_FIELD = 'gen3_discovery'; // field in the MDS response that contains the study data
 
-const loadStudiesFromMDS = async (guidType = 'discovery_metadata') => {
+export const loadStudiesFromMDS = async (guidType = 'discovery_metadata') => {
   try {
     let allStudies = [];
     let offset = 0;
@@ -43,4 +43,28 @@ const loadStudiesFromMDS = async (guidType = 'discovery_metadata') => {
   }
 };
 
-export default loadStudiesFromMDS;
+export const getSomeStudiesFromMDS = async (guidType = 'discovery_metadata', numberOfStudies = 10) => {
+  try {
+    let allStudies = [];
+    // request up to LIMIT studies from MDS at a time.
+      const url = `${mdsURL}?data=True&_guid_type=${guidType}&limit=${numberOfStudies}`;
+      const res = await fetch(url);
+      if (res.status !== 200) {
+        throw new Error(`Request for study data at ${url} failed. Response: ${JSON.stringify(res, null, 2)}`);
+      }
+      // eslint-disable-next-line no-await-in-loop
+      const jsonResponse = await res.json();
+      const studies = Object.values(jsonResponse).map((entry) => {
+        const study = { ...entry[STUDY_DATA_FIELD] };
+        // copy VLMD info if exists
+        if (studyRegistrationConfig?.dataDictionaryField && entry[studyRegistrationConfig.dataDictionaryField]) {
+          study[studyRegistrationConfig.dataDictionaryField] = entry[studyRegistrationConfig.dataDictionaryField];
+        }
+        return study;
+      });
+      allStudies = allStudies.concat(studies);
+      return allStudies;
+  } catch (err) {
+    throw new Error(`Request for study data failed: ${err}`);
+  }
+};

src/Discovery/aggMDSUtils.jsx

@@ -10,6 +10,7 @@ const getUniqueTags = ((tags) => tags.filter((v, i, a) => a.findIndex((t) => (
   t.name?.length > 0 && t.category === v.category && t.name === v.name)) === i));
 
 const loadStudiesFromAggMDSRequests = async (offset, limit) => {
+  var startTimeloadStudiesFromAggMDSRequests= performance.now();
   const url = `${aggMDSDataURL}?data=True&limit=${limit}&offset=${offset}`;
 
   const res = await fetch(url);
@@ -64,18 +65,21 @@ const loadStudiesFromAggMDSRequests = async (offset, limit) => {
 
     allStudies = allStudies.concat(editedStudies);
   });
-
+  var endTimeloadStudiesFromAggMDSRequests = performance.now();
+  console.log(
+    `Call to loadStudiesFromAggMDSRequests took ${endTimeloadStudiesFromAggMDSRequests - startTimeloadStudiesFromAggMDSRequests} milliseconds`
+  );
   return allStudies;
 };
 
-const loadStudiesFromAggMDS = async () => {
+const loadStudiesFromAggMDS = async (limit=2000) => {
   // Retrieve from aggregate MDS
 
   const offset = 0; // For pagination
-  const limit = 2000; // Total number of rows requested
-
+  // const limit = 2000; // Total number of rows requested
+  // const limit = 5;
+  var startTimeloadStudiesFromAggMDSRequests= performance.now();
   const studies = await loadStudiesFromAggMDSRequests(offset, limit);
-
   return studies;
 };
 

src/Discovery/index.tsx

@@ -1,7 +1,6 @@
 import React, { useState, useEffect } from 'react';
 import { connect } from 'react-redux';
 import _ from 'lodash';
-
 import Discovery, { AccessLevel, AccessSortDirection, DiscoveryResource } from './Discovery';
 import { DiscoveryConfig } from './DiscoveryConfig';
 import { userHasMethodForServiceOnResource } from '../authMappingUtils';
@@ -10,7 +9,7 @@ import {
 } from '../localconf';
 import isEnabled from '../helpers/featureFlags';
 import loadStudiesFromAggMDS from './aggMDSUtils';
-import loadStudiesFromMDS from './MDSUtils';
+import { loadStudiesFromMDS, getSomeStudiesFromMDS } from './MDSUtils';
 
 const populateStudiesWithConfigInfo = (studies, config) => {
   if (!config.studies) {
@@ -73,22 +72,58 @@ const DiscoveryWithMDSBackend: React.FC<{
     throw new Error('Could not find configuration for Discovery page. Check the portal config.');
   }
 
+  // Downloads and processes studies in two seperate batches
+  // to improve load time & usability
+  // Initially uses a smaller batch to load interface quickly
+  // Then a batch with all the studies
+  const [numberOfBatchesLoaded, setNumberOfBatchesLoaded] = useState(0);
+  const expectedNumberOfTotalBatches = 2;
+  const numberOfStudiesForSmallerBatch = 5;
+  const numberOfStudiesForAllStudiesBatch = 2000;
+
   useEffect(() => {
+    // If batch loading is Enabled, update the numberOfBatchesLoaded to enable calling of different batch sizes
+    // with different parameters
+    if (numberOfBatchesLoaded < expectedNumberOfTotalBatches) setNumberOfBatchesLoaded(numberOfBatchesLoaded + 1);
+
     const studyRegistrationValidationField = studyRegistrationConfig?.studyRegistrationValidationField;
     async function fetchRawStudies() {
-      let loadStudiesFunction;
+      const startTime = performance.now();
+      let loadStudiesFunction: Function;
+      let loadStudiesParameters: any;
       if (isEnabled('discoveryUseAggMDS')) {
         loadStudiesFunction = loadStudiesFromAggMDS;
+        loadStudiesParameters = numberOfBatchesLoaded === 1
+          ? numberOfStudiesForSmallerBatch
+          : numberOfStudiesForAllStudiesBatch;
       } else {
-        loadStudiesFunction = loadStudiesFromMDS;
+        loadStudiesFunction = getSomeStudiesFromMDS;
+        loadStudiesParameters = props.config?.features?.guidType;
       }
-      const rawStudiesRegistered = await loadStudiesFunction(props.config?.features?.guidType);
-      let rawStudiesUnregistered = [];
+      const rawStudiesRegistered = await loadStudiesFunction(
+        loadStudiesParameters,
+      );
+      let rawStudiesUnregistered: any[] = [];
       if (isEnabled('studyRegistration')) {
-        rawStudiesUnregistered = await loadStudiesFromMDS('unregistered_discovery_metadata');
-        rawStudiesUnregistered = rawStudiesUnregistered
-          .map((unregisteredStudy) => ({ ...unregisteredStudy, [studyRegistrationValidationField]: false }));
+        // Load fewer raw studies if on the first studies batch
+        // Otherwise load them all
+        rawStudiesUnregistered = numberOfBatchesLoaded === 1
+          ? (rawStudiesUnregistered = await getSomeStudiesFromMDS(
+            'unregistered_discovery_metadata',
+            numberOfStudiesForSmallerBatch,
+          ))
+          : await loadStudiesFromMDS('unregistered_discovery_metadata');
+        rawStudiesUnregistered = rawStudiesUnregistered.map(
+          (unregisteredStudy) => ({
+            ...unregisteredStudy,
+            [studyRegistrationValidationField]: false,
+          }),
+        );
       }
+      const endTime = performance.now();
+      console.log(
+        `Call to fetchRawStudies took ${endTime - startTime} milliseconds`,
+      );
       return _.union(rawStudiesRegistered, rawStudiesUnregistered);
     }
     fetchRawStudies().then((rawStudies) => {
@@ -167,16 +202,27 @@ const DiscoveryWithMDSBackend: React.FC<{
 
     // indicate discovery tag is active even if we didn't click a button to get here
     props.onDiscoveryPageActive();
-  }, []);
+  }, [props, numberOfBatchesLoaded]);
 
   let studyRegistrationValidationField = studyRegistrationConfig?.studyRegistrationValidationField;
   if (!isEnabled('studyRegistration')) {
     studyRegistrationValidationField = undefined;
   }
+
+  const batchLoadingInfo = {
+    // All batches all loaded if the studies are not null and
+    // their length is great than the studies for the smaller batches
+    // from loadStudiesFromAggMDS and getSomeStudiesFromMDS
+    allBatchesAreLoaded: studies === null
+      ? false
+      : studies?.length > numberOfStudiesForSmallerBatch * 2,
+  };
+
   return (
     <Discovery
       studies={studies === null ? [] : studies}
       studyRegistrationValidationField={studyRegistrationValidationField}
+      batchLoadingInfo={batchLoadingInfo}
       {...props}
     />
   );