Test/aggmds counts

src/Discovery/DiscoveryActionBar.tsx

@@ -21,6 +21,7 @@ import {
 import FileSaver from 'file-saver';
 import { DiscoveryConfig } from './DiscoveryConfig';
 import { fetchWithCreds } from '../actions';
+import { loadManifestFromResources } from './aggMDSUtils';
 import {
   manifestServiceApiPath, hostname, jobAPIPath, externalLoginOptionsUrl,
 } from '../localconf';
@@ -256,15 +257,18 @@ const handleDownloadZipClick = async (
   history,
   location,
 ) => {
+  let selectedResourcesWithManifest = selectedResources;
   if (config.features.exportToWorkspace.verifyExternalLogins) {
     const { manifestFieldName } = config.features.exportToWorkspace;
+    selectedResourcesWithManifest = await loadManifestFromResources(selectedResources, manifestFieldName);
+
     const isLinked = await checkFederatedLoginStatus(setDownloadStatus, selectedResources, manifestFieldName, history, location);
     if (!isLinked) {
       return;
     }
   }
 
-  const studyIDs = selectedResources.map((study) => study[config.minimalFieldMapping.uid]);
+  const studyIDs = selectedResourcesWithManifest.map((study) => study[config.minimalFieldMapping.uid]);
   fetchWithCreds({
     path: `${jobAPIPath}dispatch`,
     method: 'POST',
@@ -292,20 +296,22 @@ const handleDownloadZipClick = async (
             active: true,
           },
         });
-        setTimeout(checkDownloadStatus, JOB_POLLING_INTERVAL, uid, downloadStatus, setDownloadStatus, selectedResources);
+        setTimeout(checkDownloadStatus, JOB_POLLING_INTERVAL, uid, downloadStatus, setDownloadStatus, selectedResourcesWithManifest);
       }
     },
   ).catch(() => setDownloadStatus(DOWNLOAD_FAIL_STATUS));
 };
 
-const handleDownloadManifestClick = (config: DiscoveryConfig, selectedResources: any[]) => {
+const handleDownloadManifestClick = async (config: DiscoveryConfig, selectedResources: any[]) => {
   const { manifestFieldName } = config.features.exportToWorkspace;
   if (!manifestFieldName) {
     throw new Error('Missing required configuration field `config.features.exportToWorkspace.manifestFieldName`');
   }
   // combine manifests from all selected studies
-  const manifest:any = [];
-  selectedResources.forEach((study) => {
+  const manifest = [];
+  // update the manifest to handle aggregated manifestFieldName
+  const selectedResourcesWithManifest = await loadManifestFromResources(selectedResources, manifestFieldName);
+  selectedResourcesWithManifest.forEach((study) => {
     if (study[manifestFieldName]) {
       if ('commons_url' in study && !(hostname.includes(study.commons_url))) { // PlanX addition to allow hostname based DRS in manifest download clients
         // like FUSE
@@ -319,9 +325,9 @@ const handleDownloadManifestClick = (config: DiscoveryConfig, selectedResources:
       }
     }
   });
-  const projectNumber = selectedResources.map((study) => study.project_number);
-  const studyName = selectedResources.map((study) => study.study_name);
-  const repositoryName = selectedResources.map((study) => study.commons);
+  const projectNumber = selectedResourcesWithManifest.map((study) => study.project_number);
+  const studyName = selectedResourcesWithManifest.map((study) => study.study_name);
+  const repositoryName = selectedResourcesWithManifest.map((study) => study.commons);
   datadogRum.addAction('manifestDownload', {
     manifestDownloadProjectNumber: projectNumber,
     manifestDownloadStudyName: studyName,
@@ -346,9 +352,11 @@ const handleExportToWorkspaceClick = async (
     throw new Error('Missing required configuration field `config.features.exportToWorkspace.manifestFieldName`');
   }
 
+  const selectedResourcesWithManifest = await loadManifestFromResources(selectedResources, manifestFieldName);
+
   if (config.features.exportToWorkspace.verifyExternalLogins) {
     const isLinked = await checkFederatedLoginStatus(
-      setDownloadStatus, selectedResources, manifestFieldName, history, location,
+      setDownloadStatus, selectedResourcesWithManifest, manifestFieldName, history, location,
     );
     if (!isLinked) {
       return;
@@ -357,8 +365,8 @@ const handleExportToWorkspaceClick = async (
 
   setExportingToWorkspace(true);
   // combine manifests from all selected studies
-  const manifest:any = [];
-  selectedResources.forEach((study) => {
+  const manifest = [];
+  selectedResourcesWithManifest.forEach((study) => {
     if (study[manifestFieldName]) {
       if ('commons_url' in study && !(hostname.includes(study.commons_url))) { // PlanX addition to allow hostname based DRS in manifest download clients
         // like FUSE
@@ -373,9 +381,9 @@ const handleExportToWorkspaceClick = async (
     }
   });
 
-  const projectNumber = selectedResources.map((study) => study.project_number);
-  const studyName = selectedResources.map((study) => study.study_name);
-  const repositoryName = selectedResources.map((study) => study.commons);
+  const projectNumber = selectedResourcesWithManifest.map((study) => study.project_number);
+  const studyName = selectedResourcesWithManifest.map((study) => study.study_name);
+  const repositoryName = selectedResourcesWithManifest.map((study) => study.commons);
   datadogRum.addAction('exportToWorkspace', {
     exportToWorkspaceProjectNumber: projectNumber,
     exportToWorkspaceStudyName: studyName,

src/Discovery/MDSUtils.jsx

@@ -3,7 +3,7 @@ import { mdsURL, studyRegistrationConfig } from '../localconf';
 const LIMIT = 1000; // required or else mds defaults to returning 10 records
 const STUDY_DATA_FIELD = 'gen3_discovery'; // field in the MDS response that contains the study data
 
-const loadStudiesFromMDS = async (guidType = 'discovery_metadata') => {
+const loadStudiesFromMDS = async (_config, guidType = 'discovery_metadata') => {
   try {
     let allStudies = [];
     let offset = 0;

src/Discovery/aggMDSUtils.jsx

@@ -1,5 +1,7 @@
 import { discoveryConfig, aggMDSDataURL, studyRegistrationConfig } from '../localconf';
 
+const STUDY_DATA_FIELD = 'gen3_discovery'; // field in the aggMDS response that contains the study data
+
 /**
  * getUniqueTags returns a reduced subset of unique tags for the given tags.
  *
@@ -9,8 +11,8 @@ import { discoveryConfig, aggMDSDataURL, studyRegistrationConfig } from '../loca
 const getUniqueTags = ((tags) => tags.filter((v, i, a) => a.findIndex((t) => (
   t.name?.length > 0 && t.category === v.category && t.name === v.name)) === i));
 
-const loadStudiesFromAggMDSRequests = async (offset, limit) => {
-  const url = `${aggMDSDataURL}?data=True&limit=${limit}&offset=${offset}`;
+const loadStudiesFromAggMDSRequests = async (offset, limit, manifestFieldName) => {
+  const url = `${aggMDSDataURL}?data=True&limit=${limit}&offset=${offset}&counts=${manifestFieldName}`;
 
   const res = await fetch(url);
   if (res.status !== 200) {
@@ -25,13 +27,14 @@ const loadStudiesFromAggMDSRequests = async (offset, limit) => {
   let allStudies = [];
 
   commons.forEach((commonsName) => {
+
     const studies = metadataResponse[commonsName];
 
     const editedStudies = studies.map((entry, index) => {
       const keys = Object.keys(entry);
       const studyId = keys[0];
 
-      const entryUnpacked = entry[studyId].gen3_discovery;
+      const entryUnpacked = entry[studyId][STUDY_DATA_FIELD];
       entryUnpacked.study_id = studyId;
       entryUnpacked.commons = commonsName;
       entryUnpacked.frontend_uid = `${commonsName}_${index}`;
@@ -68,15 +71,40 @@ const loadStudiesFromAggMDSRequests = async (offset, limit) => {
   return allStudies;
 };
 
-const loadStudiesFromAggMDS = async () => {
+const loadStudiesFromAggMDS = async (config) => {
   // Retrieve from aggregate MDS
 
   const offset = 0; // For pagination
   const limit = 1000; // Total number of rows requested
+  const manifestFieldName = config.features?.exportToWorkspace?.manifestFieldName ?
+    config.features.exportToWorkspace.manifestFieldName : '__manifest';
 
-  const studies = await loadStudiesFromAggMDSRequests(offset, limit);
+  const studies = await loadStudiesFromAggMDSRequests(offset, limit, manifestFieldName);
 
   return studies;
 };
 
+/**
+ * Will try to get the study manifest data from the aggregateMDS and returns
+ * the study with the newly populated manifest field
+ * @param study - study to get a manifest for
+ * @param manifestFieldName - name of the manifest member (default __manifest)
+ * @returns {Promise<*>} - the updated study data
+ */
+const loadStudyFromMDS = async (study, manifestFieldName) => {
+  if (study[manifestFieldName] === 0) return { study, [manifestFieldName]: [] };
+
+  const url = `${aggMDSDataURL}/guid/${study.study_id}`;
+  const resp = await fetch(url);
+  if (resp.status !== 200) {
+    throw new Error(`Request for study info at ${url} failed. Response: ${JSON.stringify(resp, null, 2)}`);
+  }
+  const studyWithManifest = await resp.json();
+
+  return { ...study, [manifestFieldName]: studyWithManifest[STUDY_DATA_FIELD][manifestFieldName] };
+};
+
+// eslint-disable-next-line max-len
+export const loadManifestFromResources = async (selectedResources, manifestFieldName) => Promise.all(selectedResources.map((x) => loadStudyFromMDS(x, manifestFieldName)));
+
 export default loadStudiesFromAggMDS;

src/Discovery/index.tsx

@@ -81,10 +81,10 @@ const DiscoveryWithMDSBackend: React.FC<{
       } else {
         loadStudiesFunction = loadStudiesFromMDS;
       }
-      const rawStudiesRegistered = await loadStudiesFunction();
+      const rawStudiesRegistered = await loadStudiesFunction(props.config);
       let rawStudiesUnregistered = [];
       if (isEnabled('studyRegistration')) {
-        rawStudiesUnregistered = await loadStudiesFromMDS('unregistered_discovery_metadata');
+        rawStudiesUnregistered = await loadStudiesFromMDS(props.config, 'unregistered_discovery_metadata');
         rawStudiesUnregistered = rawStudiesUnregistered
           .map((unregisteredStudy) => ({ ...unregisteredStudy, [studyRegistrationValidationField]: false }));
       }