Features/last_ingestion

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -22,6 +23,7 @@ const ConfigLogTemplate = `Config --
 	Static Files: %s
 	Client Name: %s
 	Katsu URL: %v
+	WES URL: %v
 	Bento Portal Url: %s
 	Port: %d
 	Translated: %t
@@ -30,16 +32,17 @@ const ConfigLogTemplate = `Config --
 `
 
 type BentoConfig struct {
-	ServiceId          string `envconfig:"BENTO_PUBLIC_SERVICE_ID"`
-	PackageJsonPath    string `envconfig:"BENTO_PUBLIC_PACKAGE_JSON_PATH" default:"./package.json"`
-	StaticFilesPath    string `envconfig:"BENTO_PUBLIC_STATIC_FILES_PATH" default:"./www"`
-	ClientName         string `envconfig:"BENTO_PUBLIC_CLIENT_NAME"`
-	KatsuUrl           string `envconfig:"BENTO_PUBLIC_KATSU_URL"`
-	BentoPortalUrl     string `envconfig:"BENTO_PUBLIC_PORTAL_URL"`
-	Port               int    `envconfig:"INTERNAL_PORT" default:"8090"`
-	Translated         bool   `envconfig:"BENTO_PUBLIC_TRANSLATED" default:"true"`
-	BeaconUrl          string `envconfig:"BEACON_URL"`
-	BeaconUiEnabled	   bool   `envconfig:"BENTO_BEACON_UI_ENABLED"`
+	ServiceId       string `envconfig:"BENTO_PUBLIC_SERVICE_ID"`
+	PackageJsonPath string `envconfig:"BENTO_PUBLIC_PACKAGE_JSON_PATH" default:"./package.json"`
+	StaticFilesPath string `envconfig:"BENTO_PUBLIC_STATIC_FILES_PATH" default:"./www"`
+	ClientName      string `envconfig:"BENTO_PUBLIC_CLIENT_NAME"`
+	KatsuUrl        string `envconfig:"BENTO_PUBLIC_KATSU_URL"`
+	WesUrl          string `envconfig:"BENTO_PUBLIC_WES_URL"`
+	BentoPortalUrl  string `envconfig:"BENTO_PUBLIC_PORTAL_URL"`
+	Port            int    `envconfig:"INTERNAL_PORT" default:"8090"`
+	Translated      bool   `envconfig:"BENTO_PUBLIC_TRANSLATED" default:"true"`
+	BeaconUrl       string `envconfig:"BEACON_URL"`
+	BeaconUiEnabled bool   `envconfig:"BENTO_BEACON_UI_ENABLED"`
 }
 
 type JsonLike map[string]interface{}
@@ -49,8 +52,33 @@ func internalServerError(err error, c echo.Context) error {
 	return c.JSON(http.StatusInternalServerError, ErrorResponse{Message: err.Error()})
 }
 
-func identityJSONTransform(j JsonLike) JsonLike {
-	return j
+// This function assumes JSON to be an object or an array of objects.
+func jsonDeserialize(body []byte) (interface{}, error) {
+	var i interface{}
+	err := json.Unmarshal(body, &i)
+	if err != nil {
+		return nil, err
+	}
+
+	switch v := i.(type) {
+	case []interface{}:
+		// JSON array
+		jsonArray := make([]JsonLike, len(v))
+		for i, item := range v {
+			m, ok := item.(map[string]interface{})
+			if !ok {
+				// Error occurs if array elements are not JSON objects
+				return nil, errors.New("invalid JSON array element")
+			}
+			jsonArray[i] = JsonLike(m)
+		}
+		return jsonArray, nil
+	case map[string]interface{}:
+		// JSON object
+		return JsonLike(v), nil
+	default:
+		return nil, errors.New("invalid JSON: not an object or an array")
+	}
 }
 
 func main() {
@@ -86,6 +114,7 @@ func main() {
 		cfg.StaticFilesPath,
 		cfg.ClientName,
 		cfg.KatsuUrl,
+		cfg.WesUrl,
 		cfg.BentoPortalUrl,
 		cfg.Port,
 		cfg.Translated,
@@ -96,20 +125,19 @@ func main() {
 	// Set up HTTP client
 	client := &http.Client{}
 
-	// Create Katsu request helper closure
-	type responseFormatter func(JsonLike) JsonLike
-	katsuRequestJsonOnly := func(path string, qs url.Values, c echo.Context, rf responseFormatter) (JsonLike, error) {
+	// Create request helper closure
+	type responseFormatterFunc func([]byte) (interface{}, error)
+	genericRequestJsonOnly := func(url string, qs url.Values, c echo.Context, rf responseFormatterFunc) (interface{}, error) {
 		var req *http.Request
 		var err error
 
 		if qs != nil {
-			req, err = http.NewRequest(
-				"GET", fmt.Sprintf("%s%s?%s", cfg.KatsuUrl, path, qs.Encode()), nil)
+			req, err = http.NewRequest("GET", fmt.Sprintf("%s?%s", url, qs.Encode()), nil)
 			if err != nil {
 				return nil, internalServerError(err, c)
 			}
 		} else {
-			req, err = http.NewRequest("GET", fmt.Sprintf("%s%s", cfg.KatsuUrl, path), nil)
+			req, err = http.NewRequest("GET", url, nil)
 			if err != nil {
 				return nil, internalServerError(err, c)
 			}
@@ -125,44 +153,66 @@ func main() {
 
 		defer resp.Body.Close()
 
-		// Read response body and convert to a generic JSON-like data structure
-
+		// Read response body
 		body, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return nil, internalServerError(err, c)
 		}
 
-		jsonLike := make(JsonLike)
-		err = json.Unmarshal(body, &jsonLike)
+		// Apply the response formatter function to convert the body to the desired format
+		result, err := rf(body)
 		if err != nil {
 			return nil, internalServerError(err, c)
 		}
 
-		return jsonLike, nil
+		return result, nil
 	}
 
-	katsuRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatter) error {
-		jsonLike, err := katsuRequestJsonOnly(path, qs, c, rf)
-
+	katsuRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error {
+		result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.KatsuUrl, path), qs, c, rf)
 		if err != nil {
 			return err
 		}
+		return c.JSON(http.StatusOK, result)
+	}
 
-		return c.JSON(http.StatusOK, rf(jsonLike))
+	wesRequest := func(path string, qs url.Values, c echo.Context, rf responseFormatterFunc) error {
+		result, err := genericRequestJsonOnly(fmt.Sprintf("%s%s", cfg.WesUrl, path), qs, c, rf)
+		if err != nil {
+			return err
+		}
+		return c.JSON(http.StatusOK, result)
 	}
 
 	katsuRequestBasic := func(path string, c echo.Context) error {
-		return katsuRequest(path, nil, c, identityJSONTransform)
+		return katsuRequest(path, nil, c, jsonDeserialize)
+	}
+
+	wesRequestWithDetailsAndPublic := func(c echo.Context) error {
+		qs := url.Values{}
+		qs.Add("with_details", "true")
+		qs.Add("public", "true")
+		return wesRequest("/runs", qs, c, jsonDeserialize)
 	}
 
 	fetchAndSetKatsuPublic := func(c echo.Context, katsuCache *cache.Cache) (JsonLike, error) {
 		fmt.Println("'publicOverview' not found or expired in 'katsuCache' - fetching")
-		publicOverview, err := katsuRequestJsonOnly("/api/public_overview", nil, c, identityJSONTransform)
+		publicOverviewInterface, err := genericRequestJsonOnly(
+			fmt.Sprintf("%s%s", cfg.KatsuUrl, "/api/public_overview"),
+			nil,
+			c,
+			jsonDeserialize,
+		)
 		if err != nil {
 			fmt.Println("something went wrong fetching 'publicOverview' for 'katsuCache': ", err)
 			return nil, err
 		}
 
+		publicOverview, ok := publicOverviewInterface.(JsonLike)
+		if !ok {
+			return nil, fmt.Errorf("failed to assert 'publicOverview' as JsonLike")
+		}
+
 		fmt.Println("storing 'publicOverview' in 'katsuCache'")
 		katsuCache.Set("publicOverview", publicOverview, cache.DefaultExpiration)
 
@@ -287,7 +337,7 @@ func main() {
 		}
 
 		// make a get request to the Katsu API
-		return katsuRequest("/api/public", qs, c, identityJSONTransform)
+		return katsuRequest("/api/public", qs, c, jsonDeserialize)
 	})
 
 	e.GET("/fields", func(c echo.Context) error {
@@ -296,6 +346,8 @@ func main() {
 		return katsuRequestBasic("/api/public_search_fields", c)
 	})
 
+	e.GET("/wes-runs", wesRequestWithDetailsAndPublic)
+
 	e.GET("/provenance", func(c echo.Context) error {
 		// Query Katsu for datasets provenance
 		return katsuRequestBasic("/api/public_dataset", c)

src/js/components/Overview/LastIngestion.tsx

@@ -0,0 +1,59 @@
+import React, { useCallback } from 'react';
+import { Typography, Card, Space, Empty } from 'antd';
+import { CalendarOutlined } from '@ant-design/icons';
+import { useTranslation } from 'react-i18next';
+
+import { DEFAULT_TRANSLATION } from '@/constants/configConstants';
+import { useAppSelector } from '@/hooks';
+
+const formatDataType = (dataType: string) => {
+  return dataType ? dataType.charAt(0).toUpperCase() + dataType.slice(1) + 's' : 'Unknown Data Type';
+};
+
+const LastIngestionInfo: React.FC = () => {
+  const { t, i18n } = useTranslation(DEFAULT_TRANSLATION);
+  const lastEndTimesByDataType = useAppSelector((state) => state.ingestionData?.lastEndTimesByDataType) || {};
+
+  const formatDate = useCallback(
+    (dateString: string) => {
+      const date = new Date(dateString);
+      return !isNaN(date.getTime())
+        ? date.toLocaleString(i18n.language, {
+            year: 'numeric',
+            month: 'long',
+            day: '2-digit',
+            hour: '2-digit',
+            minute: '2-digit',
+            hour12: true,
+          })
+        : 'Invalid Date';
+    },
+    [i18n.language]
+  );
+
+  const hasData = Object.keys(lastEndTimesByDataType).length > 0;
+
+  return (
+    <Space direction="vertical" size={0}>
+      <Typography.Title level={3}>{t('Latest Data Ingestion')}</Typography.Title>
+      <Space direction="horizontal">
+        {hasData ? (
+          Object.entries(lastEndTimesByDataType).map(([dataType, endTime]) => (
+            <Card key={dataType}>
+              <Space direction="vertical">
+                <Typography.Text style={{ color: 'rgba(0,0,0,0.45)' }}>{t(formatDataType(dataType))}</Typography.Text>
+                <Typography.Text>
+                  <CalendarOutlined /> {formatDate(endTime)}
+                </Typography.Text>
+              </Space>
+            </Card>
+          ))
+        ) : (
+          <Empty description={t('Ingestion history is empty.')} />
+        )}
+      </Space>
+    </Space>
+  );
+};
+
+export default LastIngestionInfo;

src/js/components/Overview/PublicOverview.tsx

@@ -10,6 +10,7 @@ import ManageChartsDrawer from './Drawer/ManageChartsDrawer';
 import Counts from './Counts';
 import { useAppSelector } from '@/hooks';
 import { useTranslation } from 'react-i18next';
+import LastIngestionInfo from './LastIngestion';
 
 const PublicOverview = () => {
   const { sections } = useAppSelector((state) => state.data);
@@ -51,6 +52,7 @@ const PublicOverview = () => {
                   <Divider />
                 </div>
               ))}
+            <LastIngestionInfo />
           </Col>
         </Row>
       </div>

src/js/components/TabbedDashboard.tsx

@@ -10,6 +10,7 @@ import { makeGetAboutRequest } from '@/features/content/content.store';
 import { makeGetDataRequestThunk } from '@/features/data/data.store';
 import { makeGetSearchFields } from '@/features/search/query.store';
 import { makeGetProvenanceRequest } from '@/features/provenance/provenance.store';
+import { makeGetIngestionDataRequest } from '@/features/ingestion/ingestion.store';
 import { getBeaconConfig } from '@/features/beacon/beaconConfig.store';
 
 import Loader from './Loader';
@@ -34,6 +35,7 @@ const TabbedDashboard = () => {
     dispatch(makeGetDataRequestThunk());
     dispatch(makeGetSearchFields());
     dispatch(makeGetProvenanceRequest());
+    dispatch(makeGetIngestionDataRequest());
   }, []);
 
   const isFetchingOverviewData = useAppSelector((state) => state.data.isFetchingData);

src/js/constants/configConstants.ts

@@ -5,6 +5,7 @@ export const publicOverviewUrl = '/overview';
 export const searchFieldsUrl = '/fields';
 export const katsuUrl = '/katsu';
 export const provenanceUrl = '/provenance';
+export const lastIngestionsUrl = '/wes-runs';
 
 export const DEFAULT_TRANSLATION = 'default_translation';
 export const NON_DEFAULT_TRANSLATION = 'translation';

src/js/features/ingestion/ingestion.store.ts

@@ -0,0 +1,57 @@
+import { createSlice, createAsyncThunk, PayloadAction } from '@reduxjs/toolkit';
+import axios from 'axios';
+import { lastIngestionsUrl } from '@/constants/configConstants';
+import { printAPIError } from '@/utils/error.util';
+import { ingestionData, LastIngestionResponse } from '@/types/lastIngestionResponse';
+
+export const makeGetIngestionDataRequest = createAsyncThunk<LastIngestionResponse, void, { rejectValue: string }>(
+  'ingestionData/getIngestionData',
+  (_, { rejectWithValue }) =>
+    axios
+      .get(lastIngestionsUrl)
+      .then((res) => res.data)
+      .catch(printAPIError(rejectWithValue))
+);
+
+export interface IngestionDataState {
+  isFetchingIngestionData: boolean;
+  ingestionData: ingestionData[];
+  lastEndTimesByDataType: { [dataType: string]: string };
+}
+
+const initialState: IngestionDataState = {
+  isFetchingIngestionData: false,
+  ingestionData: [],
+  lastEndTimesByDataType: {},
+};
+
+const IngestionDataStore = createSlice({
+  name: 'ingestionData',
+  initialState,
+  reducers: {},
+  extraReducers: (builder) => {
+    builder.addCase(makeGetIngestionDataRequest.pending, (state) => {
+      state.isFetchingIngestionData = true;
+    });
+    builder.addCase(
+      makeGetIngestionDataRequest.fulfilled,
+      (state, { payload }: PayloadAction<LastIngestionResponse>) => {
+        state.ingestionData = payload;
+        payload.forEach((ingestion) => {
+          const dataType = ingestion.details.request.tags.workflow_metadata.data_type;
+          const endTime = ingestion.details.run_log.end_time;
+          const previousEndTime = state.lastEndTimesByDataType[dataType];
+          if (!previousEndTime || new Date(endTime) > new Date(previousEndTime)) {
+            state.lastEndTimesByDataType[dataType] = endTime;
+          }
+        });
+        state.isFetchingIngestionData = false;
+      }
+    );
+    builder.addCase(makeGetIngestionDataRequest.rejected, (state) => {
+      state.isFetchingIngestionData = false;
+    });
+  },
+});
+
+export default IngestionDataStore.reducer;

src/js/store.ts

@@ -4,6 +4,7 @@ import configReducer from '@/features/config/config.store';
 import contentReducer from '@/features/content/content.store';
 import dataReducer from '@/features/data/data.store';
 import queryReducer from '@/features/search/query.store';
+import IngestionDataReducer from '@/features/ingestion/ingestion.store';
 import provenanceReducer from '@/features/provenance/provenance.store';
 import beaconConfigReducer from './features/beacon/beaconConfig.store';
 import beaconQueryReducer from './features/beacon/beaconQuery.store';
@@ -15,6 +16,7 @@ export const store = configureStore({
     data: dataReducer,
     query: queryReducer,
     provenance: provenanceReducer,
+    ingestionData: IngestionDataReducer,
     beaconConfig: beaconConfigReducer,
     beaconQuery: beaconQueryReducer,
   },