From a0ec051482b5a4e8ff269fc42ff8c535e8807ea0 Mon Sep 17 00:00:00 2001 From: krwong Date: Thu, 11 Apr 2024 11:55:54 -0400 Subject: [PATCH 1/4] add StreamingMetadataService and streaming metadata to test resources --- .../services/StreamingMetadataService.java | 128 ++++++++++++++++++ .../StreamingMetadataServiceTest.java | 83 ++++++++++++ .../mini_gilmer/index/description/desc.all | 2 + src/test/resources/gilmer_fields.csv | 2 + 4 files changed, 215 insertions(+) create mode 100644 src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java create mode 100644 src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java new file mode 100644 index 00000000..b3faf860 --- /dev/null +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java @@ -0,0 +1,128 @@ +package edu.unc.lib.boxc.migration.cdm.services; + +import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; +import edu.unc.lib.boxc.migration.cdm.model.CdmFieldInfo; +import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.List; + +import static edu.unc.lib.boxc.migration.cdm.util.CLIConstants.outputLogger; + +/** + * Service for retrieving streaming metadata + * @author krwong + */ +public class StreamingMetadataService { + private static final Logger log = LoggerFactory.getLogger(StreamingMetadataService.class); + + private MigrationProject project; + private CdmFieldService fieldService; + private CdmIndexService indexService; + + public static final String STREAMING_FILE_FIELD = "stream"; + public static final String DURACLOUD_SPACE_FIELD = "duracl"; + public static final String PLAYLIST_FILE_EXTENSION = "-playlist.m3u8"; + public static final String DURACLOUD_OPEN = "open-hls"; + public static final String DURACLOUD_CAMPUS = "campus-hls"; + public static final String DURACLOUD_CLOSED = "closed-hls"; + public static final String STREAMING_HOST = "duracloud"; + + /** + * Verify if a record has streaming metadata + * @param cdmId + * @return true/false + */ + public boolean verifyRecordHasStreamingMetadata(String cdmId) throws Exception { + // check if project has streamingFile field and duracloudSpace field + fieldService.validateFieldsFile(project); + CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); + List exportFields = fieldInfo.listAllExportFields(); + if (!exportFields.contains(STREAMING_FILE_FIELD) && !exportFields.contains(DURACLOUD_SPACE_FIELD)) { + return false; + } + + // check if record has streamingFile field and duracloudSpace field + Connection conn = null; + try { + conn = indexService.openDbConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("select " + STREAMING_FILE_FIELD + ", " + DURACLOUD_SPACE_FIELD + + " from " + CdmIndexService.TB_NAME + + " where " + CdmFieldInfo.CDM_ID + " = " + cdmId); + while (rs.next()) { + if (!rs.getString(1).isEmpty() && !rs.getString(2).isEmpty()) { + return true; + } + } + return false; + } catch (SQLException e) { + throw new MigrationException("Error interacting with export index", e); + } finally { + CdmIndexService.closeDbConnection(conn); + } + } + + /** + * Retrieve streaming metadata and remap to correct values + * @param cdmId + * @return object with streamingFile, duracloudSpace, and streamingHost fields + */ + public Object[] getStreamingMetadata(String cdmId) throws Exception { + String duracloudSpace = null; + String streamingFile = null; + + // retrieve streaming metadata + Connection conn = null; + try { + conn = indexService.openDbConnection(); + Statement stmt = conn.createStatement(); + ResultSet rs = stmt.executeQuery("select " + STREAMING_FILE_FIELD + ", " + DURACLOUD_SPACE_FIELD + + " from " + CdmIndexService.TB_NAME + + " where " + CdmFieldInfo.CDM_ID + " = " + cdmId); + while (rs.next()) { + if (!rs.getString(1).isEmpty()) { + streamingFile = rs.getString(1); + } + if (!rs.getString(2).isEmpty()) { + duracloudSpace = rs.getString(2); + } + } + } catch (SQLException e) { + throw new MigrationException("Error interacting with export index", e); + } finally { + CdmIndexService.closeDbConnection(conn); + } + + // transform to playlist file extensions + streamingFile = streamingFile.split("\\.")[0] + PLAYLIST_FILE_EXTENSION; + + // transform to current duracloud space IDs + if (duracloudSpace.contains("open")) { + duracloudSpace = DURACLOUD_OPEN; + } else if (duracloudSpace.contains("campus")) { + duracloudSpace = DURACLOUD_CAMPUS; + } else if (duracloudSpace.contains("closed")) { + duracloudSpace = DURACLOUD_CLOSED; + } + + return new Object[] {streamingFile, duracloudSpace, STREAMING_HOST}; + } + + public void setProject(MigrationProject project) { + this.project = project; + } + + public void setFieldService(CdmFieldService fieldService) { + this.fieldService = fieldService; + } + + public void setIndexService(CdmIndexService indexService) { + this.indexService = indexService; + } +} diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java new file mode 100644 index 00000000..0138fc81 --- /dev/null +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java @@ -0,0 +1,83 @@ +package edu.unc.lib.boxc.migration.cdm.services; + +import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; +import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper; +import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper; +import edu.unc.lib.boxc.migration.cdm.test.SipServiceHelper; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.MockitoAnnotations.openMocks; + +public class StreamingMetadataServiceTest { + private static final String PROJECT_NAME = "proj"; + + @TempDir + public Path tmpFolder; + + private SipServiceHelper testHelper; + private MigrationProject project; + private StreamingMetadataService service; + private AutoCloseable closeable; + + @BeforeEach + public void setup() throws Exception { + closeable = openMocks(this); + project = MigrationProjectFactory.createMigrationProject( + tmpFolder, PROJECT_NAME, null, "user", + CdmEnvironmentHelper.DEFAULT_ENV_ID, BxcEnvironmentHelper.DEFAULT_ENV_ID); + testHelper = new SipServiceHelper(project, tmpFolder); + service = new StreamingMetadataService(); + service.setProject(project); + service.setFieldService(testHelper.getFieldService()); + service.setIndexService(testHelper.getIndexService()); + } + + @AfterEach + void closeService() throws Exception { + closeable.close(); + } + + @Test + public void verifyNoStreamingMetadata() throws Exception { + testHelper.indexExportData("mini_gilmer"); + + var result = service.verifyRecordHasStreamingMetadata("25"); + assertFalse(result); + } + + @Test + public void verifyHasStreamingMetadata() throws Exception { + testHelper.indexExportData("mini_gilmer"); + + var result = service.verifyRecordHasStreamingMetadata("27"); + assertTrue(result); + } + + @Test + public void getStreamingMetadataSuccess() throws Exception { + testHelper.indexExportData("mini_gilmer"); + + var result = service.getStreamingMetadata("27"); + assertEquals("gilmer_recording-playlist.m3u8", result[0]); + assertEquals("open-hls", result[1]); + assertEquals("duracloud", result[2]); + } + + @Test + public void getStreamingMetadataFail() throws Exception { + testHelper.indexExportData("mini_gilmer"); + + Exception exception = assertThrows(NullPointerException.class, () -> { + service.getStreamingMetadata("25"); + }); + } +} diff --git a/src/test/resources/descriptions/mini_gilmer/index/description/desc.all b/src/test/resources/descriptions/mini_gilmer/index/description/desc.all index aef73a25..a006fbcf 100644 --- a/src/test/resources/descriptions/mini_gilmer/index/description/desc.all +++ b/src/test/resources/descriptions/mini_gilmer/index/description/desc.all @@ -182,6 +182,8 @@ TIFF TIFF group2 +gilmer_recording.mp3 +sfc20009-open /shc/gilmer_maps/ 276_203_E.tif 50.jp2 diff --git a/src/test/resources/gilmer_fields.csv b/src/test/resources/gilmer_fields.csv index 81e39ac1..7abd8ca3 100644 --- a/src/test/resources/gilmer_fields.csv +++ b/src/test/resources/gilmer_fields.csv @@ -52,6 +52,8 @@ color,color,Color Space Raw Scan,false,n,n,y,n,BLANK coloa,coloa,Color Space filename,false,n,n,y,n,BLANK fila,fila,File Format Raw Scan,false,n,n,y,y,BLANK filb,filb,File Format filename,false,n,n,y,y,BLANK +stream,stream,StreamingFile,false,n,n,n,y,BLANK +duracl,duracl,duracloudSpace,false,n,n,n,y,identi full,full,path,false,n,n,n,n,BLANK fullrs,fullrs,Full resolution,false,n,n,y,n, dmoclcno,dmoclcno,OCLC number,false,n,n,y,n, From 023172c67fad552d2d7d92c22ecc3a57fb4059d1 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Fri, 12 Apr 2024 09:11:24 -0400 Subject: [PATCH 2/4] Fix test to account for extra columns. Update matcher due to mockito 5 --- .../lib/boxc/migration/cdm/services/CdmFieldServiceTest.java | 2 +- .../cdm/services/FieldAssessmentTemplateServiceTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java index 5c579d3b..b574f469 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/CdmFieldServiceTest.java @@ -3,7 +3,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.junit.jupiter.api.Assertions.fail; -import static org.mockito.Matchers.any; +import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.when; import static org.mockito.MockitoAnnotations.openMocks; diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java index 98061999..c22a8658 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/FieldAssessmentTemplateServiceTest.java @@ -53,7 +53,7 @@ public void allExpectedCellsPopulatedTest() throws Exception { XSSFWorkbook workbook = new XSSFWorkbook(inputStream); Sheet sheet = workbook.getSheetAt(0); - assertEquals(60, sheet.getLastRowNum()); + assertEquals(62, sheet.getLastRowNum()); assertEquals(16, sheet.getRow(0).getPhysicalNumberOfCells()); assertEquals(12, sheet.getRow(1).getPhysicalNumberOfCells()); assertEquals(12, sheet.getRow(60).getPhysicalNumberOfCells()); From 06492f0e3b96acd8d30e9ca58034a6187c68e617 Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Fri, 12 Apr 2024 09:48:08 -0400 Subject: [PATCH 3/4] DRY up query execution, cache whether project has streaming fields, make return type more specific, throw explicit error when no fields --- .../services/StreamingMetadataService.java | 80 ++++++++++--------- .../StreamingMetadataServiceTest.java | 3 +- 2 files changed, 43 insertions(+), 40 deletions(-) diff --git a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java index b3faf860..96f04247 100644 --- a/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java +++ b/src/main/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataService.java @@ -33,39 +33,31 @@ public class StreamingMetadataService { public static final String DURACLOUD_CLOSED = "closed-hls"; public static final String STREAMING_HOST = "duracloud"; + private Boolean projectHasStreamingMetadata = null; + /** * Verify if a record has streaming metadata * @param cdmId * @return true/false */ - public boolean verifyRecordHasStreamingMetadata(String cdmId) throws Exception { - // check if project has streamingFile field and duracloudSpace field - fieldService.validateFieldsFile(project); - CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); - List exportFields = fieldInfo.listAllExportFields(); - if (!exportFields.contains(STREAMING_FILE_FIELD) && !exportFields.contains(DURACLOUD_SPACE_FIELD)) { + public boolean verifyRecordHasStreamingMetadata(String cdmId) { + if (!hasProjectStreamingMetadataField()) { return false; } - // check if record has streamingFile field and duracloudSpace field - Connection conn = null; - try { - conn = indexService.openDbConnection(); - Statement stmt = conn.createStatement(); - ResultSet rs = stmt.executeQuery("select " + STREAMING_FILE_FIELD + ", " + DURACLOUD_SPACE_FIELD - + " from " + CdmIndexService.TB_NAME - + " where " + CdmFieldInfo.CDM_ID + " = " + cdmId); - while (rs.next()) { - if (!rs.getString(1).isEmpty() && !rs.getString(2).isEmpty()) { - return true; - } - } - return false; - } catch (SQLException e) { - throw new MigrationException("Error interacting with export index", e); - } finally { - CdmIndexService.closeDbConnection(conn); + var streamingFields = getStreamingFieldValues(cdmId); + return streamingFields[0] != null && streamingFields[1] != null; + } + + private boolean hasProjectStreamingMetadataField() { + if (projectHasStreamingMetadata == null) { + // check if project has streamingFile field and duracloudSpace field + fieldService.validateFieldsFile(project); + CdmFieldInfo fieldInfo = fieldService.loadFieldsFromProject(project); + List exportFields = fieldInfo.listAllExportFields(); + projectHasStreamingMetadata = exportFields.contains(STREAMING_FILE_FIELD) && exportFields.contains(DURACLOUD_SPACE_FIELD); } + return projectHasStreamingMetadata; } /** @@ -73,7 +65,30 @@ public boolean verifyRecordHasStreamingMetadata(String cdmId) throws Exception { * @param cdmId * @return object with streamingFile, duracloudSpace, and streamingHost fields */ - public Object[] getStreamingMetadata(String cdmId) throws Exception { + public String[] getStreamingMetadata(String cdmId) { + var streamingValues = getStreamingFieldValues(cdmId); + String duracloudSpace = streamingValues[1]; + String streamingFile = streamingValues[0]; + + if (duracloudSpace == null || streamingFile == null) { + throw new MigrationException("Streaming metadata not found for " + cdmId); + } + // transform to playlist file extensions + streamingFile = streamingFile.split("\\.")[0] + PLAYLIST_FILE_EXTENSION; + + // transform to current duracloud space IDs + if (duracloudSpace.contains("open")) { + duracloudSpace = DURACLOUD_OPEN; + } else if (duracloudSpace.contains("campus")) { + duracloudSpace = DURACLOUD_CAMPUS; + } else if (duracloudSpace.contains("closed")) { + duracloudSpace = DURACLOUD_CLOSED; + } + + return new String[] {streamingFile, duracloudSpace, STREAMING_HOST}; + } + + private String[] getStreamingFieldValues(String cdmId) { String duracloudSpace = null; String streamingFile = null; @@ -98,20 +113,7 @@ public Object[] getStreamingMetadata(String cdmId) throws Exception { } finally { CdmIndexService.closeDbConnection(conn); } - - // transform to playlist file extensions - streamingFile = streamingFile.split("\\.")[0] + PLAYLIST_FILE_EXTENSION; - - // transform to current duracloud space IDs - if (duracloudSpace.contains("open")) { - duracloudSpace = DURACLOUD_OPEN; - } else if (duracloudSpace.contains("campus")) { - duracloudSpace = DURACLOUD_CAMPUS; - } else if (duracloudSpace.contains("closed")) { - duracloudSpace = DURACLOUD_CLOSED; - } - - return new Object[] {streamingFile, duracloudSpace, STREAMING_HOST}; + return new String[] {streamingFile, duracloudSpace}; } public void setProject(MigrationProject project) { diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java index 0138fc81..af2b59d5 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/services/StreamingMetadataServiceTest.java @@ -1,5 +1,6 @@ package edu.unc.lib.boxc.migration.cdm.services; +import edu.unc.lib.boxc.migration.cdm.exceptions.MigrationException; import edu.unc.lib.boxc.migration.cdm.model.MigrationProject; import edu.unc.lib.boxc.migration.cdm.test.BxcEnvironmentHelper; import edu.unc.lib.boxc.migration.cdm.test.CdmEnvironmentHelper; @@ -76,7 +77,7 @@ public void getStreamingMetadataSuccess() throws Exception { public void getStreamingMetadataFail() throws Exception { testHelper.indexExportData("mini_gilmer"); - Exception exception = assertThrows(NullPointerException.class, () -> { + Exception exception = assertThrows(MigrationException.class, () -> { service.getStreamingMetadata("25"); }); } From 2d259e61ee614e8ca615738f581f5982794b3aac Mon Sep 17 00:00:00 2001 From: Ben Pennell Date: Fri, 12 Apr 2024 10:05:17 -0400 Subject: [PATCH 4/4] Update counts based on adding streaming columns --- .../java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java b/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java index 6dfa1a13..2ad76dd7 100644 --- a/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java +++ b/src/test/java/edu/unc/lib/boxc/migration/cdm/StatusCommandIT.java @@ -54,7 +54,7 @@ public void reportInitialized() throws Exception { assertOutputContains("CDM Collection Fields"); assertOutputMatches(".*Mapping File Valid: +Yes.*"); - assertOutputMatches(".*Fields: +61\n.*"); + assertOutputMatches(".*Fields: +63\n.*"); assertOutputMatches(".*Skipped: +1\n.*"); assertOutputContains("CDM Collection Exports");