Merge pull request #209 from podaac/release/1.8.0

Release/1.8.0
podaac · Mar 12, 2024 · d687d21 · d687d21
2 parents 1becac0 + 5c8777d
commit d687d21
Show file tree

Hide file tree

Showing 9 changed files with 271 additions and 15 deletions.
diff --git a/.github/workflows/build-pipeline.yml b/.github/workflows/build-pipeline.yml
@@ -24,10 +24,10 @@ jobs:
       - uses: actions/setup-java@v2
         with:
           distribution: 'adopt'
-          java-version: '8.0.232'
+          java-version: '11.0.6'
       - uses: gradle/gradle-build-action@v1
         with:
-          gradle-version: 4.0.1
+          gradle-version: 8.0.1
       - name: Install Python 3
         uses: actions/setup-python@v2
         with:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,20 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.8.0]
+### Added
+- **User contribution**
+  - Add the "fileName" and "key" properties to the granuleFile object from buildS3GranuleFile function
+### Changed
+- **PODAAC-5877**
+  - Support java 11
+  - SonarQube and Jacoco report
+### Deprecated
+### Removed
+### Fixed
+### Security
+- Snyk: Upgrade com.amazonaws:aws-java-sdk-s3: 1.12.565 -> 1.12.661
+
 ## [v1.7.0] - 2022-12-12
 ### Added
 - **PODAAC-4308**
@@ -157,4 +171,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Fixed
 
-### Security
+### Security
diff --git a/README.md b/README.md
@@ -2,9 +2,17 @@
 
 ## Installation
 
-To build the Lambda code:
-
+To build the Lambda code, Refer to following Confluence page:
+https://wiki.jpl.nasa.gov/pages/viewpage.action?spaceKey=PD&title=SonarQube%2C+Jacoco+and+Java+17+upgrade
 ```shell
+* Build with sonarQube and Jacoco report
+mvn clean verify sonar:sonar \
+ -Dsonar.projectKey=cnm2cma-opensource \
+ -Dsonar.projectName='cnm2cma-opensource' \
+ -Dsonar.host.url=http://localhost:9000 \
+ -Dsonar.token=sqp_6dc05b1aa1f622b45112927d2a0510f209776860
+
+* Makde sure using java 11 and gradle 8.3
 mvn clean dependency:copy-dependencies
 gradle build
 ```

diff --git a/build.gradle b/build.gradle
@@ -1,6 +1,6 @@
 apply plugin: 'java'
-sourceCompatibility = 1.8
-targetCompatibility = 1.8
+sourceCompatibility = 1.11
+targetCompatibility = 1.11
 
 dependencies {
     implementation fileTree(dir: 'target/dependency/', include: '*.jar')
@@ -13,7 +13,7 @@ task buildZip(type: Zip) {
     into('lib') {
         from configurations.runtimeClasspath
     }
-    archiveName 'cnmToGranule.zip'
+    archiveFileName.set('cnmToGranule.zip')
 }
 
 build.dependsOn buildZip
diff --git a/pom.xml b/pom.xml
@@ -4,7 +4,7 @@
 
     <groupId>gov.nasa.cumulus</groupId>
     <artifactId>cnm-to-granule</artifactId>
-    <version>1.7.0</version>
+    <version>1.8.0-rc.1</version>
     <packaging>jar</packaging>
 
     <name>cnm-to-granule</name>
@@ -18,11 +18,21 @@
             <id>clojars.org</id>
             <url>https://repo.clojars.org</url>
         </repository>
+        <repository>
+            <id>central</id>
+            <name>Maven Central</name>
+            <url>https://repo1.maven.org/maven2</url>
+            <releases>
+                <enabled>true</enabled>
+            </releases>
+            <snapshots>
+                <enabled>false</enabled>
+            </snapshots>
+        </repository>
     </repositories>
 
 
     <dependencies>
-
         <!-- https://mvnrepository.com/artifact/com.google.code.gson/gson -->
         <dependency>
             <groupId>com.google.code.gson</groupId>
@@ -32,12 +42,12 @@
         <dependency>
             <groupId>gov.nasa.earthdata</groupId>
             <artifactId>cumulus-message-adapter</artifactId>
-            <version>1.3.9</version>
+            <version>2.0.0</version>
         </dependency>
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-java-sdk-s3</artifactId>
-            <version>1.12.215</version>
+            <version>1.12.661</version>
             <exclusions>
                 <exclusion>
                     <groupId>com.fasterxml.jackson.core</groupId>
@@ -53,7 +63,7 @@
         <dependency>
             <groupId>com.amazonaws</groupId>
             <artifactId>aws-lambda-java-core</artifactId>
-            <version>1.2.1</version>
+            <version>1.2.3</version>
         </dependency>
         <dependency>
             <groupId>junit</groupId>
@@ -65,7 +75,78 @@
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>
             <artifactId>jackson-databind</artifactId>
-            <version>2.13.2.2</version>
+            <version>2.13.4.2</version>
+        </dependency>
+        <dependency>
+            <groupId>commons-logging</groupId>
+            <artifactId>commons-logging</artifactId>
+            <version>1.2</version>
         </dependency>
     </dependencies>
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <version>3.8.0</version>
+                <configuration>
+                    <source>11</source>
+                    <target>11</target>
+                </configuration>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+                <version>3.2.1</version>
+                <executions>
+                    <execution>
+                        <phase>package</phase>
+                        <goals>
+                            <goal>shade</goal>
+                        </goals>
+                        <configuration>
+                            <transformers>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                    <mainClass>org.wordinator.xml2docx.MakeDocx</mainClass>
+                                    <manifestEntries>
+                                        <Multi-Release>true</Multi-Release>
+                                    </manifestEntries>
+                                </transformer>
+                                <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+                            </transformers>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.jacoco</groupId>
+                <artifactId>jacoco-maven-plugin</artifactId>
+                <version>0.8.7</version>
+                <executions>
+                    <execution>
+                        <id>prepare-agent</id>
+                        <goals>
+                            <goal>prepare-agent</goal>
+                        </goals>
+                    </execution>
+                    <execution>
+                        <id>report</id>
+                        <goals>
+                            <goal>report</goal>
+                        </goals>
+                        <configuration>
+                            <formats>
+                                <format>XML</format>
+                            </formats>
+                        </configuration>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.sonarsource.scanner.maven</groupId>
+                <artifactId>sonar-maven-plugin</artifactId>
+                <version>3.10.0.2594</version>
+            </plugin>
+        </plugins>
+    </build>
 </project>
diff --git a/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java b/src/main/java/gov/nasa/cumulus/CnmToGranuleHandler.java
@@ -151,7 +151,8 @@ public String PerformFunction(String input, Context context) throws Exception {
             String uri = StringUtils.trim(cnmFile.get("uri").getAsString());
             if (StringUtils.beginsWithIgnoreCase(uri, "s3://")) {
                 granuleFile = buildS3GranuleFile(cnmFile);
-            } else if (StringUtils.beginsWithIgnoreCase(uri, "https://")) {
+            } else if (StringUtils.beginsWithIgnoreCase(uri, "https://") ||
+                    StringUtils.beginsWithIgnoreCase(uri, "http://")) {
                 granuleFile = buildHttpsGranuleFile(cnmFile);
             } else if (StringUtils.beginsWithIgnoreCase(uri, "sftp://")) {
                 granuleFile = buildSftpGranuleFile(cnmFile);
@@ -207,13 +208,18 @@ public JsonObject buildS3GranuleFile(JsonObject cnmFile) {
             granuleFile.addProperty("checksum", cnmFile.get("checksum").getAsString());
         }
         granuleFile.addProperty("type", cnmFile.get("type").getAsString());
+
+        // Add the "fileName" and "key" properties as listed in the later version of the cumulus granule file schema
+        granuleFile.addProperty("fileName", cnmFile.get("name").getAsString());
+        granuleFile.addProperty("key", url_path + '/' + cnmFile.get("name").getAsString());
         return granuleFile;
     }
 
     public JsonObject buildHttpsGranuleFile(JsonObject cnmFile){
         String uri = StringUtils.trim(cnmFile.get("uri").getAsString());
         AdapterLogger.LogInfo(this.className + " uri: " + uri);
         String path = uri.replace("https://", "");
+        path = path.replace("http://", ""); // Work-a-around to working with http links also
         //find the path by getting character from (first / plus 1) to lastIndex of /
         String url_path = path.substring(path.indexOf("/") + 1, path.lastIndexOf("/"));
 

diff --git a/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java b/src/test/java/gov/nasa/cumulus/CnmToGranuleHandlerTest.java
@@ -147,4 +147,20 @@ public void testBuildSftpGranuleFile() throws Exception {
 
         assert(expectedJson.equals(outputJson.getAsJsonObject("output")));
     }
+
+    public void testBuildHttpGranuleFile() throws Exception {
+        ClassLoader classLoader = getClass().getClassLoader();
+        File inputJsonFile = new File(classLoader.getResource("http_input.json").getFile());
+        File expectedJsonFile = new File(classLoader.getResource("http_output.json").getFile());
+
+        String input = new String(Files.readAllBytes(inputJsonFile.toPath()));
+        String expected = new String(Files.readAllBytes(expectedJsonFile.toPath()));
+        JsonObject expectedJson = new JsonParser().parse(expected).getAsJsonObject();
+
+        CnmToGranuleHandler cnmToGranuleHandler = new CnmToGranuleHandler();
+        String output = cnmToGranuleHandler.PerformFunction(input, null);
+        JsonObject outputJson = new JsonParser().parse(output).getAsJsonObject();
+
+        assert(expectedJson.equals(outputJson.getAsJsonObject("output")));
+    }
 }
diff --git a/src/test/resources/http_input.json b/src/test/resources/http_input.json
@@ -0,0 +1,88 @@
+{
+  "input": {
+    "collection": "HLSL30",
+    "identifier": "2ffc5609-6ab4-4f58-b422-4d71ff43b269",
+    "duplicationid": "HLS.L30.T11VNK.2020259T185742.v2.0",
+    "version": "1.4",
+    "submissionTime": "2021-10-29T19:47:06Z",
+    "product": {
+      "name": "HLS.L30.T11VNK.2020259T185742.v2.0",
+      "dataVersion": "2.0",
+      "id": "HLS.L30.T11VNK.2020259T185742.v2.0",
+      "files": [
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json",
+          "size": 6430,
+          "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0_stac.json",
+          "type": "metadata"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif",
+          "size": 798505,
+          "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif",
+          "size": 561155,
+          "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg",
+          "size": 194484,
+          "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3",
+          "checksumType": "SHA512",
+          "uri": "http://e4ftl01.cr.usgs.gov/L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0/HLS.L30.T11VNK.2020259T185742.v2.0.jpg",
+          "type": "browse"
+        }
+      ]
+    }
+  },
+  "config": {
+    "collection": {
+      "files": [
+        {
+          "regex": ".*.h5$",
+          "sampleFileName": "HLSL30_product_0001-of-0050.h5",
+          "type": "data",
+          "bucket": "protected"
+        },
+        {
+          "regex": ".*.iso.xml$",
+          "sampleFileName": "HLSL30_product_0001-of-0019.iso.xml",
+          "type": "metadata",
+          "bucket": "protected"
+        },
+        {
+          "regex": ".*.cmr.json$",
+          "sampleFileName": "HLSL30_product_0001-of-0019.cmr.json",
+          "type": "metadata",
+          "bucket": "public"
+        }
+      ],
+      "name": "HLSL30",
+      "granuleIdExtraction": "^(.*)((\\.cmr\\.json)|(\\.h5)|(\\.h5\\.mp))$",
+      "granuleId": "^.*$",
+      "dataType": "HLSL30",
+      "provider_path": "HLSL30/",
+      "version": "2.0",
+      "updatedAt": 1552434051881,
+      "duplicateHandling": "replace",
+      "sampleFileName": "HLSL30_product_0001-of-0050.h5",
+      "createdAt": 1552434051881,
+      "meta": {
+        "required-files": [
+          ".*.h5$",
+          ".*.iso.xml$"
+        ]
+      }
+    }
+  }
+}
diff --git a/src/test/resources/http_output.json b/src/test/resources/http_output.json
@@ -0,0 +1,43 @@
+{
+  "granules": [
+    {
+      "granuleId": "HLS.L30.T11VNK.2020259T185742.v2.0",
+      "version": "2.0",
+      "dataType": "HLSL30",
+      "files": [
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0_stac.json",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 6430,
+          "checksumType": "SHA512",
+          "checksum": "d5241ce4d88c52b45a44c48c8553f79dd8a883b87fc4da8fad4cd9506783e1e7aaedb6724a58e4f1cab27df17f0261c8b2d376ae2193d892bed02204b9d637f3",
+          "type": "metadata"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.VZA.tif",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 798505,
+          "checksumType": "SHA512",
+          "checksum": "b110a6c6375e85887b760d027be47172b8e14576bde2213155627de26169d8ac045bda05432c6495d62a795fb2bdba17d246710dba104a3d76f1cf727272dbbd",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.SZA.tif",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 561155,
+          "checksumType": "SHA512",
+          "checksum": "88955f6139b6d5cce674e78d19d72ca6861e1e59bdd764863e34442f6232084b7209cab3c159fcfc059986632a6ebb684a9338290950bceb8da273c047b6d678",
+          "type": "data"
+        },
+        {
+          "name": "HLS.L30.T11VNK.2020259T185742.v2.0.jpg",
+          "path": "L30/data/2020259/HLS.L30.T11VNK.2020259T185742.v2.0",
+          "size": 194484,
+          "checksumType": "SHA512",
+          "checksum": "9db4cb08b333b8a97b859d89f0f99500541816a70a0017d89db59b9c9dd8684e92fad200cb501a95139f0a510d11b82ea54329a218597f24d0dca98eaa66f6f3",
+          "type": "browse"
+        }
+      ]
+    }
+  ]
+}