Merge branch 'develop'

ocean-color-ac-challenge · Jun 17, 2015 · 3568764 · 3568764
2 parents 9ab8866 + 024a84b
commit 3568764
Show file tree

Hide file tree

Showing 7 changed files with 386 additions and 90 deletions.
diff --git a/README.md b/README.md
@@ -1,11 +1,17 @@
-beam-meris-ac
+Participant-B
 =============
 
 MERIS atmospheric correction effects using BEAM - using SmacOp
 
-### Installation 
+### Quick link
 
-Log on the developer sandbox and run these commands in a shell:
+* [Installation](#installation)
+* [Submitting the workflow](#submit)
+* [Test the application](#test)
+
+### <a name="installation"></a>Installation
+
+Log on the Developer Cloud Sandbox and run these commands in a shell:
 
 * Install **Java 7**
 
@@ -34,23 +40,32 @@ Enter to keep the current selection[+], or type selection number:
 
 Select java 1.7 out of the menu options by typing the correct number (here it's *3*).
 
-* Install this application
+* Install the application from github
 
 ```bash
 cd
-git clone [email protected]:ocean-color-ac-challenge/beam-meris-ac.git
-cd beam-meris-ac
+git clone [email protected]:ocean-color-ac-challenge/Participant-B.git
+cd Participant-B
 mvn install
 ```
 
-### Submitting the workflow
+* Install the application from rpm
+
+```bash
+curl -L -O https://github.com/ocean-color-ac-challenge/Participant-B/releases/download/v0.35/Participant-B-x.xx-ciop.noarch.rpm
+sudo yum -y install Participant-B-x.xx-ciop.noarch.rpm
+```
+
+where x.xx is the version of the RPM created
 
-* Via the Sandbox shell 
+### <a name="submit"></a>Submitting the workflow
+
+* Via the Sandbox shell
 
 Run this command in a shell:
 
 ```bash
-ciop-simwf
+ciop-run
 ```
 
 * Via the Web Processing Service dashboard tab
@@ -59,8 +74,149 @@ Use your browser to go to the Sandbox dashboard tab at the address http://<sandb
 
 Click on the _Invoke_ tab
 
-Below the "Process List" click on _BEAM Atmos Correction_
+Below the "Process List" click on _Participant B_
+
+Fill the parameters and click submit.
+
+### <a name="test"></a>Test the application
+
+##### Test Participant-B-01
+
+* Test Procedure
+
+Invoke the application via the Dashboard with the parameters listed in the test inputs specification
+
+* Inputs specification
+
+| Parameter   | Value |
+|-------------|---------------------------------------------------------------------------------------------------------|
+| Catalogue      | https://challenges.esa.int/eceo/datapackage/RRPAR/description?key=9d79148d-3e17-414b-9983-e4cef9e88ec6 |
+| Start date   | 2002-03-01T00:00:00Z |
+| End date     | 2012-05-09T23:59:59Z |
+| aerosolType | CONTINENTAL |
+| Aerosol Optical Depth  | 0.2 |
+| Use ECMWF data in the MERIS ADS | true |
+| List of POI for reflectance extraction |BOUS,43.367,7.9\|AAOT,45.314,12.508\|MOBY,20.828,-157.193 |
+| Flag to trigger the publishing of Level 2 products generated | true |
+| Flag to extract POI reflectances | false |
+
+* Outputs specification
+
+| Output                                                             | Size |
+|--------------------------------------------------------------------|------|
+MER_RR__1PNACR20060730_093546_000021432049_00480_23079_0000.N1.png|10.01 MB|
+MER_RR__1PNACR20060730_093546_000021432049_00480_23079_0000.N1.tgz|193.73 MB|
+MER_RR__1PRACR20030306_201313_000026082014_00243_05307_0000.N1.png|21.35 MB|
+MER_RR__1PRACR20030306_201313_000026082014_00243_05307_0000.N1.tgz|367.81 MB|
+MER_RR__1PRACR20040903_201756_000026282030_00057_13137_0000.N1.png|21.87 MB|
+MER_RR__1PRACR20040903_201756_000026282030_00057_13137_0000.N1.tgz|367.88 MB|
+MER_RR__1PRACR20050504_093419_000026382037_00022_16609_0000.N1.png|14.73 MB|
+MER_RR__1PRACR20050504_093419_000026382037_00022_16609_0000.N1.tgz|265.6 MB|
+MER_RR__1PRACR20070920_092716_000026302061_00437_29048_0000.N1.png|14.86 MB|
+MER_RR__1PRACR20070920_092716_000026302061_00437_29048_0000.N1.tgz|266 MB|
+MER_RR__1PRACR20080816_092020_000026302071_00165_33786_0000.N1.png|16.58 MB|
+MER_RR__1PRACR20080816_092020_000026302071_00165_33786_0000.N1.tgz|267.37 MB|
+
+* Test pass/fail criteria
+
+All products listed in test outputs specification are generated
+
+##### Test Participant-B-02
+
+* Test Procedure
+
+Invoke the application via the Dashboard with the parameters listed in the test inputs specification
+
+* Inputs specification
+
+| Parameter   | Value |
+|-------------|---------------------------------------------------------------------------------------------------------|
+| Catalogue      | https://challenges.esa.int/eceo/datapackage/FRSPAR/description?key=495f181f-47d3-4668-b717-d36d4a560837 |
+| Start date   | 2002-03-01T00:00:00Z |
+| End date     | 2012-05-09T23:59:59Z |
+| aerosolType | CONTINENTAL |
+| Aerosol Optical Depth  | 0.2 |
+| Use ECMWF data in the MERIS ADS | true |
+| List of POI for reflectance extraction |BOUS,43.367,7.9\|AAOT,45.314,12.508\|MOBY,20.828,-157.193 |
+| Flag to trigger the publishing of Level 2 products generated | true |
+| Flag to extract POI reflectances | false |
+
+* Outputs specification
+
+| Output                                                             | Size |
+|--------------------------------------------------------------------|------|
+MER_FRS_1PPEPA20040711_020449_000002422028_00275_12353_1787.N1.png|21.16 MB|
+MER_FRS_1PPEPA20040711_020449_000002422028_00275_12353_1787.N1.tgz|466.86 MB|
+MER_FRS_1PPEPA20040717_021529_000002642028_00361_12439_1968.N1.png|18.27 MB|
+MER_FRS_1PPEPA20040717_021529_000002642028_00361_12439_1968.N1.tgz|371.85 MB|
+
+* Test pass/fail criteria
+
+All products listed in test outputs specification are generated
+
+##### Test Participant-B-03
+
+* Test Procedure
+
+Invoke the application via the Dashboard with the parameters listed in the test inputs specification
+
+* Inputs specification
+
+| Parameter   | Value |
+|-------------|---------------------------------------------------------------------------------------------------------|
+| Catalogue      | https://challenges.esa.int/eceo/datapackage/RRPAR/description?key=9d79148d-3e17-414b-9983-e4cef9e88ec6 |
+| Start date   | 2002-03-01T00:00:00Z |
+| End date     | 2012-05-09T23:59:59Z |
+| aerosolType | CONTINENTAL |
+| Aerosol Optical Depth  | 0.2 |
+| Use ECMWF data in the MERIS ADS | true |
+| List of POI for reflectance extraction |BOUS,43.367,7.9\|AAOT,45.314,12.508\|MOBY,20.828,-157.193 |
+| Flag to trigger the publishing of Level 2 products generated | false |
+| Flag to extract POI reflectances | true |
+
+* Outputs specification
+
+| Output                                                             | Size |
+|--------------------------------------------------------------------|------|
+MER_RR__1PNACR20060730_093546_000021432049_00480_23079_0000.N1.txt|1.08 KB|
+MER_RR__1PRACR20030306_201313_000026082014_00243_05307_0000.N1.txt|0.78 KB|
+MER_RR__1PRACR20040903_201756_000026282030_00057_13137_0000.N1.txt|0.78 KB|
+MER_RR__1PRACR20050504_093419_000026382037_00022_16609_0000.N1.txt|1.06 KB|
+MER_RR__1PRACR20070920_092716_000026302061_00437_29048_0000.N1.txt|1.06 KB|
+MER_RR__1PRACR20080816_092020_000026302071_00165_33786_0000.N1.txt|1.05 KB|
+
+* Test pass/fail criteria
+
+All products listed in test outputs specification are generated
+
+
+##### Test Participant-B-04
+
+* Test Procedure
+
+Invoke the application via the Dashboard with the parameters listed in the test inputs specification
+
+* Inputs specification 
+
+| Parameter   | Value |
+|-------------|---------------------------------------------------------------------------------------------------------|
+| Catalogue      | https://challenges.esa.int/eceo/datapackage/FRSPAR/description?key=495f181f-47d3-4668-b717-d36d4a560837 |
+| Start date   | 2002-03-01T00:00:00Z |
+| End date     | 2012-05-09T23:59:59Z |
+| aerosolType | CONTINENTAL |
+| Aerosol Optical Depth  | 0.2 |
+| Use ECMWF data in the MERIS ADS | true |
+| List of POI for reflectance extraction |CHINA,27,122|
+| Flag to trigger the publishing of Level 2 products generated | false |
+| Flag to extract POI reflectances | true |
+
+* Outputs specification
 
-Fill the parameters and click submit. 
+| Output                                                             | Size |
+|--------------------------------------------------------------------|------|
+MER_FRS_1PPEPA20040711_020449_000002422028_00275_12353_1787.N1.txt|0.77 KB|
+MER_FRS_1PPEPA20040717_021529_000002642028_00361_12439_1968.N1.txt|0.77 KB|
 
+* Test pass/fail criteria
 
+All products listed in test outputs specification are generated
diff --git a/pom.xml b/pom.xml
@@ -2,15 +2,15 @@
 <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <groupId>com.terradue.ciop.app</groupId>
-  <artifactId>beam-meris-ac</artifactId>
+  <artifactId>Participant-B</artifactId>
   <packaging>rpm</packaging>
-  <name>beam-meris-ac</name>
+  <name>${project.artifactId}</name>
   <description>BEAM Atmospheric Correction Processing</description>
   <version>0.2-SNAPSHOT</version>
   <scm>
-    <connection>scm:git:[email protected]:ocean-color-ac-challenge/beam-meris-ac.git</connection>
-    <developerConnection>scm:git:[email protected]:ocean-color-ac-challenge/beam-meris-ac.git</developerConnection>
-    <url>https://github.com/ocean-color-ac-challenge/beam-meris-ac</url>
+    <connection>scm:git:[email protected]:ocean-color-ac-challenge/${project.artifactId}.git</connection>
+    <developerConnection>scm:git:[email protected]:ocean-color-ac-challenge/${project.artifactId}.git</developerConnection>
+    <url>https://github.com/ocean-color-ac-challenge/${project.artifactId}</url>
     <tag>HEAD</tag>
   </scm>
   <dependencies>
@@ -44,6 +44,11 @@
       <artifactId>beam-meris-smac</artifactId>
       <version>5.0</version>
     </dependency>
+    <dependency>
+      <groupId>org.esa.beam</groupId>
+      <artifactId>beam-pixel-extraction</artifactId>
+      <version>5.0</version>
+    </dependency>
     <dependency>
       <groupId>ncsa.hdf</groupId>
       <artifactId>lib-hdf</artifactId>
@@ -129,8 +134,8 @@ def writer = new FileWriter(new File("/application/properties.xml"))
 def xml = new MarkupBuilder(writer)
 xml.mkp.xmlDeclaration(version: "1.0", encoding: "utf-8")
 xml.properties() {
-github_repository(version:pom.version, pom.artifactId)
-user_reference(env['USER'])
+  github_repository(version:pom.version, pom.artifactId)
+  user_reference(env['USER'])
 }
 </source>
             </configuration>
@@ -263,8 +268,8 @@ user_reference(env['USER'])
           <distribution>Terradue ${project.inceptionYear}</distribution>
           <group>air</group>
           <packager>Terradue</packager>
-          <summary>ESA BEAM Toolbox atmospheric correction</summary>
-          <description/>
+          <summary>${project.artifactId}</summary>
+          <description>${project.artifactId}</description>
           <name>${project.artifactId}</name>
           <defaultDirmode>775</defaultDirmode>
           <defaultFilemode>664</defaultFilemode>
@@ -355,7 +360,7 @@ user_reference(env['USER'])
       <plugin>
         <groupId>de.jutzig</groupId>
         <artifactId>github-release-plugin</artifactId>
-        <version>1.0.3</version>
+        <version>1.1.1-dns</version>
         <executions>
           <execution>
             <id>github-upload</id>

diff --git a/src/main/app-resources/application.xml b/src/main/app-resources/application.xml
@@ -1,37 +1,42 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <application id="beam_atcorr">
   <jobTemplates>
-
     <!-- BEAM Atmos Corr operator job template  -->
     <jobTemplate id="SmacOp">
       <streamingExecutable>/application/smacop/run.sh</streamingExecutable>
-      <defaultParameters>						
+      <defaultParameters>
         <parameter id="format">BEAM-DIMAP</parameter>
-	<parameter id="startdate" title="Start date" abstract="Start date of the TOI. Used as queryable for the MERIS Level 1 catalogue" scope="runtime" type="opensearch" target="time:start">2012-04-05</parameter>
-        <parameter id="enddate" title="End date" abstract="End date of the TOI. Used as queryable for the MERIS Level 1 catalogue" scope="runtime" type="opensearch" target="time:end">2012-04-06</parameter>
-        <parameter id="aerosolType" title="aerosolType" abstract="Aerosol type: CONTINENTAL or DESERT" scope="test">CONTINENTAL</parameter>
-	<parameter id="bandNames" title="bandNames" abstract="Bands to process as a comma separated list (e.g. radiance_2,radiance_3)" scope="test">radiance_1,radiance_2,radiance_3,radiance_4,radiance_5,radiance_6,radiance_7</parameter>
-	<parameter id="tauAero550" title="tauAero550" abstract="Aerosol optical depth (e.g. 0.2)" scope="test">0.3</parameter>
-        <parameter id="useMerisADS" title="useMerisADS" abstract="Use ECMWF data in the MERIS ADS (true or false)" scope="test">true</parameter>
-	<parameter id="invalidPixel" title="invalidPixel" abstract="Default reflectance for invalid pixel (e.g. 0.0)" scope="test">0.0</parameter> 
-        <parameter id="maskExpression" title="maskExpression" abstract="Mask expression for the whole MERIS view" scope="test">not l1_flags.LAND_OCEAN</parameter>
-        <parameter id="surfPress" title="surfPress" abstract="Surface pressure (e.g. 1013.0)" scope="test">1013.0</parameter>
-        <parameter id="uH2o" title="uH2o" abstract="Relative humidity (e.g. 3.0)" scope="test">3.0</parameter>
-        <parameter id="uO3" title="uO3" abstract="Ozone content (e.g. 0.15)" scope="test">0.15</parameter>
+        <parameter id="startdate" title="Start date" abstract="Start date of the TOI. Used as queryable for the MERIS Level 1 catalogue or data package (format ISO 8601 e.g. 2015-06-03T13:21:58Z)" scope="runtime" type="opensearch" target="time:start">2002-04-05</parameter>
+        <parameter id="enddate" title="End date" abstract="End date of the TOI. Used as queryable for the MERIS Level 1 catalogue or data package (format ISO 8601 e.g. 2015-06-03T13:21:58Z)" scope="runtime" type="opensearch" target="time:end">2015-01-06</parameter>
+        <parameter id="aerosolType" title="aerosolType" abstract="Aerosol type: CONTINENTAL or DESERT" scope="runtime">CONTINENTAL</parameter>
+        <parameter id="bandNames" title="bandNames" abstract="Bands to process as a comma separated list (e.g. radiance_2,radiance_3)" scope="test">radiance_1,radiance_2,radiance_3,radiance_4,radiance_5,radiance_6,radiance_7</parameter>
+        <parameter id="invalidPixel" title="Default reflectance for invalid pixel" abstract="Default reflectance for invalid pixel (e.g. 0.0)" scope="test">0.0</parameter>
+        <parameter id="maskExpression" title="Mask expression for the MERIS whole view" abstract="Mask expression for the whole MERIS view" scope="test">not l1_flags.LAND_OCEAN</parameter>
+        <parameter id="surfPress" title="Surface pressure" abstract="Surface pressure (e.g. 1013.0)" scope="test">1013.0</parameter>
+        <parameter id="tauAero550" title="Aerosol optical depth" abstract="Aerosol optical depth (e.g. 0.2)" scope="runtime">0.2</parameter>
+        <parameter id="uH2o" title="Relative humidity" abstract="Relative humidity (e.g. 3.0)" scope="test">3.0</parameter>
+        <parameter id="uO3" title="Ozone content" abstract="Ozone content (e.g. 0.15)" scope="test">0.15</parameter>
+        <parameter id="useMerisADS" title="Use ECMWF data in the MERIS ADS" abstract="Flag to use ECMWF data in the MERIS ADS (true or false)" scope="runtime">true</parameter>
+        <parameter id="poi" title="List of POI for reflectance extraction" abstract="List of POIS (Name,Latitude,Longitude) separated by pipe (requires setting evaluate to true)" scope="runtime">BOUS,43.367,7.9|AAOT,45.314,12.508|MOBY,20.828,-157.193</parameter>
+        <parameter id="window" title="Side length of surrounding window (uneven)" abstract="The window size specifies the number of pixels surrounding the pixels derived from the given geographic coordinates to be exported">3</parameter>
+        <parameter id="aggregation" title="Pixel value aggregation method (with window size is larger than 1)" abstract="The aggregation methods that may be used are: min, exports the minimum of the window values; max, exports the maximum of the window values; mean, exports mean and standard deviation of the window values or median, exports the median of the window values">mean</parameter>
+        <parameter id="publishL2" title="Flag to trigger the publishing of Level 2 products generated" abstract="Flag to trigger the publishing of Level 2 products generated" scope="runtime">true</parameter>
+        <parameter id="pixex" title="Flag to extract POI reflectances" abstract="Flag to extract POI reflectances for evaluation purposes" scope="runtime">true</parameter>
       </defaultParameters>
       <defaultJobconf>
         <property id="mapred.task.timeout">9900000000000</property>
       </defaultJobconf>
     </jobTemplate>
-</jobTemplates>	
-<workflow id="participant-b" title="Participant B" abstract="Participant B uses the BEAM SmacOp atmospheric correction with ECMWF data">
+  </jobTemplates>
+  <workflow id="participant-b" title="Participant B" abstract="Participant B uses the BEAM SmacOp atmospheric correction with ECMWF data">
     <workflowVersion>1.0</workflowVersion>
-    <node id="node_smacop">				
-      <job id="SmacOp"></job>			
+    <node id="node_smacop">
+      <job id="SmacOp"/>
       <sources>
-        <source id="data" title="Data package URL" abstract="Data package OpenSearch Description Document URL" scope="runtime" refid="cas:series">https://challenges.esa.int/eceo/datapackage/RRPAR/description?key=9d79148d-3e17-414b-9983-e4cef9e88ec6</source>
+        <source id="source" abstract="Catalogue or Data package URL" title="Link to the Catalogue or Data package OpenSearch description document" scope="runtime" refid="cas:series">https://challenges.esa.int/eceo/datapackage/FRSPAR/description?key=495f181f-47d3-4668-b717-d36d4a560837</source>
+        <!--<source id="source" abstract="Catalogue or Data package URL" title="Link to the Catalogue or Data package OpenSearch description document" scope="runtime" refid="file:urls">/tmp/CONTEST3.list</source>-->
       </sources>
-      <parameters/>					
+      <parameters/>
     </node>
   </workflow>
 </application>
diff --git a/src/main/app-resources/pixex/libexec/PixEx.xml b/src/main/app-resources/pixex/libexec/PixEx.xml
@@ -0,0 +1,52 @@
+ <graph id="someGraphId">
+    <version>1.0</version>
+    <node id="someNodeId">
+      <operator>PixEx</operator>
+      <sources>
+        <sourceProducts>
+				<source>$variable</source>
+		</sourceProducts>
+      </sources>
+<parameters>
+    <sourceProductPaths>$variable_path/$variable</sourceProductPaths>
+    <exportBands>true</exportBands>
+    <exportTiePoints>false</exportTiePoints>
+    <exportMasks>false</exportMasks>
+    <coordinates>
+ <!--       <coordinate>
+            <name>Coord_0</name>
+            <latitude>0.0</latitude>
+            <longitude>-150.0</longitude>
+            <dateTime/>
+            <originalValues/>
+            <id>1</id>
+        </coordinate>
+			<coordinate>
+            <name>Coord_1</name>
+            <latitude>0.0</latitude>
+            <longitude>-160.0</longitude>
+            <dateTime/>
+            <originalValues/>
+            <id>2</id>
+        </coordinate>-->
+    </coordinates>
+    <timeDifference/>
+    <coordinatesFile>$coordinates</coordinatesFile>
+    <matchupFile/>
+    <windowSize>$windowSize</windowSize>
+    <outputDir>$output_path</outputDir>
+    <outputFilePrefix>$prefix</outputFilePrefix>
+    <expression>not l1_flags.LAND_OCEAN</expression>
+    <exportExpressionResult>false</exportExpressionResult>
+    <aggregatorStrategyType>$aggregatorStrategyType</aggregatorStrategyType>
+    <exportSubScenes>false</exportSubScenes>
+    <subSceneBorderSize>0</subSceneBorderSize>
+    <exportKmz>false</exportKmz>
+    <extractTimeFromFilename>false</extractTimeFromFilename>
+    <dateInterpretationPattern>yyyyMMdd</dateInterpretationPattern>
+    <filenameInterpretationPattern>*${startDate}*${endDate}*</filenameInterpretationPattern>
+    <includeOriginalInput>false</includeOriginalInput>
+    <scatterPlotVariableCombinations/>
+</parameters>
+</node>
+</graph>
diff --git a/src/main/app-resources/pixex/libexec/tidy.awk b/src/main/app-resources/pixex/libexec/tidy.awk
@@ -0,0 +1,3 @@
+NR >= 2 {print run","date","orbit","$0}; 
+
+NR ==1 { print "Run,Date,ORBIT," $0}
diff --git a/src/main/app-resources/smacop/etc/profile.rgb b/src/main/app-resources/smacop/etc/profile.rgb
@@ -0,0 +1,3 @@
+blue=not l1_flags.LAND_OCEAN ? log(0.05 + 0.21 * reflec_1 + 1.75 * reflec_2 + 0.47 * reflec_3 + 0.16 * reflec_4) : 0
+green=not l1_flags.LAND_OCEAN ? log(0.05 + 0.21 * reflec_3 + 0.50 * reflec_4 + reflec_5 + 0.38 * reflec_6) : 0
+red=not l1_flags.LAND_OCEAN ? log(0.05 + 0.35 * reflec_2 + 0.60 * reflec_5 + reflec_6 + 0.13 * reflec_7) : 0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,3 @@
		NR >= 2 {print run","date","orbit","$0};

		NR ==1 { print "Run,Date,ORBIT," $0}