diff --git a/README.md b/README.md old mode 100644 new mode 100755 index 29931a4..2cb70cd --- a/README.md +++ b/README.md @@ -1,11 +1,17 @@ -beam-meris-ac +Participant-B ============= MERIS atmospheric correction effects using BEAM - using SmacOp -### Installation +### Quick link -Log on the developer sandbox and run these commands in a shell: +* [Installation](#installation) +* [Submitting the workflow](#submit) +* [Test the application](#test) + +### Installation + +Log on the Developer Cloud Sandbox and run these commands in a shell: * Install **Java 7** @@ -34,23 +40,32 @@ Enter to keep the current selection[+], or type selection number: Select java 1.7 out of the menu options by typing the correct number (here it's *3*). -* Install this application +* Install the application from github ```bash cd -git clone git@github.com:ocean-color-ac-challenge/beam-meris-ac.git -cd beam-meris-ac +git clone git@github.com:ocean-color-ac-challenge/Participant-B.git +cd Participant-B mvn install ``` -### Submitting the workflow +* Install the application from rpm + +```bash +curl -L -O https://github.com/ocean-color-ac-challenge/Participant-B/releases/download/v0.35/Participant-B-x.xx-ciop.noarch.rpm +sudo yum -y install Participant-B-x.xx-ciop.noarch.rpm +``` + +where x.xx is the version of the RPM created -* Via the Sandbox shell +### Submitting the workflow + +* Via the Sandbox shell Run this command in a shell: ```bash -ciop-simwf +ciop-run ``` * Via the Web Processing Service dashboard tab @@ -59,8 +74,149 @@ Use your browser to go to the Sandbox dashboard tab at the address http://Test the application + +##### Test Participant-B-01 + +* Test Procedure + +Invoke the application via the Dashboard with the parameters listed in the test inputs specification + +* Inputs specification + +| Parameter | Value | +|-------------|---------------------------------------------------------------------------------------------------------| +| Catalogue | https://challenges.esa.int/eceo/datapackage/RRPAR/description?key=9d79148d-3e17-414b-9983-e4cef9e88ec6 | +| Start date | 2002-03-01T00:00:00Z | +| End date | 2012-05-09T23:59:59Z | +| aerosolType | CONTINENTAL | +| Aerosol Optical Depth | 0.2 | +| Use ECMWF data in the MERIS ADS | true | +| List of POI for reflectance extraction |BOUS,43.367,7.9\|AAOT,45.314,12.508\|MOBY,20.828,-157.193 | +| Flag to trigger the publishing of Level 2 products generated | true | +| Flag to extract POI reflectances | false | + +* Outputs specification + +| Output | Size | +|--------------------------------------------------------------------|------| +MER_RR__1PNACR20060730_093546_000021432049_00480_23079_0000.N1.png|10.01 MB| +MER_RR__1PNACR20060730_093546_000021432049_00480_23079_0000.N1.tgz|193.73 MB| +MER_RR__1PRACR20030306_201313_000026082014_00243_05307_0000.N1.png|21.35 MB| +MER_RR__1PRACR20030306_201313_000026082014_00243_05307_0000.N1.tgz|367.81 MB| +MER_RR__1PRACR20040903_201756_000026282030_00057_13137_0000.N1.png|21.87 MB| +MER_RR__1PRACR20040903_201756_000026282030_00057_13137_0000.N1.tgz|367.88 MB| +MER_RR__1PRACR20050504_093419_000026382037_00022_16609_0000.N1.png|14.73 MB| +MER_RR__1PRACR20050504_093419_000026382037_00022_16609_0000.N1.tgz|265.6 MB| +MER_RR__1PRACR20070920_092716_000026302061_00437_29048_0000.N1.png|14.86 MB| +MER_RR__1PRACR20070920_092716_000026302061_00437_29048_0000.N1.tgz|266 MB| +MER_RR__1PRACR20080816_092020_000026302071_00165_33786_0000.N1.png|16.58 MB| +MER_RR__1PRACR20080816_092020_000026302071_00165_33786_0000.N1.tgz|267.37 MB| + +* Test pass/fail criteria + +All products listed in test outputs specification are generated + +##### Test Participant-B-02 + +* Test Procedure + +Invoke the application via the Dashboard with the parameters listed in the test inputs specification + +* Inputs specification + +| Parameter | Value | +|-------------|---------------------------------------------------------------------------------------------------------| +| Catalogue | https://challenges.esa.int/eceo/datapackage/FRSPAR/description?key=495f181f-47d3-4668-b717-d36d4a560837 | +| Start date | 2002-03-01T00:00:00Z | +| End date | 2012-05-09T23:59:59Z | +| aerosolType | CONTINENTAL | +| Aerosol Optical Depth | 0.2 | +| Use ECMWF data in the MERIS ADS | true | +| List of POI for reflectance extraction |BOUS,43.367,7.9\|AAOT,45.314,12.508\|MOBY,20.828,-157.193 | +| Flag to trigger the publishing of Level 2 products generated | true | +| Flag to extract POI reflectances | false | + +* Outputs specification + +| Output | Size | +|--------------------------------------------------------------------|------| +MER_FRS_1PPEPA20040711_020449_000002422028_00275_12353_1787.N1.png|21.16 MB| +MER_FRS_1PPEPA20040711_020449_000002422028_00275_12353_1787.N1.tgz|466.86 MB| +MER_FRS_1PPEPA20040717_021529_000002642028_00361_12439_1968.N1.png|18.27 MB| +MER_FRS_1PPEPA20040717_021529_000002642028_00361_12439_1968.N1.tgz|371.85 MB| + +* Test pass/fail criteria + +All products listed in test outputs specification are generated + +##### Test Participant-B-03 + +* Test Procedure + +Invoke the application via the Dashboard with the parameters listed in the test inputs specification + +* Inputs specification + +| Parameter | Value | +|-------------|---------------------------------------------------------------------------------------------------------| +| Catalogue | https://challenges.esa.int/eceo/datapackage/RRPAR/description?key=9d79148d-3e17-414b-9983-e4cef9e88ec6 | +| Start date | 2002-03-01T00:00:00Z | +| End date | 2012-05-09T23:59:59Z | +| aerosolType | CONTINENTAL | +| Aerosol Optical Depth | 0.2 | +| Use ECMWF data in the MERIS ADS | true | +| List of POI for reflectance extraction |BOUS,43.367,7.9\|AAOT,45.314,12.508\|MOBY,20.828,-157.193 | +| Flag to trigger the publishing of Level 2 products generated | false | +| Flag to extract POI reflectances | true | + +* Outputs specification + +| Output | Size | +|--------------------------------------------------------------------|------| +MER_RR__1PNACR20060730_093546_000021432049_00480_23079_0000.N1.txt|1.08 KB| +MER_RR__1PRACR20030306_201313_000026082014_00243_05307_0000.N1.txt|0.78 KB| +MER_RR__1PRACR20040903_201756_000026282030_00057_13137_0000.N1.txt|0.78 KB| +MER_RR__1PRACR20050504_093419_000026382037_00022_16609_0000.N1.txt|1.06 KB| +MER_RR__1PRACR20070920_092716_000026302061_00437_29048_0000.N1.txt|1.06 KB| +MER_RR__1PRACR20080816_092020_000026302071_00165_33786_0000.N1.txt|1.05 KB| + +* Test pass/fail criteria + +All products listed in test outputs specification are generated + + +##### Test Participant-B-04 + +* Test Procedure + +Invoke the application via the Dashboard with the parameters listed in the test inputs specification + +* Inputs specification + +| Parameter | Value | +|-------------|---------------------------------------------------------------------------------------------------------| +| Catalogue | https://challenges.esa.int/eceo/datapackage/FRSPAR/description?key=495f181f-47d3-4668-b717-d36d4a560837 | +| Start date | 2002-03-01T00:00:00Z | +| End date | 2012-05-09T23:59:59Z | +| aerosolType | CONTINENTAL | +| Aerosol Optical Depth | 0.2 | +| Use ECMWF data in the MERIS ADS | true | +| List of POI for reflectance extraction |CHINA,27,122| +| Flag to trigger the publishing of Level 2 products generated | false | +| Flag to extract POI reflectances | true | + +* Outputs specification -Fill the parameters and click submit. +| Output | Size | +|--------------------------------------------------------------------|------| +MER_FRS_1PPEPA20040711_020449_000002422028_00275_12353_1787.N1.txt|0.77 KB| +MER_FRS_1PPEPA20040717_021529_000002642028_00361_12439_1968.N1.txt|0.77 KB| +* Test pass/fail criteria +All products listed in test outputs specification are generated diff --git a/pom.xml b/pom.xml index 6262e0c..882c809 100644 --- a/pom.xml +++ b/pom.xml @@ -2,15 +2,15 @@ 4.0.0 com.terradue.ciop.app - beam-meris-ac + Participant-B rpm - beam-meris-ac + ${project.artifactId} BEAM Atmospheric Correction Processing 0.2-SNAPSHOT - scm:git:git@github.com:ocean-color-ac-challenge/beam-meris-ac.git - scm:git:git@github.com:ocean-color-ac-challenge/beam-meris-ac.git - https://github.com/ocean-color-ac-challenge/beam-meris-ac + scm:git:git@github.com:ocean-color-ac-challenge/${project.artifactId}.git + scm:git:git@github.com:ocean-color-ac-challenge/${project.artifactId}.git + https://github.com/ocean-color-ac-challenge/${project.artifactId} HEAD @@ -44,6 +44,11 @@ beam-meris-smac 5.0 + + org.esa.beam + beam-pixel-extraction + 5.0 + ncsa.hdf lib-hdf @@ -129,8 +134,8 @@ def writer = new FileWriter(new File("/application/properties.xml")) def xml = new MarkupBuilder(writer) xml.mkp.xmlDeclaration(version: "1.0", encoding: "utf-8") xml.properties() { -github_repository(version:pom.version, pom.artifactId) -user_reference(env['USER']) + github_repository(version:pom.version, pom.artifactId) + user_reference(env['USER']) } @@ -263,8 +268,8 @@ user_reference(env['USER']) Terradue ${project.inceptionYear} air Terradue - ESA BEAM Toolbox atmospheric correction - + ${project.artifactId} + ${project.artifactId} ${project.artifactId} 775 664 @@ -355,7 +360,7 @@ user_reference(env['USER']) de.jutzig github-release-plugin - 1.0.3 + 1.1.1-dns github-upload diff --git a/src/main/app-resources/application.xml b/src/main/app-resources/application.xml index 06c3f9e..b8e3959 100644 --- a/src/main/app-resources/application.xml +++ b/src/main/app-resources/application.xml @@ -1,37 +1,42 @@ - /application/smacop/run.sh - + BEAM-DIMAP - 2012-04-05 - 2012-04-06 - CONTINENTAL - radiance_1,radiance_2,radiance_3,radiance_4,radiance_5,radiance_6,radiance_7 - 0.3 - true - 0.0 - not l1_flags.LAND_OCEAN - 1013.0 - 3.0 - 0.15 + 2002-04-05 + 2015-01-06 + CONTINENTAL + radiance_1,radiance_2,radiance_3,radiance_4,radiance_5,radiance_6,radiance_7 + 0.0 + not l1_flags.LAND_OCEAN + 1013.0 + 0.2 + 3.0 + 0.15 + true + BOUS,43.367,7.9|AAOT,45.314,12.508|MOBY,20.828,-157.193 + 3 + mean + true + true 9900000000000 - - + + 1.0 - - + + - https://challenges.esa.int/eceo/datapackage/RRPAR/description?key=9d79148d-3e17-414b-9983-e4cef9e88ec6 + https://challenges.esa.int/eceo/datapackage/FRSPAR/description?key=495f181f-47d3-4668-b717-d36d4a560837 + - + diff --git a/src/main/app-resources/pixex/libexec/PixEx.xml b/src/main/app-resources/pixex/libexec/PixEx.xml new file mode 100644 index 0000000..e611fef --- /dev/null +++ b/src/main/app-resources/pixex/libexec/PixEx.xml @@ -0,0 +1,52 @@ + + 1.0 + + PixEx + + + $variable + + + + $variable_path/$variable + true + false + false + + + + + $coordinates + + $windowSize + $output_path + $prefix + not l1_flags.LAND_OCEAN + false + $aggregatorStrategyType + false + 0 + false + false + yyyyMMdd + *${startDate}*${endDate}* + false + + + + diff --git a/src/main/app-resources/pixex/libexec/tidy.awk b/src/main/app-resources/pixex/libexec/tidy.awk new file mode 100644 index 0000000..719b4a2 --- /dev/null +++ b/src/main/app-resources/pixex/libexec/tidy.awk @@ -0,0 +1,3 @@ +NR >= 2 {print run","date","orbit","$0}; + +NR ==1 { print "Run,Date,ORBIT," $0} diff --git a/src/main/app-resources/smacop/etc/profile.rgb b/src/main/app-resources/smacop/etc/profile.rgb new file mode 100644 index 0000000..d0ed307 --- /dev/null +++ b/src/main/app-resources/smacop/etc/profile.rgb @@ -0,0 +1,3 @@ +blue=not l1_flags.LAND_OCEAN ? log(0.05 + 0.21 * reflec_1 + 1.75 * reflec_2 + 0.47 * reflec_3 + 0.16 * reflec_4) : 0 +green=not l1_flags.LAND_OCEAN ? log(0.05 + 0.21 * reflec_3 + 0.50 * reflec_4 + reflec_5 + 0.38 * reflec_6) : 0 +red=not l1_flags.LAND_OCEAN ? log(0.05 + 0.35 * reflec_2 + 0.60 * reflec_5 + reflec_6 + 0.13 * reflec_7) : 0 diff --git a/src/main/app-resources/smacop/run.sh b/src/main/app-resources/smacop/run.sh index 865c339..c9aa844 100644 --- a/src/main/app-resources/smacop/run.sh +++ b/src/main/app-resources/smacop/run.sh @@ -8,83 +8,155 @@ SUCCESS=0 ERR_NOINPUT=1 ERR_BEAM=2 ERR_NOPARAMS=5 +ERR_JAVAVERSION=10 # add a trap to exit gracefully function cleanExit () { local retval=$? local msg="" - case "$retval" in - $SUCCESS) msg="Processing successfully concluded";; - $ERR_NOPARAMS) msg="Expression not defined";; - $ERR_BEAM) msg="Beam failed to process product $product (Java returned $res).";; + case "${retval}" in + ${SUCCESS}) msg="Processing successfully concluded";; + ${ERR_NOPARAMS}) msg="Expression not defined";; + ${ERR_BEAM}) msg="Beam failed to process product $product (Java returned $res).";; + ${ERR_JAVAVERSION}) msg="The version of the JVM must be at least 1.7";; *) msg="Unknown error";; esac - [ "$retval" != "0" ] && ciop-log "ERROR" "Error $retval - $msg, processing aborted" || ciop-log "INFO" "$msg" - exit $retval + [ "${retval}" != "0" ] && ciop-log "ERROR" "Error ${retval} - ${msg}, processing aborted" || ciop-log "INFO" "${msg}" + exit ${retval} } trap cleanExit EXIT +ciop-log "INFO" "Checking Java version" +${_CIOP_APPLICATION_PATH}/shared/bin/detect_java.sh || exit ${ERR_JAVAVERSION} + # create the output folder to store the output products -mkdir -p $TMPDIR/output -export OUTPUTDIR=$TMPDIR/output -format="`ciop-getparam format`" +mkdir -p ${TMPDIR}/output +export OUTPUTDIR=${TMPDIR}/output +format="$( ciop-getparam format )" + +[ "${format}" != "BEAM-DIMAP" ] && [ "${format}" != "GeoTIFF" ] && exit ${ERR_FORMAT} + +aerosolType="$( ciop-getparam aerosolType )" +bandNames="$( ciop-getparam bandNames )" +invalidPixel="$( ciop-getparam invalidPixel )" +maskExpression="$( ciop-getparam maskExpression )" +surfPress="$( ciop-getparam surfPress )" +tauAero550="$( ciop-getparam tauAero550 )" +uH2o="$( ciop-getparam uH2o )" +uO3="$( ciop-getparam uO3 )" +useMerisADS="$( ciop-getparam useMerisADS )" -[ "$format" != "BEAM-DIMAP" ] && [ "$format" != "GeoTIFF" ] && exit $ERR_FORMAT +# Evaluation parameters +# get the POIs +echo -e "Name\tLatitude\tLongitude" > $TMPDIR/poi.csv +echo "$( ciop-getparam poi | tr ',' '\t' | tr '|' '\n' )" >> $TMPDIR/poi.csv +# get the window size +window="$( ciop-getparam window )" +# get the aggregation +aggregation="$( ciop-getparam aggregation )" -aerosolType="`ciop-getparam aerosolType`" -bandNames="`ciop-getparam bandNames`" -invalidPixel="`ciop-getparam invalidPixel`" -maskExpression="`ciop-getparam maskExpression`" -surfPress="`ciop-getparam surfPress`" -tauAero550="`ciop-getparam tauAero550`" -uH2o="`ciop-getparam uH2o`" -uO3="`ciop-getparam uO3`" -useMerisADS="`ciop-getparam useMerisADS`" +# set the processing flags +pixex="$( ciop-getparam pixex )" +publish_l2="$( ciop-getparam publishL2 )" +prd_counter=1 # loop and process all MERIS products while read inputfile do # report activity in log - ciop-log "INFO" "Retrieving $inputfile from storage" + ciop-log "INFO" "Retrieving input ${prd_counter} - ${inputfile} from storage" - # retrieve the remote geotiff product to the local temporary folder - retrieved="`opensearch-client "$inputfile" enclosure | ciop-copy -o $TMPDIR -`" + prd_ref="$( opensearch-client "${inputfile}" enclosure )" + [ -z "${prd_ref}" ] && { + ciop-log "ERR" "Could not resolve ${inputfile}" + } || { + + # retrieve the MERIS product product to the local temporary folder + retrieved="$( echo ${prd_ref} | ciop-copy -o $TMPDIR - )" - # check if the file was retrieved - [ "$?" == "0" -a -e "$retrieved" ] || exit $ERR_NOINPUT + # check if the file was retrieved + [ "$?" == "0" -a -e "${retrieved}" ] || exit ${ERR_NOINPUT} - # report activity - ciop-log "INFO" "Retrieved `basename $retrieved`, moving on to smac operator" + # report activity + ciop-log "INFO" "Retrieved $( basename ${retrieved} ), moving on to smac operator" -outputname=`basename $retrieved` - - $_CIOP_APPLICATION_PATH/shared/bin/gpt.sh SmacOp \ - -SsourceProduct=$retrieved \ - -f $format \ - -t $OUTPUTDIR/$outputname \ - -PaerosolType=$aerosolType \ - -PbandNames="$bandNames" \ - -PinvalidPixel=$invalidPixel \ - -PmaskExpression="$maskExpression" \ - -PsurfPress=$surfPress \ - -PtauAero550=$tauAero550 \ - -PuH2o=$uH2o \ - -PuO3=$uO3 \ - -PuseMerisADS=$useMerisADS - - res=$? - [ $res != 0 ] && exit $ERR_BEAM + outputname="$( basename ${retrieved} )" - tar -C $OUTPUTDIR -cvzf $TMPDIR/$outputname.tgz $outputname.dim $outputname.data + ${_CIOP_APPLICATION_PATH}/shared/bin/gpt.sh SmacOp \ + -SsourceProduct=${retrieved} \ + -f ${format} \ + -t ${OUTPUTDIR}/${outputname} \ + -PaerosolType=${aerosolType} \ + -PbandNames="${bandNames}" \ + -PinvalidPixel=${invalidPixel} \ + -PmaskExpression="${maskExpression}" \ + -PsurfPress=${surfPress} \ + -PtauAero550=${tauAero550} \ + -PuH2o=${uH2o} \ + -PuO3=${uO3} \ + -PuseMerisADS=${useMerisADS} - ciop-log "INFO" "Publishing $outputname.tgz" - ciop-publish -m $TMPDIR/$outputname.tgz - - # cleanup - rm -fr $retrieved $OUTPUTDIR/$outputname.d* $TMPDIR/$outputname.tgz + res=$? + [ ${res} != 0 ] && exit ${ERR_BEAM} + + run=${CIOP_WF_RUN_ID} + + [ "${pixex}" == "true" ] && { + # invoke pixex + l2b="$( basename ${OUTPUTDIR}/${outputname} )" + prddate="${l2b:20:2}/${l2b:18:2}/${l2b:14:4}" + ciop-log "INFO" "Apply BEAM PixEx Operator to ${l2b}" + prd_orbit=$( echo ${l2b:49:5} | sed 's/^0*//' ) + + # apply PixEx BEAM operator + ${_CIOP_APPLICATION_PATH}/shared/bin/gpt.sh \ + -Pvariable=${outputname}.dim \ + -Pvariable_path=${OUTPUTDIR} \ + -Poutput_path=${OUTPUTDIR} \ + -Pprefix=${run} \ + -Pcoordinates=${TMPDIR}/poi.csv \ + -PwindowSize=${window} \ + -PaggregatorStrategyType="${aggregation}" \ + ${_CIOP_APPLICATION_PATH}/pixex/libexec/PixEx.xml 1>&2 + + res=$? + [ ${res} != 0 ] && exit ${ERR_BEAM_PIXEX} + + result="$( find ${OUTPUTDIR} -name "${run}*measurements.txt" )" + + [ -n "${result}" ] && { + skip_lines=$( cat ${result} | grep -n "ProdID" | cut -d ":" -f 1 ) + + cat "${result}" | tail -n +${skip_lines} | tr "\t" "," | awk -f ${_CIOP_APPLICATION_PATH}/pixex/libexec/tidy.awk -v run=${run} -v date=${prddate} -v orbit=${prd_orbit} - > "${OUTPUTDIR}/${l2b}.txt" + + ciop-log "INFO" "Publishing extracted pixel values" + ciop-publish -m "${OUTPUTDIR}/${l2b}.txt" + rm -f "${OUTPUTDIR}/${l2b}.txt" + } + } + + [ "${publish_l2}" == "true" ] && { + # create RGB quicklook + ${_CIOP_APPLICATION_PATH}/shared/bin/pconvert.sh \ + -f png \ + -p ${_CIOP_APPLICATION_PATH}/smacop/etc/profile.rgb \ + -o ${OUTPUTDIR} \ + ${OUTPUTDIR}/${outputname}.dim + tar -C ${OUTPUTDIR} -cvzf ${TMPDIR}/${outputname}.tgz ${outputname}.dim ${outputname}.data + ciop-log "INFO" "Publishing $outputname.tgz" + ciop-publish -m ${TMPDIR}/${outputname}.tgz + ciop-publish -m ${OUTPUTDIR}/${outputname}.png + rm -fr ${OUTPUTDIR}/${outputname}.tgz + rm -f ${OUTPUTDIR}/${outputname}.png + } + # cleanup + rm -fr ${retrieved} ${OUTPUTDIR}/${outputname}.d* + + prd_counter=$(( ${prd_counter} + 1 )) + } done exit 0