From 40c9c89315a9d9776915ca9228e5bb1176580bdd Mon Sep 17 00:00:00 2001 From: Marek Horst Date: Tue, 10 Oct 2023 16:09:23 +0200 Subject: [PATCH] Closes #1426: Run IIS experiments by relying on spark 3.4 version WIP. Fixing the changed results order in patent and software entity exporter integration tests. Introducing required fixes for various `iis-wf-export-actionmanager` exporters relying on spark3 to let their integration tests to succeed: * setting `spark.extraListeners` and `spark.sql.queryExecutionListeners` explicitly to empty values in order to avoid relying on incompatible, spark2 compliant, cloudera listeners * setting `spark.shuffle.useOldFetchProtocol=true` in order to address `2.4 to 3.0 migration guide` requirement regarding protocol for fetching shuffle blocks backward compatibility (and avoiding `IllegalArgumentException: Unexpected message type: ` kind of errors) The following modules were covered with similar workflow.xml related changes but their spark3 compatibility was not fully tested yet: * `iis-wf-affmatching` * `iis-wf-citationmatching-direct` * `iis-wf-citationmatching` * `iis-wf-documentsclassification` * `iis-wf-import` (`content_url/core_parquet`, `infospace`, `patent`) * `iis-wf-referenceextraction` (`community`, `concept`, `covid19`, `patent`, `project/funder_report`, `researchinitiative`, `softwareurl`) * `iis-wf-transformers` (`avro2json`) --- .../wf/affmatching/dedup/oozie_app/workflow.xml | 2 +- .../wf/affmatching/main/oozie_app/workflow.xml | 2 +- .../projectbased/oozie_app/workflow.xml | 2 +- .../direct/oozie_app/workflow.xml | 2 +- .../fuzzy/oozie_app/workflow.xml | 6 +++--- .../oozie_app/workflow.xml | 2 +- .../entity/patent/oozie_app/workflow.xml | 17 ++++------------- .../entity/software/oozie_app/workflow.xml | 17 ++++------------- .../patent/default/oozie_app/workflow.xml | 10 +++++----- .../software/default/oozie_app/workflow.xml | 4 ++-- .../software/heritage/oozie_app/workflow.xml | 4 ++-- .../core_parquet/oozie_app/workflow.xml | 2 +- .../importer/infospace/oozie_app/workflow.xml | 2 +- .../wf/importer/patent/oozie_app/workflow.xml | 2 +- .../community/main/oozie_app/workflow.xml | 2 +- .../oozie_app/workflow.xml | 2 +- .../covid19/main/oozie_app/workflow.xml | 4 ++-- .../patent/main/oozie_app/workflow.xml | 8 ++++---- .../funder_report/oozie_app/workflow.xml | 2 +- .../main/oozie_app/workflow.xml | 4 ++-- .../softwareurl/main/oozie_app/workflow.xml | 2 +- .../avro2json/oozie_app/workflow.xml | 2 +- 22 files changed, 41 insertions(+), 59 deletions(-) diff --git a/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/dedup/oozie_app/workflow.xml b/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/dedup/oozie_app/workflow.xml index a854b108a..340432d74 100644 --- a/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/dedup/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/dedup/oozie_app/workflow.xml @@ -84,7 +84,7 @@ - yarn-cluster + yarn cluster document-organization deduplication eu.dnetlib.iis.wf.affmatching.AffMatchingDedupJob diff --git a/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/main/oozie_app/workflow.xml b/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/main/oozie_app/workflow.xml index 17b9be173..063379c11 100644 --- a/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/main/oozie_app/workflow.xml @@ -122,7 +122,7 @@ - yarn-cluster + yarn cluster Affiliation Matching eu.dnetlib.iis.wf.affmatching.AffMatchingJob diff --git a/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/projectbased/oozie_app/workflow.xml b/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/projectbased/oozie_app/workflow.xml index 958a9d7dd..713686743 100644 --- a/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/projectbased/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-affmatching/src/main/resources/eu/dnetlib/iis/wf/affmatching/projectbased/oozie_app/workflow.xml @@ -105,7 +105,7 @@ - yarn-cluster + yarn cluster Project based Document-Organization Matching eu.dnetlib.iis.wf.affmatching.ProjectBasedMatchingJob diff --git a/iis-wf/iis-wf-citationmatching-direct/src/main/resources/eu/dnetlib/iis/wf/citationmatching/direct/oozie_app/workflow.xml b/iis-wf/iis-wf-citationmatching-direct/src/main/resources/eu/dnetlib/iis/wf/citationmatching/direct/oozie_app/workflow.xml index e55ea0cfe..2713ff4e4 100644 --- a/iis-wf/iis-wf-citationmatching-direct/src/main/resources/eu/dnetlib/iis/wf/citationmatching/direct/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-citationmatching-direct/src/main/resources/eu/dnetlib/iis/wf/citationmatching/direct/oozie_app/workflow.xml @@ -87,7 +87,7 @@ - yarn-cluster + yarn cluster citationmatching_direct eu.dnetlib.iis.wf.citationmatching.direct.CitationMatchingDirectJob diff --git a/iis-wf/iis-wf-citationmatching/src/main/resources/eu/dnetlib/iis/wf/citationmatching/fuzzy/oozie_app/workflow.xml b/iis-wf/iis-wf-citationmatching/src/main/resources/eu/dnetlib/iis/wf/citationmatching/fuzzy/oozie_app/workflow.xml index a8ab9ac0d..4925c45e9 100644 --- a/iis-wf/iis-wf-citationmatching/src/main/resources/eu/dnetlib/iis/wf/citationmatching/fuzzy/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-citationmatching/src/main/resources/eu/dnetlib/iis/wf/citationmatching/fuzzy/oozie_app/workflow.xml @@ -103,7 +103,7 @@ - yarn-cluster + yarn cluster citation-matching-input-transformer @@ -132,7 +132,7 @@ - yarn-cluster + yarn cluster citation-matching @@ -165,7 +165,7 @@ - yarn-cluster + yarn cluster citation-matching-output-transformer diff --git a/iis-wf/iis-wf-documentsclassification/src/main/resources/eu/dnetlib/iis/wf/documentsclassification/oozie_app/workflow.xml b/iis-wf/iis-wf-documentsclassification/src/main/resources/eu/dnetlib/iis/wf/documentsclassification/oozie_app/workflow.xml index 249c326ab..47657e3d9 100644 --- a/iis-wf/iis-wf-documentsclassification/src/main/resources/eu/dnetlib/iis/wf/documentsclassification/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-documentsclassification/src/main/resources/eu/dnetlib/iis/wf/documentsclassification/oozie_app/workflow.xml @@ -86,7 +86,7 @@ - yarn-cluster + yarn cluster diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/oozie_app/workflow.xml index ab83f89ca..cf9afaf91 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/oozie_app/workflow.xml @@ -90,16 +90,6 @@ oozieActionShareLibForSpark2 oozie action sharelib for spark 2.* - - spark2ExtraListeners - com.cloudera.spark.lineage.NavigatorAppListener - spark 2.* extra listeners classname - - - spark2SqlQueryExecutionListeners - com.cloudera.spark.lineage.NavigatorQueryListener - spark 2.* sql query execution listeners classname - spark2YarnHistoryServerAddress spark 2.* yarn history server address @@ -133,7 +123,7 @@ - yarn-cluster + yarn cluster patent spark exporter eu.dnetlib.iis.wf.export.actionmanager.entity.patent.PatentExporterJob @@ -142,8 +132,9 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.extraListeners= + --conf spark.sql.queryExecutionListeners= + --conf spark.shuffle.useOldFetchProtocol=true --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} diff --git a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/oozie_app/workflow.xml index 9cb163a4d..fd8fcfb7a 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/main/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/oozie_app/workflow.xml @@ -59,16 +59,6 @@ oozieActionShareLibForSpark2 oozie action sharelib for spark 2.* - - spark2ExtraListeners - com.cloudera.spark.lineage.NavigatorAppListener - spark 2.* extra listeners classname - - - spark2SqlQueryExecutionListeners - com.cloudera.spark.lineage.NavigatorQueryListener - spark 2.* sql query execution listeners classname - spark2YarnHistoryServerAddress spark 2.* yarn history server address @@ -103,7 +93,7 @@ - yarn-cluster + yarn cluster software spark exporter eu.dnetlib.iis.wf.export.actionmanager.entity.software.SoftwareExporterJob @@ -113,8 +103,9 @@ --executor-memory=${sparkExecutorMemory} --executor-cores=${sparkExecutorCores} --driver-memory=${sparkDriverMemory} - --conf spark.extraListeners=${spark2ExtraListeners} - --conf spark.sql.queryExecutionListeners=${spark2SqlQueryExecutionListeners} + --conf spark.extraListeners= + --conf spark.sql.queryExecutionListeners= + --conf spark.shuffle.useOldFetchProtocol=true --conf spark.yarn.historyServer.address=${spark2YarnHistoryServerAddress} --conf spark.eventLog.dir=${nameNode}${spark2EventLogDir} diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml index 115d356e8..e5d7376ff 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/oozie_app/workflow.xml @@ -101,12 +101,12 @@ eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer -Iseqfile=${workingDir}/output/document_patent/patent-actionset-id - -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent2.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2_to_document2.properties, + -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2_to_document2.properties, /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document1_to_patent1.properties, /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document1.properties, /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent1.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document2.properties + /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1_to_document2.properties, + /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/document2_to_patent2.properties @@ -119,8 +119,8 @@ eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer -Iseqfile=${workingDir}/output/entities_patent/patent-entity-actionset-id - -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1.expectations, - /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2.expectations + -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent2.expectations, + /eu/dnetlib/iis/wf/export/actionmanager/entity/patent/default/output/patent1.expectations diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/oozie_app/workflow.xml index 48b538aa4..4b398f786 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/oozie_app/workflow.xml @@ -103,8 +103,8 @@ eu.dnetlib.iis.common.java.ProcessWrapper eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer -Iseqfile=${workingDir}/output/document_software_url/software-actionset-id - -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/output/document_to_software.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/output/software_to_document.properties + -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/output/software_to_document.properties, + /eu/dnetlib/iis/wf/export/actionmanager/entity/software/default/output/document_to_software.properties diff --git a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/oozie_app/workflow.xml b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/oozie_app/workflow.xml index c1bb3e41e..55f680c6b 100644 --- a/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-export-actionmanager/src/test/resources/eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/oozie_app/workflow.xml @@ -103,8 +103,8 @@ eu.dnetlib.iis.common.java.ProcessWrapper eu.dnetlib.iis.wf.export.actionmanager.sequencefile.TestingConsumer -Iseqfile=${workingDir}/output/document_software_url/software-actionset-id - -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/output/document_to_software.properties, - /eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/output/software_to_document.properties + -Pexpectation_file_paths=/eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/output/software_to_document.properties, + /eu/dnetlib/iis/wf/export/actionmanager/entity/software/heritage/output/document_to_software.properties diff --git a/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/content_url/core_parquet/oozie_app/workflow.xml b/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/content_url/core_parquet/oozie_app/workflow.xml index d60bb44e9..0805a45ab 100644 --- a/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/content_url/core_parquet/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/content_url/core_parquet/oozie_app/workflow.xml @@ -82,7 +82,7 @@ - yarn-cluster + yarn cluster import-content-url_core eu.dnetlib.iis.wf.importer.content.HiveBasedDocumentContentUrlImporterJob diff --git a/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/infospace/oozie_app/workflow.xml b/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/infospace/oozie_app/workflow.xml index b8e1ac623..299984373 100644 --- a/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/infospace/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/infospace/oozie_app/workflow.xml @@ -178,7 +178,7 @@ - yarn-cluster + yarn cluster import-infospace-hdp eu.dnetlib.iis.wf.importer.infospace.ImportInformationSpaceJob diff --git a/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/patent/oozie_app/workflow.xml b/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/patent/oozie_app/workflow.xml index aedd6f9c5..6a91a7096 100644 --- a/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/patent/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-import/src/main/resources/eu/dnetlib/iis/wf/importer/patent/oozie_app/workflow.xml @@ -77,7 +77,7 @@ - yarn-cluster + yarn cluster import-patent eu.dnetlib.iis.wf.importer.patent.PatentReaderJob diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/community/main/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/community/main/oozie_app/workflow.xml index a09a29142..3a57797df 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/community/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/community/main/oozie_app/workflow.xml @@ -62,7 +62,7 @@ - yarn-cluster + yarn cluster community-referenceextraction-input-transformer diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/concept/root_conceptid_report/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/concept/root_conceptid_report/oozie_app/workflow.xml index fa3b3fe45..075b49295 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/concept/root_conceptid_report/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/concept/root_conceptid_report/oozie_app/workflow.xml @@ -67,7 +67,7 @@ - yarn-cluster + yarn cluster referenceextraction_root_conceptid_report eu.dnetlib.iis.wf.referenceextraction.concept.RootConceptIdReportJob diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/covid19/main/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/covid19/main/oozie_app/workflow.xml index e560826bd..caf32aa40 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/covid19/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/covid19/main/oozie_app/workflow.xml @@ -93,7 +93,7 @@ ${oozieActionShareLibForSpark2} - yarn-cluster + yarn cluster covid19-referenceextraction-input-transformer eu.dnetlib.iis.wf.referenceextraction.covid19.input.Covid19ReferenceExtractionInputTransformerJob @@ -185,7 +185,7 @@ ${oozieActionShareLibForSpark2} - yarn-cluster + yarn cluster covid19-referenceextraction-output-transformer eu.dnetlib.iis.wf.referenceextraction.covid19.output.Covid19ToConceptIdTransformerJob diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml index 040bb65a9..dcfe52dd0 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/patent/main/oozie_app/workflow.xml @@ -160,7 +160,7 @@ - yarn-cluster + yarn cluster patent-referenceextraction-input-transformer eu.dnetlib.iis.wf.referenceextraction.patent.input.PatentReferenceExtractionInputTransformerJob @@ -225,7 +225,7 @@ - yarn-cluster + yarn cluster patent-metadata-retriever-input-transformer eu.dnetlib.iis.wf.referenceextraction.patent.input.PatentMetadataRetrieverInputTransformerJob @@ -249,7 +249,7 @@ - yarn-cluster + yarn cluster patent-metadata-retriever eu.dnetlib.iis.wf.referenceextraction.patent.PatentMetadataRetrieverJob @@ -292,7 +292,7 @@ - yarn-cluster + yarn cluster patent-metadata-extractor eu.dnetlib.iis.wf.referenceextraction.patent.PatentMetadataExtractorJob diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/project/funder_report/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/project/funder_report/oozie_app/workflow.xml index b8d22bb38..ef1073550 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/project/funder_report/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/project/funder_report/oozie_app/workflow.xml @@ -76,7 +76,7 @@ ${oozieActionShareLibForSpark2} - yarn-cluster + yarn cluster referenceextraction_project_funding_report eu.dnetlib.iis.wf.referenceextraction.project.ProjectFunderReportJob diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/researchinitiative/main/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/researchinitiative/main/oozie_app/workflow.xml index d729cce50..a44334674 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/researchinitiative/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/researchinitiative/main/oozie_app/workflow.xml @@ -76,7 +76,7 @@ ${oozieActionShareLibForSpark2} - yarn-cluster + yarn cluster researchinitiative-input-metadata-transformer eu.dnetlib.iis.wf.referenceextraction.researchinitiative.ResearchInitiativeReferenceExtractionInputTransformerJob @@ -107,7 +107,7 @@ ${oozieActionShareLibForSpark2} - yarn-cluster + yarn cluster researchinitiative-concept-transformer eu.dnetlib.iis.wf.referenceextraction.researchinitiative.ResearchInitiativeMetadataTransformerJob diff --git a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/softwareurl/main/oozie_app/workflow.xml b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/softwareurl/main/oozie_app/workflow.xml index 45802da95..bc12a4976 100644 --- a/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/softwareurl/main/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-referenceextraction/src/main/resources/eu/dnetlib/iis/wf/referenceextraction/softwareurl/main/oozie_app/workflow.xml @@ -202,7 +202,7 @@ - yarn-cluster + yarn cluster referenceextraction_softwareurl_webcrawl eu.dnetlib.iis.wf.referenceextraction.softwareurl.CachedWebCrawlerJob diff --git a/iis-wf/iis-wf-transformers/src/main/resources/eu/dnetlib/iis/wf/transformers/avro2json/oozie_app/workflow.xml b/iis-wf/iis-wf-transformers/src/main/resources/eu/dnetlib/iis/wf/transformers/avro2json/oozie_app/workflow.xml index 6484e88d2..7d33e70c6 100644 --- a/iis-wf/iis-wf-transformers/src/main/resources/eu/dnetlib/iis/wf/transformers/avro2json/oozie_app/workflow.xml +++ b/iis-wf/iis-wf-transformers/src/main/resources/eu/dnetlib/iis/wf/transformers/avro2json/oozie_app/workflow.xml @@ -79,7 +79,7 @@ - yarn-cluster + yarn cluster avro2json_transformer_job eu.dnetlib.iis.wf.transformers.avro2json.Avro2JsonTransformer