From 14c82a818278c271d12f3c34d21148f3571e7bea Mon Sep 17 00:00:00 2001 From: David Darras Date: Tue, 20 Feb 2024 11:55:17 +0100 Subject: [PATCH 1/5] chore: upgrade lib versions --- .github/workflows/build.yml | 18 +++---- .github/workflows/develop-release.yml | 18 +++---- .github/workflows/release.yml | 18 +++---- pom.xml | 71 ++++++++++++--------------- 4 files changed, 53 insertions(+), 72 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 1974c49..de5c302 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -9,24 +9,20 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: 11 + java-version: 21 + distribution: 'temurin' - name: Get current version id: version run: echo "::set-output name=prop::$(mvn -f pom.xml help:evaluate -Dexpression=project.version -q -DforceStdout)" - run: echo ${{steps.version.outputs.prop}} - - name: Get lunatic-model version - id: lunatic - run: echo "::set-output name=prop::$(mvn -f pom.xml help:evaluate -Dexpression=lunatic.model.version -q -DforceStdout)" - - name: Install lunatic-model - run: mvn install:install-file -Dfile=lib/lunatic-model-${{steps.lunatic.outputs.prop}}.jar -DgroupId=fr.insee.lunatic -DartifactId=lunatic-model -Dversion=${{steps.lunatic.outputs.prop}} -Dpackaging=jar - name: Build with Maven - run: mvn install --file pom.xml + run: mvn install --no-transfer-progress - name: Upload jar - uses: actions/upload-artifact@v2 + uses: actions/upload-artifact@v4 with: name: jar path: target/*.jar \ No newline at end of file diff --git a/.github/workflows/develop-release.yml b/.github/workflows/develop-release.yml index 4d2cdd7..9fc1abe 100644 --- a/.github/workflows/develop-release.yml +++ b/.github/workflows/develop-release.yml @@ -11,18 +11,14 @@ jobs: build-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: 11 - - name: Get lunatic-model version - id: lunatic - run: echo "::set-output name=prop::$(mvn -f pom.xml help:evaluate -Dexpression=lunatic.model.version -q -DforceStdout)" - - name: Install lunatic-model - run: mvn install:install-file -Dfile=lib/lunatic-model-${{steps.lunatic.outputs.prop}}.jar -DgroupId=fr.insee.lunatic -DartifactId=lunatic-model -Dversion=${{steps.lunatic.outputs.prop}} -Dpackaging=jar + java-version: 21 + distribution: 'temurin' - name: Build with Maven - run: mvn install --file pom.xml + run: mvn install --no-transfer-progress - name: Get current version id: version run: echo "::set-output name=prop::$(mvn -f pom.xml help:evaluate -Dexpression=project.version -q -DforceStdout)" @@ -39,7 +35,7 @@ jobs: prerelease: false - name: Upload Release Asset id: upload-release-asset - uses: actions/upload-release-asset@v1 + uses: actions/upload-release-asset@v4 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 71d3a29..1edf730 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -11,18 +11,14 @@ jobs: build-release: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: Set up JDK 11 - uses: actions/setup-java@v1 + - uses: actions/checkout@v4 + - name: Set up JDK 21 + uses: actions/setup-java@v4 with: - java-version: 11 - - name: Get lunatic-model version - id: lunatic - run: echo "::set-output name=prop::$(mvn -f pom.xml help:evaluate -Dexpression=lunatic.model.version -q -DforceStdout)" - - name: Install lunatic-model - run: mvn install:install-file -Dfile=lib/lunatic-model-${{steps.lunatic.outputs.prop}}.jar -DgroupId=fr.insee.lunatic -DartifactId=lunatic-model -Dversion=${{steps.lunatic.outputs.prop}} -Dpackaging=jar + java-version: 21 + distribution: 'temurin' - name: Build with Maven - run: mvn install --file pom.xml + run: mvn install --no-transfer-progress - name: Get current version id: version run: echo "::set-output name=prop::$(mvn -f pom.xml help:evaluate -Dexpression=project.version -q -DforceStdout)" @@ -39,7 +35,7 @@ jobs: prerelease: false - name: Upload Release Asset id: upload-release-asset - uses: actions/upload-release-asset@v1 + uses: actions/upload-release-asset@v4 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} with: diff --git a/pom.xml b/pom.xml index 618a8d2..ea13d53 100644 --- a/pom.xml +++ b/pom.xml @@ -11,17 +11,14 @@ Queen Batch - 11.0.2 - 1.11 - 1.11 - 5.2.1.RELEASE - 2.17.1 - 1.15.2 + 21 + 6.1.3 + 2.22.1 + 1.19.5 UTF-8 UTF-8 - 11 - 9.7.0-8 - 2.2.3 + 11.5 + 2.5.1 @@ -49,36 +46,31 @@ org.springframework.data spring-data-jpa - 2.2.3.RELEASE - - - - org.springframework.data - spring-data-mongodb - 2.2.3.RELEASE + 3.1.2 + commons-io commons-io - 2.6 + 2.15.1 org.apache.commons commons-lang3 - 3.10 + 3.12.0 org.postgresql postgresql - 42.3.8 + 42.5.4 org.liquibase liquibase-core - 3.8.9 + 4.25.1 @@ -86,7 +78,7 @@ org.junit.jupiter junit-jupiter-engine - 5.4.0 + 5.10.1 org.testcontainers @@ -94,23 +86,24 @@ ${testcontainers.version} test + + - org.testcontainers - mongodb - ${testcontainers.version} - test + com.google.code.gson + gson + 2.10.1 - com.googlecode.json-simple json-simple 1.1.1 - - - com.google.code.gson - gson - 2.8.6 + + + junit + junit + + @@ -119,6 +112,7 @@ log4j-core ${log4j.version} + org.apache.logging.log4j log4j-api @@ -140,12 +134,12 @@ org.eclipse.persistence org.eclipse.persistence.moxy - 2.5.0 + 4.0.1 org.eclipse.persistence eclipselink - 2.6.0 + 4.0.1 javax.xml.bind @@ -160,7 +154,7 @@ org.jdom jdom2 - 2.0.6 + 2.0.6.1 @@ -170,16 +164,15 @@ org.apache.maven.plugins maven-surefire-plugin - 3.0.0-M4 + 3.1.2 alphabetical UnitTests.java TestEndToEndJpa.java - TestEndToEndMongo.java 500 - -Xmx2048m -Xmx1G -XX:PermSize=1024m + -Xmx2048m -Xmx1G true @@ -188,7 +181,7 @@ org.apache.maven.plugins maven-eclipse-plugin - 2.9 + 2.10 true false @@ -243,7 +236,7 @@ maven-resources-plugin - 2.7 + 3.3.1 UTF-8 From 0bc8cf6464605c64bcdaa4fc7f80cb9c68df466c Mon Sep 17 00:00:00 2001 From: David Darras Date: Tue, 20 Feb 2024 11:56:23 +0100 Subject: [PATCH 2/5] fix: delete mongo functionalities --- README.md | 8 +- .../batch/config/ApplicationContext.java | 37 +-- .../insee/queen/batch/config/ConditonJpa.java | 6 +- .../queen/batch/config/ConditonMongo.java | 7 +- .../dao/impl/jpa/CampaignDaoJpaImpl.java | 4 - .../batch/dao/impl/jpa/CommentDaoJpaImpl.java | 3 - .../batch/dao/impl/jpa/DataDaoJpaImpl.java | 3 - .../dao/impl/jpa/NomenclatureDaoJpaImpl.java | 3 - .../dao/impl/jpa/ParadataEventDaoJpaImpl.java | 4 - .../impl/jpa/PersonalizationDaoJpaImpl.java | 3 - .../jpa/QuestionnaireModelDaoJpaImpl.java | 3 - .../jpa/RequiredNomenclatureDaoJpaImpl.java | 3 - .../dao/impl/jpa/StateDataDaoJpaImpl.java | 4 - .../dao/impl/jpa/SurveyUnitDaoJpaImpl.java | 3 - .../dao/mongo/impl/CampaignDaoMongoImpl.java | 84 ------- .../dao/mongo/impl/CommentDaoMongoImpl.java | 74 ------ .../dao/mongo/impl/DataDaoMongoImpl.java | 109 --------- .../mongo/impl/NomenclatureDaoMongoImpl.java | 101 --------- .../mongo/impl/ParadataEventDaoMongoImpl.java | 97 -------- .../impl/PersonalizationDaoMongoImpl.java | 128 ----------- .../impl/QuestionnaireModelDaoMongoImpl.java | 127 ----------- .../dao/mongo/impl/StateDataDaoMongoImpl.java | 86 ------- .../mongo/impl/SurveyUnitDaoMongoImpl.java | 212 ------------------ .../fr/insee/queen/batch/object/Campaign.java | 5 - .../fr/insee/queen/batch/object/Comment.java | 4 - .../fr/insee/queen/batch/object/Data.java | 4 - .../fr/insee/queen/batch/object/Metadata.java | 8 +- .../queen/batch/object/Nomenclature.java | 2 - .../queen/batch/object/Personalization.java | 4 - .../batch/object/QuestionnaireModel.java | 2 - .../insee/queen/batch/object/StateData.java | 11 +- .../insee/queen/batch/object/SurveyUnit.java | 10 - .../queen/batch/service/DatabaseService.java | 38 +--- .../queen/batch/service/DeleteService.java | 42 ++-- .../batch/service/ExtractionService.java | 14 +- .../queen/batch/service/LoadService.java | 39 ++-- .../fr/insee/queen/batch/TestEndToEndJpa.java | 1 - .../insee/queen/batch/TestEndToEndMongo.java | 86 ------- 38 files changed, 49 insertions(+), 1330 deletions(-) delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/CampaignDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/CommentDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/NomenclatureDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/PersonalizationDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/QuestionnaireModelDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/StateDataDaoMongoImpl.java delete mode 100644 src/main/java/fr/insee/queen/batch/dao/mongo/impl/SurveyUnitDaoMongoImpl.java delete mode 100644 src/test/java/fr/insee/queen/batch/TestEndToEndMongo.java diff --git a/README.md b/README.md index 6f31066..858b421 100644 --- a/README.md +++ b/README.md @@ -4,12 +4,12 @@ Batch using to implement QUEEN DB from xml files. ## Requirements For building and running the application you need: -- [JDK 1.11](https://jdk.java.net/archive/) +- JDK 21 - Maven 3 # Add lunatic librairy to project ``` shell -mvn install:install-file -Dfile=lib/lunatic-model-2.2.3.jar -DgroupId=fr.insee.lunatic -DartifactId=lunatic-model -Dversion=2.2.3 -Dpackaging=jar +mvn install:install-file -Dfile=lib/lunatic-model-2.5.1.jar -DgroupId=fr.insee.lunatic -DartifactId=lunatic-model -Dversion=2.5.1 -Dpackaging=jar ``` ## Install and excute unit tests and ent-to-end tests @@ -19,7 +19,7 @@ mvn clean install ``` ## Running batch -Use the folowing cmd : +Use the following cmd : ``` shell echo $@ java8 -Xms64m -Xmx512m -classpath '/path/to/lib/*' -Dlog4j.configurationFile=file:/path/to/log4j2/config/log4j2.xml -Dproperties.path=/path/to/properties -DcheminLog=/path/to/log fr.insee.queen.batch.Lanceur $@ @@ -38,7 +38,6 @@ fr.insee.queen.persistence.database.schema = XXXXXXX fr.insee.queen.persistence.database.user = XXXXXXX fr.insee.queen.persistence.database.password = XXXXXXX fr.insee.queen.persistence.database.driver = org.postgresql.Driver -fr.insee.queen.application.persistenceType = (JPA or MONGODB) fr.insee.queen.folder.in=path/to/in fr.insee.queen.folder.out=path/to/out fr.insee.queen.paradata.id=idSu @@ -58,7 +57,6 @@ Before committing code please ensure, - spring-data-jpa - commons-lang3 - postgresql -- mongoDb - liquibase - spring-test - test-containers diff --git a/src/main/java/fr/insee/queen/batch/config/ApplicationContext.java b/src/main/java/fr/insee/queen/batch/config/ApplicationContext.java index 6ff3f39..31a4540 100644 --- a/src/main/java/fr/insee/queen/batch/config/ApplicationContext.java +++ b/src/main/java/fr/insee/queen/batch/config/ApplicationContext.java @@ -10,20 +10,13 @@ import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.ComponentScan; -import org.springframework.context.annotation.Conditional; import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.PropertySource; -import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.datasource.DataSourceUtils; import org.springframework.jdbc.datasource.DriverManagerDataSource; import org.springframework.jdbc.datasource.SingleConnectionDataSource; -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; - @Configuration @ComponentScan("fr.insee.queen.batch.*") @PropertySource(value = {"classpath:/queen-bo.properties", "file:${properties.path}/queen-bo.properties"}, ignoreResourceNotFound = true) @@ -72,7 +65,6 @@ public class ApplicationContext { * @return new Datasource */ @Bean("dataSource") - @Conditional(value= ConditonJpa.class) public DataSource dataSource() { DriverManagerDataSource dataSource = new DriverManagerDataSource(); dataSource.setDriverClassName(dbDriver); @@ -89,7 +81,6 @@ public DataSource dataSource() { * @throws SQLException */ @Bean("connection") - @Conditional(value= ConditonJpa.class) public Connection connection(@Autowired @Qualifier("dataSource") DataSource dataSource) throws SQLException { return DataSourceUtils.getConnection(dataSource); } @@ -100,7 +91,6 @@ public Connection connection(@Autowired @Qualifier("dataSource") DataSource data * @return JdbcTemplate */ @Bean("jdbcTemplate") - @Conditional(value= ConditonJpa.class) public JdbcTemplate jdbcTemplate(@Autowired @Qualifier("dataSource") DataSource dataSource) { JdbcTemplate jdbcTemplate = null; try { @@ -111,32 +101,7 @@ public JdbcTemplate jdbcTemplate(@Autowired @Qualifier("dataSource") DataSource jdbcTemplate.setResultsMapCaseInsensitive(true); return jdbcTemplate; } - - /** - * Method used to create the connection with the mongo database - * @return - */ - @Bean - @Conditional(value= ConditonMongo.class) - public MongoClient mongo() { - ConnectionString connectionString = new ConnectionString(String.format("mongodb://%s:%s/%s", dbHost, dbPort, dbSchema)); - MongoClientSettings mongoClientSettings = MongoClientSettings.builder() - .applyConnectionString(connectionString) - .build(); - return MongoClients.create(mongoClientSettings); - } - - /** - * Method used to create the mongoTemplate - * @return - * @throws Exception - */ - @Conditional(value= ConditonMongo.class) - @Bean - public MongoTemplate mongoTemplate() throws Exception { - return new MongoTemplate(mongo(), dbSchema); - } - + /** * Bean to get the Folder_in value * @return diff --git a/src/main/java/fr/insee/queen/batch/config/ConditonJpa.java b/src/main/java/fr/insee/queen/batch/config/ConditonJpa.java index 506e988..21a4b22 100644 --- a/src/main/java/fr/insee/queen/batch/config/ConditonJpa.java +++ b/src/main/java/fr/insee/queen/batch/config/ConditonJpa.java @@ -2,7 +2,6 @@ import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; -import org.springframework.core.env.Environment; import org.springframework.core.type.AnnotatedTypeMetadata; /** @@ -16,11 +15,10 @@ public class ConditonJpa implements Condition{ /** * This method override the Condition.class and checking if * the application needs to be launch in JPA persistence mode + * There is no mongo anymore but we keep this class now as it is used by other jars */ @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - Environment env = context.getEnvironment(); - return null != env - && "JPA".equals(env.getProperty("fr.insee.queen.application.persistenceType")); + return true; } } diff --git a/src/main/java/fr/insee/queen/batch/config/ConditonMongo.java b/src/main/java/fr/insee/queen/batch/config/ConditonMongo.java index 29aa3a7..1e10c11 100644 --- a/src/main/java/fr/insee/queen/batch/config/ConditonMongo.java +++ b/src/main/java/fr/insee/queen/batch/config/ConditonMongo.java @@ -2,7 +2,6 @@ import org.springframework.context.annotation.Condition; import org.springframework.context.annotation.ConditionContext; -import org.springframework.core.env.Environment; import org.springframework.core.type.AnnotatedTypeMetadata; /** @@ -16,11 +15,11 @@ public class ConditonMongo implements Condition{ /** * This method override the Condition.class and checking if * the application needs to be launch in MONGO persistence mode + * + * There is no mongo anymore but we keep this class now as it is used by other jars */ @Override public boolean matches(ConditionContext context, AnnotatedTypeMetadata metadata) { - Environment env = context.getEnvironment(); - return null != env - && "MONGODB".equals(env.getProperty("fr.insee.queen.application.persistenceType")); + return false; } } diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CampaignDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CampaignDaoJpaImpl.java index efd41ac..f0aeaf5 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CampaignDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CampaignDaoJpaImpl.java @@ -6,12 +6,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.CampaignDao; import fr.insee.queen.batch.object.Campaign; @@ -21,7 +18,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class CampaignDaoJpaImpl implements CampaignDao { @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CommentDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CommentDaoJpaImpl.java index 5f2b70d..a14294c 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CommentDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/CommentDaoJpaImpl.java @@ -8,11 +8,9 @@ import org.postgresql.util.PGobject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.CommentDao; import fr.insee.queen.batch.object.SurveyUnit; @@ -22,7 +20,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class CommentDaoJpaImpl implements CommentDao { @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java index e106e5b..02d2170 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java @@ -10,11 +10,9 @@ import org.postgresql.util.PGobject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.DataDao; import fr.insee.queen.batch.object.SurveyUnit; @@ -24,7 +22,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class DataDaoJpaImpl implements DataDao { @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/NomenclatureDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/NomenclatureDaoJpaImpl.java index b26c069..fc53056 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/NomenclatureDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/NomenclatureDaoJpaImpl.java @@ -7,12 +7,10 @@ import org.postgresql.util.PGobject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.NomenclatureDao; import fr.insee.queen.batch.object.Nomenclature; @@ -22,7 +20,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class NomenclatureDaoJpaImpl implements NomenclatureDao { @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java index 7409323..8058b79 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java @@ -4,7 +4,6 @@ import java.sql.SQLException; import java.util.*; -import fr.insee.queen.batch.dao.mongo.impl.ParadataEventDaoMongoImpl; import fr.insee.queen.batch.object.ParadataEvent; import org.json.simple.JSONArray; import org.json.simple.JSONObject; @@ -13,12 +12,10 @@ import org.postgresql.util.PGobject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.ParadataEventDao; import fr.insee.queen.batch.service.DatabaseService; @@ -28,7 +25,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class ParadataEventDaoJpaImpl implements ParadataEventDao{ @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/PersonalizationDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/PersonalizationDaoJpaImpl.java index 7a179e4..dbc93bd 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/PersonalizationDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/PersonalizationDaoJpaImpl.java @@ -12,12 +12,10 @@ import org.postgresql.util.PGobject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.PersonalizationDao; import fr.insee.queen.batch.object.Personalization; import fr.insee.queen.batch.object.SurveyUnit; @@ -28,7 +26,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class PersonalizationDaoJpaImpl implements PersonalizationDao{ @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/QuestionnaireModelDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/QuestionnaireModelDaoJpaImpl.java index 5753899..c570bc1 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/QuestionnaireModelDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/QuestionnaireModelDaoJpaImpl.java @@ -8,12 +8,10 @@ import org.postgresql.util.PGobject; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.QuestionnaireModelDao; import fr.insee.queen.batch.object.QuestionnaireModel; @@ -23,7 +21,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class QuestionnaireModelDaoJpaImpl implements QuestionnaireModelDao{ @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/RequiredNomenclatureDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/RequiredNomenclatureDaoJpaImpl.java index 87c8a6c..ac3dd51 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/RequiredNomenclatureDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/RequiredNomenclatureDaoJpaImpl.java @@ -5,11 +5,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.RequiredNomenclatureDao; import fr.insee.queen.batch.object.Nomenclature; @@ -19,7 +17,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class RequiredNomenclatureDaoJpaImpl implements RequiredNomenclatureDao{ @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/StateDataDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/StateDataDaoJpaImpl.java index 0d7507f..91dfd05 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/StateDataDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/StateDataDaoJpaImpl.java @@ -6,14 +6,11 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.StateDataDao; import fr.insee.queen.batch.object.StateData; -import fr.insee.queen.batch.utils.PathUtils; /** * Service for the StateData entity that implements the interface associated @@ -21,7 +18,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class StateDataDaoJpaImpl implements StateDataDao{ @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/SurveyUnitDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/SurveyUnitDaoJpaImpl.java index 51daaac..167edf6 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/SurveyUnitDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/SurveyUnitDaoJpaImpl.java @@ -10,12 +10,10 @@ import fr.insee.queen.batch.object.StateData; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; import fr.insee.queen.batch.dao.SurveyUnitDao; import fr.insee.queen.batch.object.SurveyUnit; @@ -25,7 +23,6 @@ * */ @Service -@Conditional(value= ConditonJpa.class) public class SurveyUnitDaoJpaImpl implements SurveyUnitDao { @Autowired diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/CampaignDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/CampaignDaoMongoImpl.java deleted file mode 100644 index 3f23fe6..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/CampaignDaoMongoImpl.java +++ /dev/null @@ -1,84 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.CampaignDao; -import fr.insee.queen.batch.object.Campaign; - -/** - * Service for the Campaign entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class CampaignDaoMongoImpl implements CampaignDao { - - @Autowired - MongoTemplate mongoTemplate; - - /** - * Implements the creation of a Campaign in database - * @param campaign - * @throws SQLException - */ - @Override - public void create(Campaign campaign) { - if(campaign.getLabel().equals(null) || campaign.getLabel().isBlank()) { - campaign.setLabel(""); - } - mongoTemplate.save(campaign, "campaign"); - } - - /** - * Implementation to check if an Campaign already exist in database - * @param id - * @return boolean - * @throws SQLException - */ - @Override - public boolean exist(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, Campaign.class, "campaign") != null; - } - - /** - * Retrieve the Campaign by the id passed in parameter - * @param id - * @return Campaign object - */ - @Override - public Campaign findById(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, Campaign.class, "campaign"); - } - - /** - * Retrieve all the campaigns - */ - @Override - public List findAll() { - return mongoTemplate.findAll(Campaign.class, "campaign"); - } - - /** - * Delete a campaign by his id - */ - @Override - public void delete(String id) { - Campaign campaign = new Campaign(); - campaign = findById(id); - mongoTemplate.remove(campaign, "campaign"); - } -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/CommentDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/CommentDaoMongoImpl.java deleted file mode 100644 index fa1c132..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/CommentDaoMongoImpl.java +++ /dev/null @@ -1,74 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; -import java.util.List; - -import org.json.simple.JSONObject; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.Constants; -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.CommentDao; -import fr.insee.queen.batch.object.Comment; -import fr.insee.queen.batch.object.SurveyUnit; - -/** - * Service for the Comment entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class CommentDaoMongoImpl implements CommentDao { - - @Autowired - MongoTemplate mongoTemplate; - - /** - * Delete all the comments for a list of SU - */ - @Override - public void deleteCommentBySurveyUnitIds(List lstSu) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").in(lstSu)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getComment() != null) { - mongoTemplate.remove(su.getComment(), Constants.COMMENT); - } - }); - } - - /** - * Delete all the comment for a campaign - */ - @Override - public void deleteCommentByCampaignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where("campaign.id").is(campaignId)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getComment() != null) { - mongoTemplate.remove(su.getComment(), Constants.COMMENT); - } - }); - } - - /** - * Create a comment for a SurveyUnit - */ - @Override - public void createComment(SurveyUnit surveyUnit) throws SQLException { - Comment comment = new Comment(); - comment.setId(surveyUnit.getComment().getId()); - comment.setValue(new JSONObject()); - comment.setSurveyUnit(surveyUnit); - mongoTemplate.save(comment, Constants.COMMENT); - } - -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java deleted file mode 100644 index 009db1f..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java +++ /dev/null @@ -1,109 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; -import java.util.List; -import java.util.concurrent.atomic.AtomicInteger; - -import org.json.simple.JSONObject; -import org.json.simple.parser.ParseException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.Constants; -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.DataDao; -import fr.insee.queen.batch.object.Data; -import fr.insee.queen.batch.object.SurveyUnit; - -/** - * Service for the Data entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class DataDaoMongoImpl implements DataDao { - - @Autowired - MongoTemplate mongoTemplate; - - /** - * Implements the creation of data in database - * @param surveyUnit - * @throws SQLException - */ - @Override - public void createData(SurveyUnit surveyUnit) { - Data data = new Data(); - data.setId(surveyUnit.getData().getId()); - data.setValue(surveyUnit.getData().getValue()); - data.setSurveyUnit(surveyUnit); - mongoTemplate.save(data, Constants.DATA); - } - - /** - * Get the all the data for a SU - */ - @Override - public JSONObject getDataBySurveyUnitId(String suId) throws ParseException { - Query query = new Query(); - query.fields().include("value"); - return mongoTemplate.findOne(query, Data.class, Constants.DATA).getValue(); - } - - /** - * Delete all the data for a list of SU - */ - @Override - public int deleteDataBySurveyUnitIds(List lstSu) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").in(lstSu)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - AtomicInteger nb= new AtomicInteger(); - suList.stream().forEach(su -> { - if(su != null && su.getData() != null) { - mongoTemplate.remove(su.getData(), Constants.DATA); - nb.getAndIncrement(); - } - }); - return nb.get(); - } - - @Override - public int setDataToEmptyBySurveyUnitIds(List lstSu) { - return 0; - } - - /** - * Delete the data for a Campaign id - */ - @Override - public void deleteDataByCampaignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where("campaign.id").is(campaignId)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getData() != null) { - mongoTemplate.remove(su.getData(), Constants.DATA); - } - }); - } - - /** - * Update data for a SurveyUnit - */ - @Override - public void updateData(SurveyUnit surveyUnit) { - Query query = new Query(); - query.addCriteria(Criteria.where("surveyUnit.id").is(surveyUnit.getId())); - Data dataTemp = mongoTemplate.findOne(query, Data.class, Constants.DATA); - dataTemp.setValue(surveyUnit.getData().getValue()); - dataTemp.setSurveyUnit(surveyUnit); - mongoTemplate.save(dataTemp, Constants.DATA); - } - -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/NomenclatureDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/NomenclatureDaoMongoImpl.java deleted file mode 100644 index d31da11..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/NomenclatureDaoMongoImpl.java +++ /dev/null @@ -1,101 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.core.query.Update; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.NomenclatureDao; -import fr.insee.queen.batch.object.Nomenclature; -import fr.insee.queen.batch.object.QuestionnaireModel; - -/** - * Service for the Nomenclature entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class NomenclatureDaoMongoImpl implements NomenclatureDao { - - @Autowired - MongoTemplate mongoTemplate; - - /** - * Implements the creation of a Nomenclature in database - * @param nomenclature - * @throws SQLException - */ - @Override - public void create(Nomenclature nomenclature) throws SQLException { - mongoTemplate.save(nomenclature, "nomenclature"); - } - - /** - * Implements the delete of a Nomenclature in database - * @param nomenclature - */ - @Override - public void delete(Nomenclature nomenclature) { - mongoTemplate.remove(nomenclature, "nomenclature"); - } - - /** - * Implements the update of a Nomenclature in database - * @param nomenclature - * @throws SQLException - */ - @Override - public void update(Nomenclature nomenclature) throws SQLException { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(nomenclature.getId())); - Update update = new Update(); - update.set("label", nomenclature.getLabel()); - update.set("value", nomenclature.getValue().toJSONString()); - mongoTemplate.findAndModify(query, update, Nomenclature.class, "nomenclature"); - } - - /** - * Retrieve the nomenclature by the id passed in parameter - * @param id - * @return Nomenclature object - */ - @Override - public Nomenclature findById(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, Nomenclature.class, "nomenclature"); - } - - /** - * Check if nomenclature exist - */ - @Override - public boolean exist(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, Nomenclature.class, "nomenclature") != null; - } - - /** - * Check is nomenclature is used - */ - @Override - public boolean isUsed(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("nomenclatures.id").is(id)); - QuestionnaireModel qm = mongoTemplate.findOne(query, QuestionnaireModel.class, "questionnaire_model"); - if(qm != null) { - return true; - } else { - return false; - } - } - -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java deleted file mode 100644 index d521230..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java +++ /dev/null @@ -1,97 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.ParadataEventDao; -import fr.insee.queen.batch.service.DatabaseService; -import org.json.simple.JSONArray; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -/** - * Service for the Paradata-Event entity that implements the interface associated - * - * @author scorcaud - */ -@Service -@Conditional(value = ConditonMongo.class) -public class ParadataEventDaoMongoImpl implements ParadataEventDao { - - @Autowired - MongoTemplate mongoTemplate; - - @Autowired - DatabaseService databaseService; - - /** - * Method used to retreive all the paradata for a SurveyUnit - * - * @throws ParseException - */ - @SuppressWarnings("unchecked") - @Override - public JSONObject findBySurveyUnitId(String suId) throws ParseException { - JSONParser parser = new JSONParser(); - JSONArray array = new JSONArray(); - List value = new ArrayList<>(); - Query query = new Query(); - query.addCriteria(Criteria.where("value." + databaseService.getKeyParadataIdSu()).is(suId)); - query.fields().include("value"); - query.fields().exclude("_id"); - value = mongoTemplate.find(query, String.class, "paradata_event"); - for (String val : value) { - JSONObject jsonObjectTemp = (JSONObject) parser.parse(val); - JSONObject jsonObjectTemp2 = (JSONObject) jsonObjectTemp.get("value"); - array.add(jsonObjectTemp2.get(databaseService.getKeyParadataEvents())); - } - JSONObject jsobObjectClean = new JSONObject(); - jsobObjectClean.put("idSu", suId); - jsobObjectClean.put("events", array); - return jsobObjectClean; - } - - @Override - public void deleteParadataById(String id) throws SQLException { - ParadataEvent paradataEvent = findById(id); - mongoTemplate.remove(paradataEvent, "paradata_event"); - } - - public ParadataEvent findById(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, ParadataEvent.class, "paradata_event"); - } - - /** - * This method create a paradata_event, only use for for testing in the creation of the dataset - */ - @Override - public void createParadata(JSONObject jsonParadata) throws SQLException { - ParadataEvent paradata = new ParadataEvent(UUID.randomUUID(), jsonParadata); - mongoTemplate.save(paradata, "paradata_event"); - } - - /** - * This class is used only to create Paradata object for testing - */ - static class ParadataEvent { - UUID id; - JSONObject value; - - public ParadataEvent(UUID id, JSONObject value) { - this.id = id; - this.value = value; - } - } -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/PersonalizationDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/PersonalizationDaoMongoImpl.java deleted file mode 100644 index 949145a..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/PersonalizationDaoMongoImpl.java +++ /dev/null @@ -1,128 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; -import java.util.List; -import java.util.UUID; - -import org.json.simple.JSONArray; -import org.json.simple.parser.ParseException; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.PersonalizationDao; -import fr.insee.queen.batch.object.Personalization; -import fr.insee.queen.batch.object.SurveyUnit; - -/** - * Service for the Personalization entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class PersonalizationDaoMongoImpl implements PersonalizationDao{ - - @Autowired - MongoTemplate mongoTemplate; - - /** - * Create a Personalization in database - * @param surveyUnit - * @throws SQLException - */ - @Override - public void createPersonalization(SurveyUnit surveyUnit) { - Personalization personalization = new Personalization(); - personalization.setId(surveyUnit.getPersonalization().getId()); - personalization.setValue(surveyUnit.getPersonalization().getValue()); - personalization.setSurveyUnit(surveyUnit); - mongoTemplate.save(personalization, "personalization"); - } - - /** - * Update a personalization by a survey unit - * @param surveyUnit - * @throws SQLException - */ - @Override - public void updatePersonalization(SurveyUnit surveyUnit) { - Query query = new Query(); - query.addCriteria(Criteria.where("surveyUnit.id").is(surveyUnit.getId())); - Personalization personalizationTemp = mongoTemplate.findOne(query, Personalization.class, "personalization"); - if(personalizationTemp != null) { - personalizationTemp.setValue(surveyUnit.getPersonalization().getValue()); - personalizationTemp.setSurveyUnit(surveyUnit); - mongoTemplate.save(personalizationTemp, "personalization"); - } else { - createPersonalization(surveyUnit); - } - } - - /** - * Retrieves all the personalization for a Survey Unit - * @param surveyUnitId - * @return - */ - @Override - public void deleteByCampaignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where("campaign.id").is(campaignId)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getPersonalization() != null) { - mongoTemplate.remove(su.getPersonalization(), "personalization"); - } - }); - } - - /** - * Delete all the personalization for a campaign - * @param campaignId - */ - @Override - public void deleteBySurveyUnitIds(List lstSu) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").in(lstSu)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getPersonalization() != null) { - mongoTemplate.remove(su.getPersonalization(), "personalization"); - } - }); - } - - /** - * Delete all the personalizations for a list of Survey - * @param lstSu - */ - @Override - public List findBySurveyUnitId(String surveyUnitId) { - Query query = new Query(); - query.addCriteria(Criteria.where("surveyUnit.id").is(surveyUnitId)); - return mongoTemplate.find(query, Personalization.class, "personalization"); - } - - /** - * Retrieve the value by the id passed in parameter - * @param id - * @return JSONObject object - * @throws ParseException - */ - @Override - public JSONArray getValueById(UUID id) throws ParseException { - Query query = new Query(); - query.fields().include("value"); - query.addCriteria(Criteria.where("id").is(id)); - JSONArray value = mongoTemplate.findOne(query, Personalization.class, "personalization").getValue(); - if(value != null) { - return value; - } else { - return new JSONArray(); - } - } -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/QuestionnaireModelDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/QuestionnaireModelDaoMongoImpl.java deleted file mode 100644 index 1fb228e..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/QuestionnaireModelDaoMongoImpl.java +++ /dev/null @@ -1,127 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.data.mongodb.core.query.Update; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.CampaignDao; -import fr.insee.queen.batch.dao.QuestionnaireModelDao; -import fr.insee.queen.batch.object.Campaign; -import fr.insee.queen.batch.object.QuestionnaireModel; - -/** - * Service for the QuestionnaireModel entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class QuestionnaireModelDaoMongoImpl implements QuestionnaireModelDao{ - - @Autowired - MongoTemplate mongoTemplate; - - @Autowired - CampaignDao campaignDao; - - /** - * Implements the creation of a QuestionnaireModel in database - * @param questionnaireModel - * @throws SQLException - */ - @Override - public void create(QuestionnaireModel questionnaireModel, String campaignId) throws SQLException { - mongoTemplate.save(questionnaireModel, "questionnaire_model"); - } - - /** - * Implementation to check if a QuestionnaireModel already exist in database - * @param id - * @return boolean - * @throws SQLException - */ - @Override - public boolean exist(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, QuestionnaireModel.class, "questionnaire_model") != null; - } - - - /** - * Retrieve the QuestionnaireModel by the id passed in parameter - * @param id - * @return QuestionnaireModel object - */ - @Override - public QuestionnaireModel findById(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, QuestionnaireModel.class, "questionnaire_model"); - } - - /** - * Retrieve the QuestionnaireModel by the id passed in parameter - * @param id - * @return QuestionnaireModel object - */ - @Override - public List findByCampaignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(campaignId)); - return mongoTemplate.findOne(query, Campaign.class, "campaign").getQuestionnaireModels(); - } - - /** - * Implements the update of a questionnaireModel in database - * @param questionnaireModel - * @throws SQLException - */ - @Override - public void updateCampaignId(QuestionnaireModel questionnaireModel) throws SQLException { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(questionnaireModel.getId())); - Update update = new Update(); - Campaign campaign = campaignDao.findById(questionnaireModel.getCampaignId()); - if(campaign != null) { - update.set("campaign", campaign); - } - mongoTemplate.findAndModify(query, update, QuestionnaireModel.class, "questionnaire_model"); - } - - /** - * Implementation to check if a QuestionnaireModel already exist in database - * @param id - * @return boolean - * @throws SQLException - */ - @Override - public boolean existForCampaign(String id, String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where("campign.id").is(campaignId)); - return mongoTemplate.findOne(query, QuestionnaireModel.class, "questionnaire_model") != null; - } - - /** - * Delete association between campaign and questionnaire model - * @param questionnaireModel - * @throws SQLException - */ - @Override - public void deleteCampaignIdForQuestionnaireModel(String campaignId) throws SQLException { - List qmList = findByCampaignId(campaignId); - qmList.stream().forEach(qm -> { - qm.setCampaignId(null); - mongoTemplate.save(qm, "questionnaire_model"); - }); - - } -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/StateDataDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/StateDataDaoMongoImpl.java deleted file mode 100644 index e6914e2..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/StateDataDaoMongoImpl.java +++ /dev/null @@ -1,86 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.time.Instant; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.StateDataDao; -import fr.insee.queen.batch.object.StateData; -import fr.insee.queen.batch.object.SurveyUnit; -import fr.insee.queen.batch.utils.PathUtils; - -/** - * Service for the StateData entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class StateDataDaoMongoImpl implements StateDataDao{ - - @Autowired - MongoTemplate mongoTemplate; - - @Override - public void createStateData(StateData stateData) { - mongoTemplate.save(stateData, "state_data"); - } - /** - * Delete a state for a list of Survey units - * @param lstSu - */ - @Override - public void deleteStateDataBySU(List lstSu) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").in(lstSu)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getStateData() != null) { - mongoTemplate.remove(su.getStateData(), "state_data"); - } - }); - } - - /** - * Delete a state for all Survey units of a campaign - * @param campaignId - */ - @Override - public void deleteStateDataByCampaignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where("campaign.id").is(campaignId)); - List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); - suList.stream().forEach(su -> { - if(su != null && su.getStateData() != null) { - mongoTemplate.remove(su.getStateData(), "state_data"); - } - }); - } - - /** - * Update state of a Survey Unit - * @param suId - * @param state - */ - @Override - public void updateSurveyUnitStateById(String suId, String state) { - Query query = new Query(); - query.addCriteria(Criteria.where("surveyUnit.id").is(suId)); - query.fields().include("surveyUnit"); - query.fields().include("currentPage"); - query.fields().include("date"); - StateData su = mongoTemplate.findOne(query, StateData.class, "state_data"); - if(su != null) { - su.setState(state); - su.setDate(Instant.now().toEpochMilli()); - mongoTemplate.save(su, "state_data"); - } - } -} diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/SurveyUnitDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/SurveyUnitDaoMongoImpl.java deleted file mode 100644 index 6a05b44..0000000 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/SurveyUnitDaoMongoImpl.java +++ /dev/null @@ -1,212 +0,0 @@ -package fr.insee.queen.batch.dao.mongo.impl; - -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Conditional; -import org.springframework.data.mongodb.core.MongoTemplate; -import org.springframework.data.mongodb.core.query.Criteria; -import org.springframework.data.mongodb.core.query.Query; -import org.springframework.stereotype.Service; - -import fr.insee.queen.batch.Constants; -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.CampaignDao; -import fr.insee.queen.batch.dao.SurveyUnitDao; -import fr.insee.queen.batch.object.Metadata; -import fr.insee.queen.batch.object.StateData; -import fr.insee.queen.batch.object.SurveyUnit; - -/** - * Service for the SurveyUnit entity that implements the interface associated - * @author scorcaud - * - */ -@Service -@Conditional(value= ConditonMongo.class) -public class SurveyUnitDaoMongoImpl implements SurveyUnitDao { - - @Autowired - MongoTemplate mongoTemplate; - - @Autowired - CampaignDao campaignDao; - - /** - * Implements the creation of a SurveyUnit in database - * @param surveyUnit - * @param idCampaign - * @throws SQLException - */ - @Override - public void createSurveyUnit(SurveyUnit surveyUnit) { - mongoTemplate.save(surveyUnit, Constants.SURVEY_UNIT); - } - - /** - * Implementation to check if an SurveyUnit already exist in database - * @param xmlId - * @return boolean - * @throws SQLException - */ - @Override - public boolean existSurveyUnit(String xmlId) { - Query query = new Query(); - query.addCriteria(Criteria.where("_id").is(xmlId)); - return mongoTemplate.findOne(query, SurveyUnit.class, Constants.SURVEY_UNIT) != null; - } - - /** - * Get all SU for a campaign - * @param campaignId - * @return - */ - @Override - public List getAllSurveyUnitByCamapignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where(Constants.CAMPAIGN_ID).is(campaignId)); - List surveyUnits = mongoTemplate.find(query, SurveyUnit.class, Constants.SURVEY_UNIT); - List surveyIds = new ArrayList<>(); - if(!surveyUnits.isEmpty()) { - for(SurveyUnit su : surveyUnits) { - surveyIds.add(su.getId()); - } - } - return surveyIds; - } - - @Override - public List getAllSurveyUnitsWithStateByCampaignId(String campaignId) { - return null; - } - - /** - * Get unexisting SurveyUnits - * @param lstSu - * @return - */ - @Override - public List findUnexistingSurveyUnitsInList(List lstSu) { - List unexistingSu = new ArrayList<>(); - lstSu.stream().forEach(su -> { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(su)); - SurveyUnit suTemp = mongoTemplate.findOne(query, SurveyUnit.class, Constants.SURVEY_UNIT); - if(suTemp == null) { - unexistingSu.add(su); - } - }); - return unexistingSu; - } - - @Override - public List findSurveyUnitsByStateByCampaignId(String campaignId, String state){ - return null; - } - - /** - * Delete a list of SU - * @param lstSu - */ - @Override - public void deleteSurveyUnits(List lstSu) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").in(lstSu)); - List suList = mongoTemplate.find(query, SurveyUnit.class, Constants.SURVEY_UNIT); - suList.stream().forEach(su -> { - if(su != null) { - mongoTemplate.remove(su, Constants.SURVEY_UNIT); - } - }); - } - - /** - * Delete all SU for a campaign - * @param id - */ - @Override - public void deleteSurveyUnitByCampaignId(String campaignId) { - Query query = new Query(); - query.addCriteria(Criteria.where(Constants.CAMPAIGN_ID).is(campaignId)); - List suList = mongoTemplate.find(query, SurveyUnit.class, Constants.SURVEY_UNIT); - suList.stream().forEach(su -> { - if(su != null) { - mongoTemplate.remove(su, Constants.SURVEY_UNIT); - } - }); - } - - /** - * Update a SurveyUnit - * @param surveyUnit - */ - @Override - public void updateSurveyUnit(SurveyUnit surveyUnit) { - Query query = new Query(); - query.addCriteria(Criteria.where("id").is(surveyUnit.getId())); - SurveyUnit su = mongoTemplate.findOne(query, SurveyUnit.class, Constants.SURVEY_UNIT); - su.setCampaign(surveyUnit.getCampaign()); - su.setComment(surveyUnit.getComment()); - su.setData(surveyUnit.getData()); - su.setPersonalization(surveyUnit.getPersonalization()); - su.setStateData(surveyUnit.getStateData()); - mongoTemplate.save(su, Constants.SURVEY_UNIT); - } - - /** - * Get Survey unit by his id - * @param id - * @return - */ - @Override - public String findQuestionnaireIdBySurveyUnitId(String id) { - Query query = new Query(); - query.fields().include("questionnaireModel.id"); - query.addCriteria(Criteria.where("id").is(id)); - return mongoTemplate.findOne(query, SurveyUnit.class, Constants.SURVEY_UNIT).getQuestionnaireModel().getId(); - } - - /** - * Delete meta data by campaign id - * @param id - */ - @Override - public void deleteMetaDataByCampaignId(String id) { - Query query = new Query(); - query.addCriteria(Criteria.where(Constants.CAMPAIGN_ID).is(id)); - List mdList = mongoTemplate.find(query, Metadata.class, "metadata"); - mdList.stream().forEach(md -> { - if(md != null) { - mongoTemplate.remove(md, "metadata"); - } - }); - } - - /** - * Get ids for survey unit with state "validated" - * @return - */ - @Override - public List findSurveyUnitsValidatedIdsByCampaignId(String campaignId) { - List surveysIdsAll = getAllSurveyUnitByCamapignId(campaignId); - List surveysToRemove = new ArrayList<>(); - if(!surveysIdsAll.isEmpty()) { - surveysIdsAll.stream().forEach(su -> { - Query query = new Query(); - query.addCriteria(Criteria.where("surveyUnit.id").is(su)); - query.addCriteria(Criteria.where("state").is("VALIDATED")); - StateData stateTemp = mongoTemplate.findOne(query, StateData.class, "state_data"); - if(stateTemp == null) { - surveysToRemove.add(su); - } - }); - if(!surveysToRemove.isEmpty()) { - surveysIdsAll.removeAll(surveysToRemove); - } - } - return surveysIdsAll; - } -} - diff --git a/src/main/java/fr/insee/queen/batch/object/Campaign.java b/src/main/java/fr/insee/queen/batch/object/Campaign.java index 3cf0266..1068e5c 100644 --- a/src/main/java/fr/insee/queen/batch/object/Campaign.java +++ b/src/main/java/fr/insee/queen/batch/object/Campaign.java @@ -2,16 +2,12 @@ import java.util.List; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; - /** * Object XmlCampaign : represent the campaign in XML file * * @author Claudel Benjamin * */ -@Document(collection="campaign") public class Campaign { /** * The id of campaign @@ -25,7 +21,6 @@ public class Campaign { /** * The list of survey unit associated to campaign */ - @DBRef private List questionnaireModels; /** diff --git a/src/main/java/fr/insee/queen/batch/object/Comment.java b/src/main/java/fr/insee/queen/batch/object/Comment.java index 502b795..5084301 100644 --- a/src/main/java/fr/insee/queen/batch/object/Comment.java +++ b/src/main/java/fr/insee/queen/batch/object/Comment.java @@ -3,10 +3,7 @@ import java.util.UUID; import org.json.simple.JSONObject; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -@Document(collection="comment") public class Comment { /** @@ -22,7 +19,6 @@ public class Comment { /** * SurveyUnit associated to the comment */ - @DBRef private SurveyUnit surveyUnit; /** diff --git a/src/main/java/fr/insee/queen/batch/object/Data.java b/src/main/java/fr/insee/queen/batch/object/Data.java index e9f459a..1139d25 100644 --- a/src/main/java/fr/insee/queen/batch/object/Data.java +++ b/src/main/java/fr/insee/queen/batch/object/Data.java @@ -3,10 +3,7 @@ import java.util.UUID; import org.json.simple.JSONObject; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -@Document(collection="data") public class Data { /** * Id of the data @@ -21,7 +18,6 @@ public class Data { /** * SurveyUnit related to the data */ - @DBRef private SurveyUnit surveyUnit; public Data(){ diff --git a/src/main/java/fr/insee/queen/batch/object/Metadata.java b/src/main/java/fr/insee/queen/batch/object/Metadata.java index e9422eb..d9ce23c 100644 --- a/src/main/java/fr/insee/queen/batch/object/Metadata.java +++ b/src/main/java/fr/insee/queen/batch/object/Metadata.java @@ -1,12 +1,8 @@ package fr.insee.queen.batch.object; import java.util.UUID; - -import javax.persistence.Column; - +import jakarta.persistence.Column; import org.json.simple.JSONObject; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; /** * Object Metadata @@ -14,7 +10,6 @@ * @author Claudel Benjamin * */ -@Document(collection="metadata") public class Metadata { /** * The id of the Metadata @@ -30,7 +25,6 @@ public class Metadata { /** * The campaign associated to the Metadata */ - @DBRef private Campaign campaign; public Metadata() { diff --git a/src/main/java/fr/insee/queen/batch/object/Nomenclature.java b/src/main/java/fr/insee/queen/batch/object/Nomenclature.java index 1434c63..75deb3e 100644 --- a/src/main/java/fr/insee/queen/batch/object/Nomenclature.java +++ b/src/main/java/fr/insee/queen/batch/object/Nomenclature.java @@ -1,7 +1,6 @@ package fr.insee.queen.batch.object; import org.json.simple.JSONArray; -import org.springframework.data.mongodb.core.mapping.Document; /** * Entity Nomenclature : represent the entity table in DB @@ -9,7 +8,6 @@ * @author Claudel Benjamin * */ -@Document(collection="nomenclature") public class Nomenclature { private String id; diff --git a/src/main/java/fr/insee/queen/batch/object/Personalization.java b/src/main/java/fr/insee/queen/batch/object/Personalization.java index 84998be..f6dd44f 100644 --- a/src/main/java/fr/insee/queen/batch/object/Personalization.java +++ b/src/main/java/fr/insee/queen/batch/object/Personalization.java @@ -3,17 +3,13 @@ import java.util.UUID; import org.json.simple.JSONArray; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -@Document(collection="personalization") public class Personalization { private UUID id; private JSONArray value; - @DBRef private SurveyUnit surveyUnit; public Personalization() { diff --git a/src/main/java/fr/insee/queen/batch/object/QuestionnaireModel.java b/src/main/java/fr/insee/queen/batch/object/QuestionnaireModel.java index cc8d818..c520ddb 100644 --- a/src/main/java/fr/insee/queen/batch/object/QuestionnaireModel.java +++ b/src/main/java/fr/insee/queen/batch/object/QuestionnaireModel.java @@ -1,7 +1,6 @@ package fr.insee.queen.batch.object; import org.json.simple.JSONObject; -import org.springframework.data.mongodb.core.mapping.Document; /** * Object XmlQuestionnaireModel : represent the questionnaire model in XML file @@ -9,7 +8,6 @@ * @author Claudel Benjamin * */ -@Document(collection="questionnaire_model") public class QuestionnaireModel { /** * The id of questionnaire model diff --git a/src/main/java/fr/insee/queen/batch/object/StateData.java b/src/main/java/fr/insee/queen/batch/object/StateData.java index 5bb822e..6af452a 100644 --- a/src/main/java/fr/insee/queen/batch/object/StateData.java +++ b/src/main/java/fr/insee/queen/batch/object/StateData.java @@ -1,21 +1,17 @@ package fr.insee.queen.batch.object; -import java.util.UUID; +import jakarta.persistence.Column; +import org.springframework.data.annotation.Id; -import javax.persistence.Column; -import javax.persistence.Id; +import java.util.UUID; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; -@Document(collection="state_data") public class StateData { /** * The id of the state data */ @Id - @org.springframework.data.annotation.Id @Column(name = "id") protected UUID id; @@ -40,7 +36,6 @@ public class StateData { /** * The SurveyUnit associated to the StateData */ - @DBRef private SurveyUnit surveyUnit; /** diff --git a/src/main/java/fr/insee/queen/batch/object/SurveyUnit.java b/src/main/java/fr/insee/queen/batch/object/SurveyUnit.java index 8a10ff4..95b830f 100644 --- a/src/main/java/fr/insee/queen/batch/object/SurveyUnit.java +++ b/src/main/java/fr/insee/queen/batch/object/SurveyUnit.java @@ -1,15 +1,11 @@ package fr.insee.queen.batch.object; -import org.springframework.data.mongodb.core.mapping.DBRef; -import org.springframework.data.mongodb.core.mapping.Document; - /** * Object XmlSurveyUnit : represent the survey unit in XML file * * @author Claudel Benjamin * */ -@Document(collection="survey_unit") public class SurveyUnit { /** @@ -20,36 +16,30 @@ public class SurveyUnit { /** * Campaign associated to the survey-unit */ - @DBRef private Campaign campaign; /** * Questionnaire associated to the survey-unit */ - @DBRef private QuestionnaireModel questionnaireModel; /** * The JSON comment of survey unit */ - @DBRef private Comment comment; /** * The JSON data of survey unit */ - @DBRef private Data data; /** * The stateData of the survey-unit */ - @DBRef private StateData stateData; /** * The JSON personalization of survey unit */ - @DBRef private Personalization personalization; /** diff --git a/src/main/java/fr/insee/queen/batch/service/DatabaseService.java b/src/main/java/fr/insee/queen/batch/service/DatabaseService.java index 4a74e7b..077d9ad 100644 --- a/src/main/java/fr/insee/queen/batch/service/DatabaseService.java +++ b/src/main/java/fr/insee/queen/batch/service/DatabaseService.java @@ -13,13 +13,9 @@ import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.ApplicationContext; -import org.springframework.context.annotation.Conditional; import org.springframework.core.env.Environment; -import org.springframework.data.mongodb.core.MongoTemplate; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonJpa; -import fr.insee.queen.batch.config.ConditonMongo; import fr.insee.queen.batch.exception.DataBaseException; /** @@ -36,13 +32,10 @@ public class DatabaseService { @Autowired Environment env; - @Autowired(required=false) + @Autowired @Qualifier("dataSource") DataSource dataSource; - @Autowired(required=false) - MongoTemplate mongoTemplate; - @Autowired String getKeyParadataIdSu; @@ -59,19 +52,14 @@ public class DatabaseService { * @throws SQLException */ public void checkDatabaseAccess() throws DataBaseException, SQLException { - if(dataSource != null) { - checkDatabaseAccessJpa(); - } else { - checkDatabaseAccessMongo(); - } + checkDatabaseAccessJpa(); } - + /** * Check if the tables all exists * @throws DataBaseException * @throws SQLException */ - @Conditional(value= ConditonJpa.class) public void checkDatabaseAccessJpa() throws DataBaseException, SQLException { Connection connection = null; ResultSet rs = null; @@ -96,30 +84,12 @@ public void checkDatabaseAccessJpa() throws DataBaseException, SQLException { } } - /** - * Check if the collections all exists - * @throws DataBaseException - * @throws SQLException - */ - @Conditional(value= ConditonMongo.class) - public void checkDatabaseAccessMongo() throws DataBaseException, SQLException { - Set collectionsNames = mongoTemplate.getCollectionNames(); - for (String tableName : lstTable) { - if (!collectionsNames.contains(tableName)) { - missingTable.add(tableName); - } - } - if (!missingTable.isEmpty()) { - throw new DataBaseException(String.format("Missing tables in database : [%s]", String.join(",", missingTable))); - } - } - /** * This method return true if the persistence type is JPA * @return */ public boolean isJpaDatabase() { - return "JPA".equals(env.getProperty("fr.insee.queen.application.persistenceType")); + return true; } /** diff --git a/src/main/java/fr/insee/queen/batch/service/DeleteService.java b/src/main/java/fr/insee/queen/batch/service/DeleteService.java index b4a298e..a9e2ce4 100644 --- a/src/main/java/fr/insee/queen/batch/service/DeleteService.java +++ b/src/main/java/fr/insee/queen/batch/service/DeleteService.java @@ -90,9 +90,8 @@ public BatchErrorCode deleteSample(String in, String out, BatchErrorCode returnC if(!campaignDao.exist(sample.getCampaign().getId())) { throw new BatchException(String.format("Campaign %s does not exist in database", sample.getCampaign().getId())); } - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(false); - } + connection.setAutoCommit(false); + if(sample.getSurveyUnits() ==null || sample.getSurveyUnits().isEmpty()) { try { // Delete Campaign @@ -109,15 +108,11 @@ public BatchErrorCode deleteSample(String in, String out, BatchErrorCode returnC logger.log(Level.WARN, "Following survey-units deleted successfully : [{}]", String.join(",", lstSu)); logger.log(Level.WARN, "Campaign {} deleted successfully", sample.getCampaign().getId()); } catch (Exception e) { - if(databaseService.isJpaDatabase()) { - connection.rollback(); - connection.setAutoCommit(true); - } + connection.rollback(); + connection.setAutoCommit(true); throw new DataBaseException("Error during delete campaign in DB ... Rollback : " + e.getMessage()); } finally { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(true); - } + connection.setAutoCommit(true); } }else { // Delete SU @@ -140,15 +135,11 @@ public BatchErrorCode deleteSample(String in, String out, BatchErrorCode returnC returnCode = BatchErrorCode.OK_FONCTIONAL_WARNING; } } catch (Exception e) { - if(databaseService.isJpaDatabase()) { - connection.rollback(); - connection.setAutoCommit(true); - } + connection.rollback(); + connection.setAutoCommit(true); throw new DataBaseException("Error during delete survey-units in DB ... Rollback : " + e.getMessage()); } finally { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(true); - } + connection.setAutoCommit(true); } } @@ -158,10 +149,9 @@ public BatchErrorCode deleteSample(String in, String out, BatchErrorCode returnC public BatchErrorCode deleteNomenclature(String pathToNomenclature, String pathToJson, BatchErrorCode returnCode) throws BatchException, SQLException, DataBaseException { List lstNomenclature; - if(databaseService.isJpaDatabase()) { - requiredNomenclatureDao = context.getBean(RequiredNomenclatureDao.class); - connection.setAutoCommit(false); - } + requiredNomenclatureDao = context.getBean(RequiredNomenclatureDao.class); + connection.setAutoCommit(false); + try { lstNomenclature = XmlUtils.xmlToNomenclature(false, pathToNomenclature, pathToJson); @@ -175,15 +165,11 @@ public BatchErrorCode deleteNomenclature(String pathToNomenclature, logger.log(Level.INFO, "Nomenclature {} succesfully deleted", nomenclature.getId()); } } catch (Exception e) { - if(databaseService.isJpaDatabase()) { - connection.rollback(); - connection.setAutoCommit(true); - } + connection.rollback(); + connection.setAutoCommit(true); throw new DataBaseException("Error during delete nomenclatures in DB ... Rollback : " + e.getMessage()); } finally { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(true); - } + connection.setAutoCommit(true); } return returnCode; } diff --git a/src/main/java/fr/insee/queen/batch/service/ExtractionService.java b/src/main/java/fr/insee/queen/batch/service/ExtractionService.java index 489dcf9..03266b5 100644 --- a/src/main/java/fr/insee/queen/batch/service/ExtractionService.java +++ b/src/main/java/fr/insee/queen/batch/service/ExtractionService.java @@ -180,9 +180,7 @@ public void extractCampaign(BatchOption batchOption, Campaign c, List ls */ @SuppressWarnings("resource") public void extractParadata(BatchOption batchOption, Campaign c, String out, List lstSu) throws IOException, SQLException, DataBaseException { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(false); - } + connection.setAutoCommit(false); try { lstSu.stream().forEach(id -> stateDataDao.updateSurveyUnitStateById(id, "TOEXTRACT")); Gson gson = new GsonBuilder().setPrettyPrinting().create(); @@ -213,15 +211,11 @@ public void extractParadata(BatchOption batchOption, Campaign c, String out, Lis lstSu.stream().forEach(id -> stateDataDao.updateSurveyUnitStateById(id, "EXTRACTED")); } catch (Exception e) { logger.log(Level.WARN, "Error message : {}", e.getMessage()); - if(databaseService.isJpaDatabase()) { - connection.rollback(); - connection.setAutoCommit(true); - } + connection.rollback(); + connection.setAutoCommit(true); throw new DataBaseException("Error during update state of SU in DB ... Rollback : " + e.getMessage()); } finally { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(true); - } + connection.setAutoCommit(true); } } diff --git a/src/main/java/fr/insee/queen/batch/service/LoadService.java b/src/main/java/fr/insee/queen/batch/service/LoadService.java index 85d809f..6aff02d 100644 --- a/src/main/java/fr/insee/queen/batch/service/LoadService.java +++ b/src/main/java/fr/insee/queen/batch/service/LoadService.java @@ -110,9 +110,8 @@ public class LoadService { */ public BatchErrorCode loadNomenclature(String pathToNomenclature, String pathToJson, BatchErrorCode returnCode) throws BatchException, SQLException, DataBaseException { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(false); - } + connection.setAutoCommit(false); + NomenclatureDao nomenclatureDao = context.getBean(NomenclatureDao.class); List lstNomenclature; try { @@ -131,15 +130,12 @@ public BatchErrorCode loadNomenclature(String pathToNomenclature, } } } catch (Exception e) { - if(databaseService.isJpaDatabase()) { - connection.rollback(); - connection.setAutoCommit(true); - } + connection.rollback(); + connection.setAutoCommit(true); + throw new DataBaseException("Error during create or update nomenclature in DB ... Rollback : " + e.getMessage()); } finally { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(true); - } + connection.setAutoCommit(true); } if (returnCode == BatchErrorCode.OK) { logger.info("Success to load nomenclatures.xml"); @@ -157,9 +153,8 @@ public BatchErrorCode loadNomenclature(String pathToNomenclature, * @throws DataBaseException */ public BatchErrorCode loadSample(String pathSampleIn, String pathSampleOut, BatchErrorCode returnCode) throws BatchException, SQLException { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(false); - } + connection.setAutoCommit(false); + try { questionnaireModelDao = context.getBean(QuestionnaireModelDao.class); campaignDao = context.getBean(CampaignDao.class); @@ -167,8 +162,7 @@ public BatchErrorCode loadSample(String pathSampleIn, String pathSampleOut, Batc dataDao = context.getBean(DataDao.class); commentDao = context.getBean(CommentDao.class); personalizationDao = context.getBean(PersonalizationDao.class); - if(databaseService.isJpaDatabase()) - requiredNomenclatureDao = context.getBean(RequiredNomenclatureDao.class); + requiredNomenclatureDao = context.getBean(RequiredNomenclatureDao.class); // Retrieve complete xml Objects Sample sample = xmlUtils.createSample(pathSampleIn); returnCode = createOrUpdateCampaign(sample, pathSampleOut, returnCode); @@ -186,9 +180,8 @@ private BatchErrorCode createOrUpdateCampaign(Sample sample, String pathSampleOu // Forcing uppercase sample.getCampaign().setId(sample.getCampaign().getId().toUpperCase()); - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(false); - } + connection.setAutoCommit(false); + if (campaignDao.exist(sample.getCampaign().getId())) { logger.log(Level.INFO, "Campaign {} already exist in database", sample.getCampaign().getId()); returnCode = BatchErrorCode.OK_FONCTIONAL_WARNING; @@ -202,15 +195,11 @@ private BatchErrorCode createOrUpdateCampaign(Sample sample, String pathSampleOu } returnCode = createOrUpdateSurveyUnit(sample, pathSampleOut+"sample", returnCode); } catch (Exception e) { - if(databaseService.isJpaDatabase()) { - connection.rollback(); - connection.setAutoCommit(true); - } + connection.rollback(); + connection.setAutoCommit(true); throw new DataBaseException("Error during create or update campaign in DB ... Rollback : " + e.getMessage()); } finally { - if(databaseService.isJpaDatabase()) { - connection.setAutoCommit(true); - } + connection.setAutoCommit(true); } return returnCode; } diff --git a/src/test/java/fr/insee/queen/batch/TestEndToEndJpa.java b/src/test/java/fr/insee/queen/batch/TestEndToEndJpa.java index 9e4a6c2..ed95fc2 100644 --- a/src/test/java/fr/insee/queen/batch/TestEndToEndJpa.java +++ b/src/test/java/fr/insee/queen/batch/TestEndToEndJpa.java @@ -70,7 +70,6 @@ public static void initContainer() { logger.info("Tests starts"); postgreSQLContainer = new PostgreSQLContainer("postgres") .withDatabaseName("queen").withUsername("queen").withPassword("queen"); - System.setProperty("fr.insee.queen.application.persistenceType", "JPA"); postgreSQLContainer.start(); System.setProperty("fr.insee.queen.persistence.database.host", postgreSQLContainer.getContainerIpAddress()); System.setProperty("fr.insee.queen.persistence.database.port", diff --git a/src/test/java/fr/insee/queen/batch/TestEndToEndMongo.java b/src/test/java/fr/insee/queen/batch/TestEndToEndMongo.java deleted file mode 100644 index 7f8418e..0000000 --- a/src/test/java/fr/insee/queen/batch/TestEndToEndMongo.java +++ /dev/null @@ -1,86 +0,0 @@ -package fr.insee.queen.batch; - -import java.io.IOException; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.junit.ClassRule; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.BeforeEach; -import org.springframework.context.annotation.AnnotationConfigApplicationContext; -import org.testcontainers.containers.MongoDBContainer; - -import com.mongodb.ConnectionString; -import com.mongodb.MongoClientSettings; -import com.mongodb.client.MongoClient; -import com.mongodb.client.MongoClients; -import com.mongodb.client.MongoDatabase; - -import fr.insee.queen.batch.config.ApplicationContext; -import fr.insee.queen.batch.service.DatasetService; - -public class TestEndToEndMongo extends TestEndToEnd { - - private static final Logger logger = LogManager.getLogger(TestEndToEndMongo.class); - - org.springframework.context.ApplicationContext context = new AnnotationConfigApplicationContext(ApplicationContext.class); - - DatasetService datasetService = context.getBean(DatasetService.class); - - /** - * This ClassRule create a PostgresSQL container that represents our database - * for the tests - */ - @ClassRule - public static MongoDBContainer mongoDBContainer; - - @BeforeEach - public void initData() throws Exception { - datasetService.createDataSet(); - } - - @AfterEach - public void dropData() throws Exception { - ConnectionString connectionString = new ConnectionString(String.format("mongodb://%s:%s/%s", mongoDBContainer.getContainerIpAddress(), - mongoDBContainer.getFirstMappedPort(), "test")); - MongoClientSettings mongoClientSettings = MongoClientSettings.builder() - .applyConnectionString(connectionString) - .build(); - MongoClient mongo = MongoClients.create(mongoClientSettings); - MongoDatabase database = mongo.getDatabase("test"); - database.drop(); - } - - /** - * This method initialize the test by starting the PostgreSQL container. It also - * set all the properties correctly from the property file. - * - * @throws IOException - */ - @BeforeAll - public static void initContainer() throws IOException { - logger.info("Tests starts"); - mongoDBContainer = new MongoDBContainer("mongo:4.0.10"); - System.setProperty("fr.insee.queen.application.persistenceType", "MONGODB"); - mongoDBContainer.start(); - System.setProperty("fr.insee.queen.persistence.database.host", mongoDBContainer.getContainerIpAddress()); - System.setProperty("fr.insee.queen.persistence.database.port", - Integer.toString(mongoDBContainer.getFirstMappedPort())); - System.setProperty("fr.insee.queen.persistence.database.schema", "test"); - System.setProperty("fr.insee.queen.persistence.database.user", "queen"); - System.setProperty("fr.insee.queen.persistence.database.password", "queen"); - System.setProperty("fr.insee.queen.folder.in", "src/test/resources/in"); - System.setProperty("fr.insee.queen.folder.out", "src/test/resources/out"); - System.setProperty("spring.data.mongodb.uri", mongoDBContainer.getReplicaSetUrl()); - } - - @AfterAll - public static void closeContainer() { - if(mongoDBContainer!=null && mongoDBContainer.isRunning()) { - mongoDBContainer.close(); - } - } - -} From 1a01deb8b4322a7822ef9126d23551a30fd11637 Mon Sep 17 00:00:00 2001 From: David Darras Date: Tue, 19 Mar 2024 15:05:31 +0100 Subject: [PATCH 3/5] fix: resolve dtd reference vulnerability && try-with-resources - disable access to external dtd to avoid XXE - add try with resources when handling files --- README.md | 7 +- pom.xml | 2 +- .../queen/batch/service/LoadService.java | 3 + .../fr/insee/queen/batch/utils/XmlUtils.java | 137 +++--------------- 4 files changed, 25 insertions(+), 124 deletions(-) diff --git a/README.md b/README.md index 858b421..8d84f84 100644 --- a/README.md +++ b/README.md @@ -5,12 +5,7 @@ Batch using to implement QUEEN DB from xml files. ## Requirements For building and running the application you need: - JDK 21 -- Maven 3 - -# Add lunatic librairy to project -``` shell -mvn install:install-file -Dfile=lib/lunatic-model-2.5.1.jar -DgroupId=fr.insee.lunatic -DartifactId=lunatic-model -Dversion=2.5.1 -Dpackaging=jar -``` +- Maven 3 ## Install and excute unit tests and ent-to-end tests Use the maven clean and maven install diff --git a/pom.xml b/pom.xml index ea13d53..e461cbf 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 fr.insee.queen queen-batch - 4.2.2 + 4.3.0 jar QueenBatch Queen Batch diff --git a/src/main/java/fr/insee/queen/batch/service/LoadService.java b/src/main/java/fr/insee/queen/batch/service/LoadService.java index 6aff02d..98f7c02 100644 --- a/src/main/java/fr/insee/queen/batch/service/LoadService.java +++ b/src/main/java/fr/insee/queen/batch/service/LoadService.java @@ -12,6 +12,7 @@ import java.util.ArrayList; import java.util.List; +import javax.xml.XMLConstants; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -214,6 +215,8 @@ private BatchErrorCode createOrUpdateSurveyUnit(Sample sample, String pathSample Reader reader = new InputStreamReader(inputStream, StandardCharsets.UTF_8); InputSource is = new InputSource(reader); DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); DocumentBuilder db = dbf.newDocumentBuilder(); Document doc = db.parse(is); StreamResult sr = null; diff --git a/src/main/java/fr/insee/queen/batch/utils/XmlUtils.java b/src/main/java/fr/insee/queen/batch/utils/XmlUtils.java index 402e0fa..efebea0 100644 --- a/src/main/java/fr/insee/queen/batch/utils/XmlUtils.java +++ b/src/main/java/fr/insee/queen/batch/utils/XmlUtils.java @@ -1,8 +1,5 @@ package fr.insee.queen.batch.utils; -import fr.insee.lunatic.conversion.JSONCleaner; -import fr.insee.lunatic.conversion.XMLLunaticFlatToJSONLunaticFlatTranslator; -import fr.insee.lunatic.conversion.XMLLunaticToXMLLunaticFlatTranslator; import fr.insee.lunatic.conversion.data.XMLLunaticDataToJSON; import fr.insee.lunatic.conversion.data.XMLLunaticToXSDData; import fr.insee.lunatic.utils.Modele; @@ -13,7 +10,6 @@ import fr.insee.queen.batch.exception.ValidateException; import fr.insee.queen.batch.object.Comment; import fr.insee.queen.batch.object.*; -import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -67,19 +63,16 @@ public class XmlUtils { * @throws IOException */ public static NodeList getXmlNodeFile(String filename, String nodeName) throws IOException { - FileInputStream fis = null; - try { + try(FileInputStream fis = new FileInputStream(filename)) { // an instance of factory that gives a document builder DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); // Compliant dbf.setAttribute(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); // compliant DocumentBuilder db = dbf.newDocumentBuilder(); - fis = new FileInputStream(new File(filename)); Document doc = db.parse(fis); doc.getDocumentElement().normalize(); return doc.getElementsByTagName(nodeName); } catch (Exception e) { - if (fis != null) fis.close(); logger.log(Level.ERROR, e.getMessage(), e); return null; } @@ -96,20 +89,17 @@ public static NodeList getXmlNodeFile(String filename, String nodeName) throws I * @throws BatchException */ public static boolean validateXMLSchema(URL model, String xmlPath) throws ValidateException, IOException, XMLStreamException { - FileInputStream fis = null; XMLStreamReader xmlEncoding = null; - FileReader fr = null; ValidateException ve = null; - try { + try(FileInputStream fis = new FileInputStream(xmlPath); + FileReader fr = new FileReader(xmlPath)) { SchemaFactory factory = SchemaFactory.newInstance(XMLConstants.W3C_XML_SCHEMA_NS_URI); - Schema schema = factory.newSchema(model); factory.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); factory.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); + Schema schema = factory.newSchema(model); Validator validator = schema.newValidator(); validator.setProperty(XMLConstants.ACCESS_EXTERNAL_DTD, ""); validator.setProperty(XMLConstants.ACCESS_EXTERNAL_SCHEMA, ""); - fis = new FileInputStream(new File(xmlPath)); - fr = new FileReader(xmlPath); xmlEncoding = XMLInputFactory.newInstance().createXMLStreamReader(fr); if (xmlEncoding.getCharacterEncodingScheme().equals("UTF8") || xmlEncoding.getCharacterEncodingScheme().equals(StandardCharsets.UTF_8.toString())) { validator.validate(new StreamSource(fis)); @@ -121,11 +111,13 @@ public static boolean validateXMLSchema(URL model, String xmlPath) throws Valida } catch (Exception e) { ve = new ValidateException("Error during validation : " + e.getMessage()); } finally { - if (fis != null) fis.close(); - if (fr != null) fr.close(); - if (xmlEncoding != null) xmlEncoding.close(); + if (xmlEncoding != null) { + xmlEncoding.close(); + } + } + if (ve != null) { + throw ve; } - if (ve != null) throw ve; logger.log(Level.INFO, "{} validate with {}", xmlPath, model); return true; } @@ -146,7 +138,7 @@ public static void validateXMLSchemaData(String xmlPath) throws ValidateExceptio if (!questionnaireXml.hasChildNodes()) { return; } - Transformer transformer = TransformerFactory.newInstance().newTransformer(); + Transformer transformer = getTransformer(); transformer.transform(new DOMSource(questionnaireXml), new StreamResult(Files.createTempFile("tempQuestionnaire", ".xml").toFile())); XMLLunaticToXSDData xmlLunaticToXSDData = new XMLLunaticToXSDData(); // Creating Data.xsd @@ -185,7 +177,7 @@ public static void validateXMLSchemaQuestionnaire(String xmlPath) throws Validat logger.log(Level.INFO, "No questionnaire tag to validate"); return; } - Transformer transformer = TransformerFactory.newInstance().newTransformer(); + Transformer transformer = getTransformer(); transformer.transform(new DOMSource(questionnaireXml), new StreamResult(Files.createTempFile("tempQuestionnaire", ".xml").toFile())); questionnaireXml.setDocumentURI(Constants.TEMP_FOLDER + "/tempQuestionnaire.xml"); SchemaValidator sv = new SchemaValidator(Modele.HIERARCHICAL); @@ -345,54 +337,6 @@ public static Campaign xmlToCampaign(String fileName) throws IOException { return campaign; } - /** - * get questionnaire model in xml file - * - * @param fileName the file name - * @return questionnaire model - * @throws IOException - */ - public static QuestionnaireModel xmlToQuestionnaireModel(String fileName) throws IOException { - QuestionnaireModel questionnaireModel = null; - NodeList lstNodeQuestionnaireModel = getXmlNodeFile(fileName, "QuestionnaireModelId"); - if (lstNodeQuestionnaireModel != null && lstNodeQuestionnaireModel.getLength() == 1) { - for (int itr = 0; itr < lstNodeQuestionnaireModel.getLength(); itr++) { - Node nodeQuestionnaireModel = lstNodeQuestionnaireModel.item(itr); - if (nodeQuestionnaireModel.getNodeType() == Node.ELEMENT_NODE) { - Element e = (Element) nodeQuestionnaireModel; - questionnaireModel = new QuestionnaireModel(); - questionnaireModel.setId(e.getElementsByTagName("Id").item(0).getTextContent()); - questionnaireModel.setLabel(e.getElementsByTagName("Label").item(0).getTextContent()); - } - } - } else { - logger.log(Level.ERROR, "Log error => morethan one questionnaire model in file"); - } - return questionnaireModel; - } - - /** - * get nomenclature in xml file - * - * @param fileName the file name - * @return nomenclature - * @throws IOException - */ - public static List xmlToNomenclatures(String fileName) throws IOException { - List nomenclatures = new ArrayList<>(); - NodeList lstNodeNomenclature = getXmlNodeFile(fileName, "NomenclatureId"); - if (lstNodeNomenclature != null) { - for (int itr = 0; itr < lstNodeNomenclature.getLength(); itr++) { - Node nodeQuestionnaireModel = lstNodeNomenclature.item(itr); - if (nodeQuestionnaireModel.getNodeType() == Node.ELEMENT_NODE) { - Element e = (Element) nodeQuestionnaireModel; - nomenclatures.add(e.getTextContent()); - } - } - } - return nomenclatures; - } - /** * get nomenclature in xml file * @@ -420,37 +364,6 @@ public static Personalization xmlToPersonalization(Node personalization) { return null; } - /** - * This method takes an XML Node and parse it to create - * a JSONObject associated using the Lunatic library - * - * @param fileName - * @param nodeName - * @return JSONObject - * @throws Exception - */ - public static JSONObject lunaticXmlToJSON(String fileName, String nodeName) throws Exception { - NodeList questionnaireXml = getXmlNodeFile(fileName, nodeName); - Node nodeQuestionnaireXml = null; - String questionnaireString = null; - JSONCleaner jsonCleaner = new JSONCleaner(); - if (questionnaireXml != null) { - for (int itr = 0; itr < questionnaireXml.getLength(); itr++) { - nodeQuestionnaireXml = questionnaireXml.item(itr); - questionnaireString = nodeToString(nodeQuestionnaireXml); - } - } - if (StringUtils.isBlank(questionnaireString)) { - return new JSONObject(); - } - XMLLunaticToXMLLunaticFlatTranslator translator = new XMLLunaticToXMLLunaticFlatTranslator(); - XMLLunaticFlatToJSONLunaticFlatTranslator translator2 = new XMLLunaticFlatToJSONLunaticFlatTranslator(); - String stringQuestionnaire = jsonCleaner.clean(translator2.translate(translator.generate(questionnaireString))); - JSONParser parser = new JSONParser(); - return (JSONObject) parser.parse(stringQuestionnaire); - - } - /** * Transform xml to data from SurveyUnit Element * @@ -478,7 +391,7 @@ public static JSONObject dataXmlToJSON(Node data) throws Exception { if (data != null && (data.getChildNodes().getLength() > 1 || (data.getChildNodes().getLength() == 1 && data.getChildNodes().item(0).getNodeType() == Node.ELEMENT_NODE))) { Document dataXml = DocumentBuilderFactory.newInstance().newDocumentBuilder().newDocument(); - Transformer transformer = TransformerFactory.newInstance().newTransformer(); + Transformer transformer = getTransformer(); XMLLunaticDataToJSON xmlLunaticDataToJSON = new XMLLunaticDataToJSON(); File fileDataXml = Files.createTempFile(Constants.TEMP_FOLDER, "tempFileData", ".xml").toFile(); Node copyNode = dataXml.importNode(data, true); @@ -506,22 +419,6 @@ public static JSONObject dataXmlToJSON(Node data) throws Exception { return new JSONObject(); } - /** - * This method convert a Node Object in String - * - * @param node - * @return String of the node Object passed in parameter - * @throws TransformerException - */ - private static String nodeToString(Node node) throws TransformerException { - StringWriter sw = new StringWriter(); - Transformer t = TransformerFactory.newInstance().newTransformer(); - t.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); - t.setOutputProperty(OutputKeys.INDENT, "yes"); - t.transform(new DOMSource(node), new StreamResult(sw)); - return sw.toString(); - } - /** * This method takes a document and an id in entry, it removes the surveyUnit node * identified by its id in the document @@ -544,7 +441,7 @@ public static StreamResult removeSurveyUnitNode(Document doc, String xmlId) thro Node parent = node.getParentNode(); parent.removeChild(node); DOMSource domSource = new DOMSource(doc); - Transformer transformer = TransformerFactory.newInstance().newTransformer(); + Transformer transformer = getTransformer(); StringWriter sw = new StringWriter(); StreamResult sr = new StreamResult(sw); transformer.transform(domSource, sr); @@ -640,4 +537,10 @@ public List xmlToSurveyUnits(String fileName, Campaign campaign) thr return surveyUnits; } + private static Transformer getTransformer() throws TransformerConfigurationException { + TransformerFactory tf = TransformerFactory.newInstance(); + tf.setAttribute(XMLConstants.ACCESS_EXTERNAL_DTD, ""); + tf.setAttribute(XMLConstants.ACCESS_EXTERNAL_STYLESHEET, ""); + return tf.newTransformer(); + } } From d2b0e7e0ce9e140566384ffbb6c6f28fb714441b Mon Sep 17 00:00:00 2001 From: David Darras Date: Wed, 20 Mar 2024 13:17:45 +0100 Subject: [PATCH 4/5] security: update dependency libs --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index e461cbf..79df975 100644 --- a/pom.xml +++ b/pom.xml @@ -14,7 +14,7 @@ 21 6.1.3 2.22.1 - 1.19.5 + 1.19.7 UTF-8 UTF-8 11.5 @@ -65,7 +65,7 @@ org.postgresql postgresql - 42.5.4 + 42.7.3 org.liquibase From 5ead1c2a226c8d04097026e93c36632c8dbb96a0 Mon Sep 17 00:00:00 2001 From: David Darras Date: Wed, 20 Mar 2024 15:50:11 +0100 Subject: [PATCH 5/5] fix: get rid of jpa check --- .../fr/insee/queen/batch/service/DatabaseService.java | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/main/java/fr/insee/queen/batch/service/DatabaseService.java b/src/main/java/fr/insee/queen/batch/service/DatabaseService.java index 077d9ad..2b1cb6a 100644 --- a/src/main/java/fr/insee/queen/batch/service/DatabaseService.java +++ b/src/main/java/fr/insee/queen/batch/service/DatabaseService.java @@ -84,14 +84,6 @@ public void checkDatabaseAccessJpa() throws DataBaseException, SQLException { } } - /** - * This method return true if the persistence type is JPA - * @return - */ - public boolean isJpaDatabase() { - return true; - } - /** * Get the key idSu for paradata * @return