Modifications for web batch (#48)

* [feat] - add idsu in paradata table

* feat: read and delete paradata V2

* fix: get date from state data for platine synchronize

* Update ParadataEventDaoJpaImpl.java

pom.xml

@@ -5,7 +5,7 @@
 	<modelVersion>4.0.0</modelVersion>
 	<groupId>fr.insee.queen</groupId>
 	<artifactId>queen-batch</artifactId>
-	<version>4.1.14</version>
+	<version>4.2.1</version>
 	<packaging>jar</packaging>
 	<name>QueenBatch</name>
 	<description>Queen Batch</description>

src/main/java/fr/insee/queen/batch/dao/DataDao.java

@@ -40,7 +40,7 @@ public interface DataDao {
 	 * Delete data by a list of SU
 	 * @param lstSu
 	 */
-	void deleteDataBySurveyUnitIds(List<String> lstSu);
+	int deleteDataBySurveyUnitIds(List<String> lstSu);
 
 	/**
 	 * Delete data by a campaign Id

src/main/java/fr/insee/queen/batch/dao/ParadataEventDao.java

@@ -24,4 +24,6 @@ public interface ParadataEventDao {
 	 * @throws ParseException 
 	 */
 	JSONObject findBySurveyUnitId(String suId) throws ParseException;
+
+	void deleteParadataById(String id) throws SQLException;
 }

src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java

@@ -61,14 +61,14 @@ public JSONObject getDataBySurveyUnitId(String suId) throws ParseException {
 	 * Delete all data for a list of SU
 	 */
 	@Override
-	public void deleteDataBySurveyUnitIds(List<String> lstSu) {
+	public int deleteDataBySurveyUnitIds(List<String> lstSu) {
 		String values = lstSu.stream().map(id->"?").collect(Collectors.joining(","));
 		StringBuilder qStringBuilder = new StringBuilder("DELETE FROM data AS d ")
 		.append("USING survey_unit AS su ")
 		.append("WHERE su.id = d.survey_unit_id ")
 		.append("AND su.id IN (%s)");
 		String qString = String.format(qStringBuilder.toString(), values);
-		jdbcTemplate.update(qString, lstSu.toArray());
+		return jdbcTemplate.update(qString, lstSu.toArray());
 	}
 
 	/**

src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java

@@ -1,10 +1,11 @@
 package fr.insee.queen.batch.dao.impl.jpa;
 
+import java.sql.ResultSet;
 import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
+import java.util.*;
 
+import fr.insee.queen.batch.dao.mongo.impl.ParadataEventDaoMongoImpl;
+import fr.insee.queen.batch.object.ParadataEvent;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -14,6 +15,7 @@
 import org.springframework.beans.factory.annotation.Qualifier;
 import org.springframework.context.annotation.Conditional;
 import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.RowMapper;
 import org.springframework.stereotype.Service;
 
 import fr.insee.queen.batch.config.ConditonJpa;
@@ -45,24 +47,46 @@ public class ParadataEventDaoJpaImpl implements ParadataEventDao{
 	public JSONObject findBySurveyUnitId(String suId) throws ParseException {
 		JSONParser parser = new JSONParser();
 		JSONArray array = new JSONArray();
-		List<String> value = new ArrayList<>();
-		StringBuilder qString = new StringBuilder("SELECT value FROM paradata_event WHERE EXISTS ( select 1 ")
-				.append("from jsonb_each_text(value) as t(")
-				.append(databaseService.getKeyParadataIdSu())
-				.append(",v) where v ilike ?)");
-		value =  jdbcTemplate.queryForList(qString.toString(), new Object[]{suId}, String.class);
-		for(String val : value) {
-			JSONObject jsonObjectTemp = (JSONObject) parser.parse(val);
-			array.add(jsonObjectTemp.get(databaseService.getKeyParadataEvents()));
+		JSONArray arrayIds = new JSONArray();
+		StringBuilder qString = new StringBuilder("SELECT * FROM paradata_event WHERE value->>'").append(
+				this.databaseService.getKeyParadataIdSu()).append("' =  ? ");
+		String paradataKey = databaseService.getKeyParadataEvents();
+		List<ParadataEvent> listParadatas =  jdbcTemplate.query(qString.toString(),new ParadataMapper(), new Object[]{suId});
+		for(ParadataEvent paradata : listParadatas) {
+			JSONObject jsonObjectTemp = (JSONObject) parser.parse(paradata.getValue());
+			array.add(jsonObjectTemp.get(paradataKey));
+			JSONObject item = new JSONObject();
+			item.put("id",paradata.getId());
+			arrayIds.add(item);
 		}
 		JSONObject jsobObjectClean = new JSONObject();
 		jsobObjectClean.put("idSu", suId);
 		jsobObjectClean.put("events", array);
+		jsobObjectClean.put("ids", arrayIds);
 		return jsobObjectClean;
 	}
-
+
+	private static final class ParadataMapper implements RowMapper<ParadataEvent> {
+		public ParadataEvent mapRow(ResultSet rs, int rowNum) throws SQLException {
+			ParadataEvent infos = new ParadataEvent();
+			infos.setId(rs.getString("id"));
+			infos.setValue(rs.getString("value"));
+			return infos;
+		}
+
+
+	}
+
+	@Override
+	public void deleteParadataById(String id) throws SQLException {
+
+		String stringDelete = "DELETE FROM paradata_event WHERE id = ?::uuid";
+		jdbcTemplate.update(stringDelete,new Object[]{id});
+	}
+
+
 	/**
-	 * This method create a paradata_event, only use for for testing in the creation of the dataset 
+	 * This method create a paradata_event, only use for testing in the creation of the dataset
 	 */
 	public void createParadata(JSONObject paradata) throws SQLException {
 		StringBuilder qString = new StringBuilder("INSERT INTO paradata_event (id, value) VALUES (?, ?)");

src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java

@@ -2,6 +2,7 @@
 
 import java.sql.SQLException;
 import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
 
 import org.json.simple.JSONObject;
 import org.json.simple.parser.ParseException;
@@ -58,16 +59,18 @@ public JSONObject getDataBySurveyUnitId(String suId) throws ParseException {
 	 * Delete all the data for a list of SU
 	 */
 	@Override
-	public void deleteDataBySurveyUnitIds(List<String> lstSu) {
+	public int deleteDataBySurveyUnitIds(List<String> lstSu) {
 		Query query = new Query();
 		query.addCriteria(Criteria.where("id").in(lstSu));
 		List<SurveyUnit> suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit");
+		AtomicInteger nb= new AtomicInteger();
 		suList.stream().forEach(su -> {
 			if(su != null && su.getData() != null) {
 				mongoTemplate.remove(su.getData(), Constants.DATA);
+				nb.getAndIncrement();
 			}
 		});
-
+		return nb.get();
 	}
 
 	/**

src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java

@@ -1,10 +1,8 @@
 package fr.insee.queen.batch.dao.mongo.impl;
 
-import java.sql.SQLException;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.UUID;
-
+import fr.insee.queen.batch.config.ConditonMongo;
+import fr.insee.queen.batch.dao.ParadataEventDao;
+import fr.insee.queen.batch.service.DatabaseService;
 import org.json.simple.JSONArray;
 import org.json.simple.JSONObject;
 import org.json.simple.parser.JSONParser;
@@ -16,69 +14,84 @@
 import org.springframework.data.mongodb.core.query.Query;
 import org.springframework.stereotype.Service;
 
-import fr.insee.queen.batch.config.ConditonMongo;
-import fr.insee.queen.batch.dao.ParadataEventDao;
-import fr.insee.queen.batch.service.DatabaseService;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.UUID;
 
 /**
  * Service for the Paradata-Event entity that implements the interface associated
- * @author scorcaud
  *
+ * @author scorcaud
  */
 @Service
-@Conditional(value= ConditonMongo.class)
-public class ParadataEventDaoMongoImpl implements ParadataEventDao{
+@Conditional(value = ConditonMongo.class)
+public class ParadataEventDaoMongoImpl implements ParadataEventDao {
+
+    @Autowired
+    MongoTemplate mongoTemplate;
+
+    @Autowired
+    DatabaseService databaseService;
+
+    /**
+     * Method used to retreive all the paradata for a SurveyUnit
+     *
+     * @throws ParseException
+     */
+    @SuppressWarnings("unchecked")
+    @Override
+    public JSONObject findBySurveyUnitId(String suId) throws ParseException {
+        JSONParser parser = new JSONParser();
+        JSONArray array = new JSONArray();
+        List<String> value = new ArrayList<>();
+        Query query = new Query();
+        query.addCriteria(Criteria.where("value." + databaseService.getKeyParadataIdSu()).is(suId));
+        query.fields().include("value");
+        query.fields().exclude("_id");
+        value = mongoTemplate.find(query, String.class, "paradata_event");
+        for (String val : value) {
+            JSONObject jsonObjectTemp = (JSONObject) parser.parse(val);
+            JSONObject jsonObjectTemp2 = (JSONObject) jsonObjectTemp.get("value");
+            array.add(jsonObjectTemp2.get(databaseService.getKeyParadataEvents()));
+        }
+        JSONObject jsobObjectClean = new JSONObject();
+        jsobObjectClean.put("idSu", suId);
+        jsobObjectClean.put("events", array);
+        return jsobObjectClean;
+    }
+
+    @Override
+    public void deleteParadataById(String id) throws SQLException {
+        ParadataEvent paradataEvent =  findById(id);
+        mongoTemplate.remove(paradataEvent, "paradata_event");
+    }
+
+    public ParadataEvent findById(String id) {
+        Query query = new Query();
+        query.addCriteria(Criteria.where("id").is(id));
+        return mongoTemplate.findOne(query, ParadataEvent.class, "paradata_event");
+    }
+
+    /**
+     * This method create a paradata_event, only use for for testing in the creation of the dataset
+     */
+    @Override
+    public void createParadata(JSONObject jsonParadata) throws SQLException {
+        ParadataEvent paradata = new ParadataEvent(UUID.randomUUID(), jsonParadata);
+        mongoTemplate.save(paradata, "paradata_event");
+    }
+
+    /**
+     * This class is used only to create Paradata object for testing
+     */
+    static class ParadataEvent {
+        UUID id;
+        JSONObject value;
 
-	@Autowired
-	MongoTemplate mongoTemplate;
-
-	@Autowired
-	DatabaseService databaseService;
-
-	/**
-	 * Method used to retreive all the paradata for a SurveyUnit
-	 * @throws ParseException 
-	 */
-	@SuppressWarnings("unchecked")
-	@Override
-	public JSONObject findBySurveyUnitId(String suId) throws ParseException {
-		JSONParser parser = new JSONParser();
-		JSONArray array = new JSONArray();
-		List<String> value = new ArrayList<>();
-		Query query = new Query();
-		query.addCriteria(Criteria.where("value." +databaseService.getKeyParadataIdSu()).is(suId));
-		query.fields().include("value");
-		query.fields().exclude("_id");
-		value =  mongoTemplate.find(query, String.class, "paradata_event");
-		for(String val : value) {
-			JSONObject jsonObjectTemp = (JSONObject) parser.parse(val);
-			JSONObject jsonObjectTemp2 = (JSONObject) jsonObjectTemp.get("value");
-			array.add(jsonObjectTemp2.get(databaseService.getKeyParadataEvents()));
-		}
-		JSONObject jsobObjectClean = new JSONObject();
-		jsobObjectClean.put("idSu", suId);
-		jsobObjectClean.put("events", array);
-		return jsobObjectClean;
-	}
-
-	/**
-	 * This method create a paradata_event, only use for for testing in the creation of the dataset 
-	 */
-	@Override
-	public void createParadata(JSONObject jsonParadata) throws SQLException {
-		ParadataEvent paradata = new ParadataEvent(UUID.randomUUID(), jsonParadata);
-		mongoTemplate.save(paradata, "paradata_event");
-	}
-
-	/**
-	 * This class is used only to create Paradata object for testing
-	 */
-	static class ParadataEvent{
-		UUID id;
-		JSONObject value;
-		public ParadataEvent(UUID id, JSONObject value) {
-			this.id = id;
-			this.value = value;
-		}
-	}
+        public ParadataEvent(UUID id, JSONObject value) {
+            this.id = id;
+            this.value = value;
+        }
+    }
 }

src/main/java/fr/insee/queen/batch/object/ParadataEvent.java

@@ -0,0 +1,23 @@
+package fr.insee.queen.batch.object;
+
+public class ParadataEvent {
+
+    private String id;
+    private String value;
+
+    public String getId() {
+        return id;
+    }
+
+    public void setId(String id) {
+        this.id = id;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(String value) {
+        this.value = value;
+    }
+}

src/main/java/fr/insee/queen/batch/service/ExtractionService.java

@@ -204,7 +204,9 @@ public void extractParadata(BatchOption batchOption, Campaign c, String out, Lis
 					new File(fileName.toString()).delete();
 				}
 				FileWriter fileWriter = new FileWriter(fileName.toString(), true);
-				fileWriter.write(gson.toJson(JsonParser.parseString(paradataEventDao.findBySurveyUnitId(id).toJSONString())));
+				JSONObject paradatas = paradataEventDao.findBySurveyUnitId(id);
+				paradatas.remove("ids");
+				fileWriter.write(gson.toJson(JsonParser.parseString(paradatas.toJSONString())));
 				fileWriter.flush();
 				fileWriter.close();
 			}