From ba42d39132bc14f83b94123dd9c7bd57e9df8d9c Mon Sep 17 00:00:00 2001 From: Betty Becuwe <77614323+BettyB979@users.noreply.github.com> Date: Tue, 23 Jan 2024 16:04:06 +0100 Subject: [PATCH] Modifications for web batch (#48) * [feat] - add idsu in paradata table * feat: read and delete paradata V2 * fix: get date from state data for platine synchronize * Update ParadataEventDaoJpaImpl.java --- pom.xml | 2 +- .../fr/insee/queen/batch/dao/DataDao.java | 2 +- .../queen/batch/dao/ParadataEventDao.java | 2 + .../batch/dao/impl/jpa/DataDaoJpaImpl.java | 4 +- .../dao/impl/jpa/ParadataEventDaoJpaImpl.java | 52 ++- .../dao/mongo/impl/DataDaoMongoImpl.java | 7 +- .../mongo/impl/ParadataEventDaoMongoImpl.java | 139 +++--- .../queen/batch/object/ParadataEvent.java | 23 + .../batch/service/ExtractionService.java | 4 +- .../fr/insee/queen/batch/utils/PathUtils.java | 441 +++++++++--------- 10 files changed, 378 insertions(+), 298 deletions(-) create mode 100644 src/main/java/fr/insee/queen/batch/object/ParadataEvent.java diff --git a/pom.xml b/pom.xml index 711363c..b1fd2c8 100644 --- a/pom.xml +++ b/pom.xml @@ -5,7 +5,7 @@ 4.0.0 fr.insee.queen queen-batch - 4.1.14 + 4.2.1 jar QueenBatch Queen Batch diff --git a/src/main/java/fr/insee/queen/batch/dao/DataDao.java b/src/main/java/fr/insee/queen/batch/dao/DataDao.java index 61f7520..e105461 100644 --- a/src/main/java/fr/insee/queen/batch/dao/DataDao.java +++ b/src/main/java/fr/insee/queen/batch/dao/DataDao.java @@ -40,7 +40,7 @@ public interface DataDao { * Delete data by a list of SU * @param lstSu */ - void deleteDataBySurveyUnitIds(List lstSu); + int deleteDataBySurveyUnitIds(List lstSu); /** * Delete data by a campaign Id diff --git a/src/main/java/fr/insee/queen/batch/dao/ParadataEventDao.java b/src/main/java/fr/insee/queen/batch/dao/ParadataEventDao.java index 54f12d1..0c25224 100644 --- a/src/main/java/fr/insee/queen/batch/dao/ParadataEventDao.java +++ b/src/main/java/fr/insee/queen/batch/dao/ParadataEventDao.java @@ -24,4 +24,6 @@ public interface ParadataEventDao { * @throws ParseException */ JSONObject findBySurveyUnitId(String suId) throws ParseException; + + void deleteParadataById(String id) throws SQLException; } diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java index 3e6535e..e819852 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/DataDaoJpaImpl.java @@ -61,14 +61,14 @@ public JSONObject getDataBySurveyUnitId(String suId) throws ParseException { * Delete all data for a list of SU */ @Override - public void deleteDataBySurveyUnitIds(List lstSu) { + public int deleteDataBySurveyUnitIds(List lstSu) { String values = lstSu.stream().map(id->"?").collect(Collectors.joining(",")); StringBuilder qStringBuilder = new StringBuilder("DELETE FROM data AS d ") .append("USING survey_unit AS su ") .append("WHERE su.id = d.survey_unit_id ") .append("AND su.id IN (%s)"); String qString = String.format(qStringBuilder.toString(), values); - jdbcTemplate.update(qString, lstSu.toArray()); + return jdbcTemplate.update(qString, lstSu.toArray()); } /** diff --git a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java index e465bc7..7409323 100644 --- a/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/impl/jpa/ParadataEventDaoJpaImpl.java @@ -1,10 +1,11 @@ package fr.insee.queen.batch.dao.impl.jpa; +import java.sql.ResultSet; import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; +import java.util.*; +import fr.insee.queen.batch.dao.mongo.impl.ParadataEventDaoMongoImpl; +import fr.insee.queen.batch.object.ParadataEvent; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -14,6 +15,7 @@ import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.context.annotation.Conditional; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Service; import fr.insee.queen.batch.config.ConditonJpa; @@ -45,24 +47,46 @@ public class ParadataEventDaoJpaImpl implements ParadataEventDao{ public JSONObject findBySurveyUnitId(String suId) throws ParseException { JSONParser parser = new JSONParser(); JSONArray array = new JSONArray(); - List value = new ArrayList<>(); - StringBuilder qString = new StringBuilder("SELECT value FROM paradata_event WHERE EXISTS ( select 1 ") - .append("from jsonb_each_text(value) as t(") - .append(databaseService.getKeyParadataIdSu()) - .append(",v) where v ilike ?)"); - value = jdbcTemplate.queryForList(qString.toString(), new Object[]{suId}, String.class); - for(String val : value) { - JSONObject jsonObjectTemp = (JSONObject) parser.parse(val); - array.add(jsonObjectTemp.get(databaseService.getKeyParadataEvents())); + JSONArray arrayIds = new JSONArray(); + StringBuilder qString = new StringBuilder("SELECT * FROM paradata_event WHERE value->>'").append( + this.databaseService.getKeyParadataIdSu()).append("' = ? "); + String paradataKey = databaseService.getKeyParadataEvents(); + List listParadatas = jdbcTemplate.query(qString.toString(),new ParadataMapper(), new Object[]{suId}); + for(ParadataEvent paradata : listParadatas) { + JSONObject jsonObjectTemp = (JSONObject) parser.parse(paradata.getValue()); + array.add(jsonObjectTemp.get(paradataKey)); + JSONObject item = new JSONObject(); + item.put("id",paradata.getId()); + arrayIds.add(item); } JSONObject jsobObjectClean = new JSONObject(); jsobObjectClean.put("idSu", suId); jsobObjectClean.put("events", array); + jsobObjectClean.put("ids", arrayIds); return jsobObjectClean; } - + + private static final class ParadataMapper implements RowMapper { + public ParadataEvent mapRow(ResultSet rs, int rowNum) throws SQLException { + ParadataEvent infos = new ParadataEvent(); + infos.setId(rs.getString("id")); + infos.setValue(rs.getString("value")); + return infos; + } + + + } + + @Override + public void deleteParadataById(String id) throws SQLException { + + String stringDelete = "DELETE FROM paradata_event WHERE id = ?::uuid"; + jdbcTemplate.update(stringDelete,new Object[]{id}); + } + + /** - * This method create a paradata_event, only use for for testing in the creation of the dataset + * This method create a paradata_event, only use for testing in the creation of the dataset */ public void createParadata(JSONObject paradata) throws SQLException { StringBuilder qString = new StringBuilder("INSERT INTO paradata_event (id, value) VALUES (?, ?)"); diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java index a627855..f3cb029 100644 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/DataDaoMongoImpl.java @@ -2,6 +2,7 @@ import java.sql.SQLException; import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; import org.json.simple.JSONObject; import org.json.simple.parser.ParseException; @@ -58,16 +59,18 @@ public JSONObject getDataBySurveyUnitId(String suId) throws ParseException { * Delete all the data for a list of SU */ @Override - public void deleteDataBySurveyUnitIds(List lstSu) { + public int deleteDataBySurveyUnitIds(List lstSu) { Query query = new Query(); query.addCriteria(Criteria.where("id").in(lstSu)); List suList = mongoTemplate.find(query, SurveyUnit.class, "survey_unit"); + AtomicInteger nb= new AtomicInteger(); suList.stream().forEach(su -> { if(su != null && su.getData() != null) { mongoTemplate.remove(su.getData(), Constants.DATA); + nb.getAndIncrement(); } }); - + return nb.get(); } /** diff --git a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java index 4df50b1..d521230 100644 --- a/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java +++ b/src/main/java/fr/insee/queen/batch/dao/mongo/impl/ParadataEventDaoMongoImpl.java @@ -1,10 +1,8 @@ package fr.insee.queen.batch.dao.mongo.impl; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - +import fr.insee.queen.batch.config.ConditonMongo; +import fr.insee.queen.batch.dao.ParadataEventDao; +import fr.insee.queen.batch.service.DatabaseService; import org.json.simple.JSONArray; import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; @@ -16,69 +14,84 @@ import org.springframework.data.mongodb.core.query.Query; import org.springframework.stereotype.Service; -import fr.insee.queen.batch.config.ConditonMongo; -import fr.insee.queen.batch.dao.ParadataEventDao; -import fr.insee.queen.batch.service.DatabaseService; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; /** * Service for the Paradata-Event entity that implements the interface associated - * @author scorcaud * + * @author scorcaud */ @Service -@Conditional(value= ConditonMongo.class) -public class ParadataEventDaoMongoImpl implements ParadataEventDao{ +@Conditional(value = ConditonMongo.class) +public class ParadataEventDaoMongoImpl implements ParadataEventDao { + + @Autowired + MongoTemplate mongoTemplate; + + @Autowired + DatabaseService databaseService; + + /** + * Method used to retreive all the paradata for a SurveyUnit + * + * @throws ParseException + */ + @SuppressWarnings("unchecked") + @Override + public JSONObject findBySurveyUnitId(String suId) throws ParseException { + JSONParser parser = new JSONParser(); + JSONArray array = new JSONArray(); + List value = new ArrayList<>(); + Query query = new Query(); + query.addCriteria(Criteria.where("value." + databaseService.getKeyParadataIdSu()).is(suId)); + query.fields().include("value"); + query.fields().exclude("_id"); + value = mongoTemplate.find(query, String.class, "paradata_event"); + for (String val : value) { + JSONObject jsonObjectTemp = (JSONObject) parser.parse(val); + JSONObject jsonObjectTemp2 = (JSONObject) jsonObjectTemp.get("value"); + array.add(jsonObjectTemp2.get(databaseService.getKeyParadataEvents())); + } + JSONObject jsobObjectClean = new JSONObject(); + jsobObjectClean.put("idSu", suId); + jsobObjectClean.put("events", array); + return jsobObjectClean; + } + + @Override + public void deleteParadataById(String id) throws SQLException { + ParadataEvent paradataEvent = findById(id); + mongoTemplate.remove(paradataEvent, "paradata_event"); + } + + public ParadataEvent findById(String id) { + Query query = new Query(); + query.addCriteria(Criteria.where("id").is(id)); + return mongoTemplate.findOne(query, ParadataEvent.class, "paradata_event"); + } + + /** + * This method create a paradata_event, only use for for testing in the creation of the dataset + */ + @Override + public void createParadata(JSONObject jsonParadata) throws SQLException { + ParadataEvent paradata = new ParadataEvent(UUID.randomUUID(), jsonParadata); + mongoTemplate.save(paradata, "paradata_event"); + } + + /** + * This class is used only to create Paradata object for testing + */ + static class ParadataEvent { + UUID id; + JSONObject value; - @Autowired - MongoTemplate mongoTemplate; - - @Autowired - DatabaseService databaseService; - - /** - * Method used to retreive all the paradata for a SurveyUnit - * @throws ParseException - */ - @SuppressWarnings("unchecked") - @Override - public JSONObject findBySurveyUnitId(String suId) throws ParseException { - JSONParser parser = new JSONParser(); - JSONArray array = new JSONArray(); - List value = new ArrayList<>(); - Query query = new Query(); - query.addCriteria(Criteria.where("value." +databaseService.getKeyParadataIdSu()).is(suId)); - query.fields().include("value"); - query.fields().exclude("_id"); - value = mongoTemplate.find(query, String.class, "paradata_event"); - for(String val : value) { - JSONObject jsonObjectTemp = (JSONObject) parser.parse(val); - JSONObject jsonObjectTemp2 = (JSONObject) jsonObjectTemp.get("value"); - array.add(jsonObjectTemp2.get(databaseService.getKeyParadataEvents())); - } - JSONObject jsobObjectClean = new JSONObject(); - jsobObjectClean.put("idSu", suId); - jsobObjectClean.put("events", array); - return jsobObjectClean; - } - - /** - * This method create a paradata_event, only use for for testing in the creation of the dataset - */ - @Override - public void createParadata(JSONObject jsonParadata) throws SQLException { - ParadataEvent paradata = new ParadataEvent(UUID.randomUUID(), jsonParadata); - mongoTemplate.save(paradata, "paradata_event"); - } - - /** - * This class is used only to create Paradata object for testing - */ - static class ParadataEvent{ - UUID id; - JSONObject value; - public ParadataEvent(UUID id, JSONObject value) { - this.id = id; - this.value = value; - } - } + public ParadataEvent(UUID id, JSONObject value) { + this.id = id; + this.value = value; + } + } } diff --git a/src/main/java/fr/insee/queen/batch/object/ParadataEvent.java b/src/main/java/fr/insee/queen/batch/object/ParadataEvent.java new file mode 100644 index 0000000..3058bba --- /dev/null +++ b/src/main/java/fr/insee/queen/batch/object/ParadataEvent.java @@ -0,0 +1,23 @@ +package fr.insee.queen.batch.object; + +public class ParadataEvent { + + private String id; + private String value; + + public String getId() { + return id; + } + + public void setId(String id) { + this.id = id; + } + + public String getValue() { + return value; + } + + public void setValue(String value) { + this.value = value; + } +} diff --git a/src/main/java/fr/insee/queen/batch/service/ExtractionService.java b/src/main/java/fr/insee/queen/batch/service/ExtractionService.java index c4b3a75..489dcf9 100644 --- a/src/main/java/fr/insee/queen/batch/service/ExtractionService.java +++ b/src/main/java/fr/insee/queen/batch/service/ExtractionService.java @@ -204,7 +204,9 @@ public void extractParadata(BatchOption batchOption, Campaign c, String out, Lis new File(fileName.toString()).delete(); } FileWriter fileWriter = new FileWriter(fileName.toString(), true); - fileWriter.write(gson.toJson(JsonParser.parseString(paradataEventDao.findBySurveyUnitId(id).toJSONString()))); + JSONObject paradatas = paradataEventDao.findBySurveyUnitId(id); + paradatas.remove("ids"); + fileWriter.write(gson.toJson(JsonParser.parseString(paradatas.toJSONString()))); fileWriter.flush(); fileWriter.close(); } diff --git a/src/main/java/fr/insee/queen/batch/utils/PathUtils.java b/src/main/java/fr/insee/queen/batch/utils/PathUtils.java index db3d4e5..6d27a9b 100644 --- a/src/main/java/fr/insee/queen/batch/utils/PathUtils.java +++ b/src/main/java/fr/insee/queen/batch/utils/PathUtils.java @@ -25,221 +25,234 @@ * - getListFileName * - isFileExist * - getExtensionByStringHandling - * + * * @author Claudel Benjamin - * */ public class PathUtils { - private static final Logger logger = LogManager.getLogger(PathUtils.class); - - private PathUtils() { - throw new IllegalStateException("Utility class"); - } - - /** - * This method is used to create all the directories from the FOLDER_IN property - * @param FOLDER_IN - * @throws FolderException - */ - public static void createFolderTreeIn(String FOLDER_IN) throws FolderException { - if (PathUtils.isDirectoryExist(FOLDER_IN)) { - if (!PathUtils.isDirectoryExist(FOLDER_IN + "/nomenclatures/json")) { - try { - if(!PathUtils.isDirectoryExist(FOLDER_IN + "/nomenclatures/")) { - PathUtils.createDirectory(FOLDER_IN + "/nomenclatures"); - } - if(!PathUtils.isDirectoryExist(FOLDER_IN + "/nomenclatures/json")) { - PathUtils.createDirectory(FOLDER_IN + "/nomenclatures/json"); - } - } catch (Exception e){ - throw new FolderException(String.format("Directory '%s/nomenclatures/json' has not been created", FOLDER_IN)); - } - } - if (!PathUtils.isDirectoryExist(FOLDER_IN + "/sample")) { - try { - PathUtils.createDirectory(FOLDER_IN + "/sample"); - } catch (Exception e){ - throw new FolderException(String.format("Directory '%s/sample' has not been created", FOLDER_IN)); - } - } - if (!PathUtils.isDirectoryExist(FOLDER_IN + "/processing")) { - try { - PathUtils.createDirectory(FOLDER_IN + "/processing"); - } catch (Exception e){ - throw new FolderException(String.format("Directory '%s/processing' has not been created", FOLDER_IN)); - } - } - } else { - throw new FolderException(String.format("Directory '%s' doesn't exist", FOLDER_IN)); - } - } - - /** - * This method is used to create all the directories from the FOLDER_OUT property - * @param FOLDER_OUT - * @throws FolderException - */ - public static void createFolderTreeOut(String FOLDER_OUT) throws FolderException { - if (PathUtils.isDirectoryExist(FOLDER_OUT)) { - if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/nomenclatures/json")) { - throw new FolderException(String.format("Directory '%s/nomenclatures/json' doesn't exist", FOLDER_OUT)); - } - if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/sample")) { - throw new FolderException(String.format("Directory '%s/sample' doesn't exist", FOLDER_OUT)); - } - if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/extractdata")) { - throw new FolderException(String.format("Directory '%s/extractdata' doesn't exist", FOLDER_OUT)); - } - } else { - throw new FolderException(String.format("Directory '%s' doesn't exist", FOLDER_OUT)); - } - } - - /** - * This method create all the directories for the extraction - * @param batchOption - * @param out - * @param campaignId - */ - public static void createFolderTreeExtract(BatchOption batchOption, String out, String campaignId) { - if(!PathUtils.isDirectoryExist(out + "/extractdata/" + campaignId)) { - PathUtils.createDirectory(out + "/extractdata/" + campaignId); - } - if(batchOption.equals(BatchOption.EXTRACTDATA)) { - PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/differential"); - PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/differential/data"); - PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/differential/paradata"); - } - if(batchOption.equals(BatchOption.EXTRACTDATACOMPLETE)) { - PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/complete"); - PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/complete/data"); - PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/complete/paradata"); - } - } - - /** - * This method get the current time for the naming of files - * @return the exact time in String format - */ - public static String getTimestampForPath() { - SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); - Date dateNow = new Date(); - return formatter.format(dateNow); - } - - /** - * This method check if a directory contains an error file created after the step "clean and reset" - * @param pathToDirectory - * @param fileType - * @param extension - * @return boolean - */ - public static boolean isDirContainsErrorFile(Path pathToDirectory, String fileType, String extension) { - boolean isDirContainsErrorFile = false; - List fileNames = getListFileName(pathToDirectory); - for(int i=0; i < fileNames.size(); i++) { - if(fileNames.get(i).contains(fileType) && fileNames.get(i).contains(extension)) { - isDirContainsErrorFile = true; - } - } - return isDirContainsErrorFile; - } - - /** - * Check if a directory exists - * - * @param path path of directory - * @return true if directory exists - */ - public static boolean isDirectoryExist(String path) { - File tmpDir = new File(path); - return tmpDir.exists() && tmpDir.isDirectory(); - } - - /** - * Check if a directory contains a file with a given extension - * - * @param directory directory to check - * @return true if directory contains the given file extension - */ - public static boolean isDirContainsFileExtension(final Path directory, String filename) { - try{ - Iterator i = Files.newDirectoryStream(directory).iterator(); - while (i.hasNext()) { - if(filename.equals(i.next().getFileName().toString())){ - return true; - } - } - } catch (IOException e) { - logger.log(Level.ERROR, e.getMessage(), e); - return false; - } - return false; - } - - /** - * get the list of filenames in a given directory - * - * @param directory directory to explore - * @return the list of filenames in the directory - */ - public static List getListFileName(final Path directory) { - List listFileName = new ArrayList<>(); - try{ - Iterator i = Files.newDirectoryStream(directory).iterator(); - while (i.hasNext()) { - String fileName = i.next().getFileName().toString(); - listFileName.add(fileName); - } - } catch (IOException e) { - logger.log(Level.ERROR, e.getMessage(), e); - } - return listFileName; - } - - /** - * Check if a file exists - * - * @param fileName fileName to check - * @return true if file exists - */ - public static boolean isFileExist(String fileName) { - File tmpDir = new File(fileName); - return tmpDir.exists() && tmpDir.isFile(); - } - - /** - * get the extention of a given filename - * - * @param filename filename to check - * @return the extention of the file - */ - public static Optional getExtensionByStringHandling(String filename) { - return Optional.ofNullable(filename).filter(f -> f.contains(".")) - .map(f -> f.substring(filename.lastIndexOf('.') + 1)); - } - - /** - * get the file name without extention of a given filename - * - * @param filename filename to check - * @return the filename without extention - */ - public static String getFileNameWithoutExtension(String filename) { - return filename.replaceFirst("[.][^.]+$", ""); - } - - /** - * Moving a file to a destination - * @param file - * @param destination - */ - public static void moveFile(String file, String destination) { - new File(file).renameTo(new File(destination)); - } - - public static void createDirectory(String path) { - File sampleDir = new File(path); - sampleDir.mkdir(); - } + private static final Logger logger = LogManager.getLogger(PathUtils.class); + + private PathUtils() { + throw new IllegalStateException("Utility class"); + } + + /** + * This method is used to create all the directories from the FOLDER_IN property + * + * @param FOLDER_IN + * @throws FolderException + */ + public static void createFolderTreeIn(String FOLDER_IN) throws FolderException { + if (PathUtils.isDirectoryExist(FOLDER_IN)) { + if (!PathUtils.isDirectoryExist(FOLDER_IN + "/nomenclatures/json")) { + try { + if (!PathUtils.isDirectoryExist(FOLDER_IN + "/nomenclatures/")) { + PathUtils.createDirectory(FOLDER_IN + "/nomenclatures"); + } + if (!PathUtils.isDirectoryExist(FOLDER_IN + "/nomenclatures/json")) { + PathUtils.createDirectory(FOLDER_IN + "/nomenclatures/json"); + } + } catch (Exception e) { + throw new FolderException(String.format("Directory '%s/nomenclatures/json' has not been created", FOLDER_IN)); + } + } + if (!PathUtils.isDirectoryExist(FOLDER_IN + "/sample")) { + try { + PathUtils.createDirectory(FOLDER_IN + "/sample"); + } catch (Exception e) { + throw new FolderException(String.format("Directory '%s/sample' has not been created", FOLDER_IN)); + } + } + if (!PathUtils.isDirectoryExist(FOLDER_IN + "/processing")) { + try { + PathUtils.createDirectory(FOLDER_IN + "/processing"); + } catch (Exception e) { + throw new FolderException(String.format("Directory '%s/processing' has not been created", FOLDER_IN)); + } + } + } else { + throw new FolderException(String.format("Directory '%s' doesn't exist", FOLDER_IN)); + } + } + + /** + * This method is used to create all the directories from the FOLDER_OUT property + * + * @param FOLDER_OUT + * @throws FolderException + */ + public static void createFolderTreeOut(String FOLDER_OUT) throws FolderException { + if (PathUtils.isDirectoryExist(FOLDER_OUT)) { + if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/nomenclatures/json")) { + try { + if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/nomenclatures/")) { + PathUtils.createDirectory(FOLDER_OUT + "/nomenclatures"); + } + PathUtils.createDirectory(FOLDER_OUT + "/nomenclatures/json"); + + } catch (Exception e) { + throw new FolderException(String.format("Directory '%s/nomenclatures/json' has not been created", FOLDER_OUT)); + } + } + if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/sample")) { + PathUtils.createDirectory(FOLDER_OUT + "/sample"); + } + if (!PathUtils.isDirectoryExist(FOLDER_OUT + "/extractdata")) { + PathUtils.createDirectory(FOLDER_OUT + "/extractdata"); + } + } else { + throw new FolderException(String.format("Directory '%s' doesn't exist", FOLDER_OUT)); + } + } + + /** + * This method create all the directories for the extraction + * + * @param batchOption + * @param out + * @param campaignId + */ + public static void createFolderTreeExtract(BatchOption batchOption, String out, String campaignId) { + if (!PathUtils.isDirectoryExist(out + "/extractdata/" + campaignId)) { + PathUtils.createDirectory(out + "/extractdata/" + campaignId); + } + if (batchOption.equals(BatchOption.EXTRACTDATA)) { + PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/differential"); + PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/differential/data"); + PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/differential/paradata"); + } + if (batchOption.equals(BatchOption.EXTRACTDATACOMPLETE)) { + PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/complete"); + PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/complete/data"); + PathUtils.createDirectory(out + "/extractdata/" + campaignId + "/complete/paradata"); + } + } + + /** + * This method get the current time for the naming of files + * + * @return the exact time in String format + */ + public static String getTimestampForPath() { + SimpleDateFormat formatter = new SimpleDateFormat("yyyyMMddHHmmss"); + Date dateNow = new Date(); + return formatter.format(dateNow); + } + + /** + * This method check if a directory contains an error file created after the step "clean and reset" + * + * @param pathToDirectory + * @param fileType + * @param extension + * @return boolean + */ + public static boolean isDirContainsErrorFile(Path pathToDirectory, String fileType, String extension) { + boolean isDirContainsErrorFile = false; + List fileNames = getListFileName(pathToDirectory); + for (int i = 0; i < fileNames.size(); i++) { + if (fileNames.get(i).contains(fileType) && fileNames.get(i).contains(extension)) { + isDirContainsErrorFile = true; + } + } + return isDirContainsErrorFile; + } + + /** + * Check if a directory exists + * + * @param path path of directory + * @return true if directory exists + */ + public static boolean isDirectoryExist(String path) { + File tmpDir = new File(path); + return tmpDir.exists() && tmpDir.isDirectory(); + } + + /** + * Check if a directory contains a file with a given extension + * + * @param directory directory to check + * @return true if directory contains the given file extension + */ + public static boolean isDirContainsFileExtension(final Path directory, String filename) { + try { + Iterator i = Files.newDirectoryStream(directory).iterator(); + while (i.hasNext()) { + if (filename.equals(i.next().getFileName().toString())) { + return true; + } + } + } catch (IOException e) { + logger.log(Level.ERROR, e.getMessage(), e); + return false; + } + return false; + } + + /** + * get the list of filenames in a given directory + * + * @param directory directory to explore + * @return the list of filenames in the directory + */ + public static List getListFileName(final Path directory) { + List listFileName = new ArrayList<>(); + try { + Iterator i = Files.newDirectoryStream(directory).iterator(); + while (i.hasNext()) { + String fileName = i.next().getFileName().toString(); + listFileName.add(fileName); + } + } catch (IOException e) { + logger.log(Level.ERROR, e.getMessage(), e); + } + return listFileName; + } + + /** + * Check if a file exists + * + * @param fileName fileName to check + * @return true if file exists + */ + public static boolean isFileExist(String fileName) { + File tmpDir = new File(fileName); + return tmpDir.exists() && tmpDir.isFile(); + } + + /** + * get the extention of a given filename + * + * @param filename filename to check + * @return the extention of the file + */ + public static Optional getExtensionByStringHandling(String filename) { + return Optional.ofNullable(filename).filter(f -> f.contains(".")) + .map(f -> f.substring(filename.lastIndexOf('.') + 1)); + } + + /** + * get the file name without extention of a given filename + * + * @param filename filename to check + * @return the filename without extention + */ + public static String getFileNameWithoutExtension(String filename) { + return filename.replaceFirst("[.][^.]+$", ""); + } + + /** + * Moving a file to a destination + * + * @param file + * @param destination + */ + public static void moveFile(String file, String destination) { + new File(file).renameTo(new File(destination)); + } + + public static void createDirectory(String path) { + File sampleDir = new File(path); + sampleDir.mkdir(); + } }