Skip to content

Commit

Permalink
Merge pull request #27 from pagopa/optimization
Browse files Browse the repository at this point in the history 
Optimized taxonomy retrieval
  • Loading branch information
@aomegax
aomegax authored Nov 27, 2023
2 parents 566405f + 608f6b7 commit c05ba84
Show file tree
Hide file tree
Showing 5 changed files with 57 additions and 57 deletions.
2 changes: 1 addition & 1 deletion integration-test/features/config/config.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
"input_container_name": "input",
"output_container_name": "output",
"csv_blob_name": "taxonomy.csv",
"json_blob_name": "taxonomy.json",
"json_blob_name": "taxonomy_standard.json",
"taxonomy_csv_example": "codice_tipo_ente_creditore;tipo_ente_creditore;progressivo_macro_area_per_ente_creditore;nome_macro_area;descrizione_macro_area;codice_tipologia_servizio;tipo_servizio;motivo_giuridico_della_riscossione;descrizione_tipo_servizio;versione_tassonomia;dati_specifici_di_incasso;data_inizio_validita;data_fine_validita;combinazione_topic_subtopic\n05;SERVIZIO SANITARIO NAZIONALE;01;SERVIZIO SANITARIO NAZIONALE;I servizi sanitari sono quelli di assistenza sanitaria a tutela della salute della popolazione e degli animali. Tali servizi vengono erogati dalle ASL e dagli Ospedali, che a loro volta si distinguono in Presidi ospedalieri e Aziende ospedaliere.;153;incassi servizi sociali;SP;pagamento servizi rivolti a Persone non autosufficienti o portatrici di particolari patologie e disabilità;36;9/0501153SP/;07/03/2023;01/01/2080;Benessere sociale + Altro",
"force_taxomy_example": false
}
Expand Down
2 changes: 1 addition & 1 deletion postman-collection/Taxonomy.postman_collection.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,7 +87,7 @@
"response": []
},
{
"name": "Healthcheck",
"name": "Info - Healthcheck",
"request": {
"method": "GET",
"header": [],
Expand Down
72 changes: 30 additions & 42 deletions src/main/java/it/gov/pagopa/taxonomy/TaxonomyGetFunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
import com.azure.storage.blob.models.BlobErrorCode;
import com.azure.storage.blob.models.BlobStorageException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import com.microsoft.azure.functions.ExecutionContext;
Expand All @@ -22,9 +21,7 @@
import it.gov.pagopa.taxonomy.exception.AppErrorCodeMessageEnum;
import it.gov.pagopa.taxonomy.exception.AppException;
import it.gov.pagopa.taxonomy.model.function.ErrorMessage;
import it.gov.pagopa.taxonomy.model.json.TaxonomyJson;
import it.gov.pagopa.taxonomy.model.json.TaxonomyStandard;
import it.gov.pagopa.taxonomy.model.json.TaxonomyTopicFlag;
import it.gov.pagopa.taxonomy.model.json.TaxonomyMetadata;
import it.gov.pagopa.taxonomy.util.AppConstant;
import it.gov.pagopa.taxonomy.util.AppMessageUtil;
import it.gov.pagopa.taxonomy.util.AppUtil;
Expand All @@ -33,7 +30,6 @@
import java.text.MessageFormat;
import java.time.Instant;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.logging.Level;
Expand Down Expand Up @@ -125,6 +121,7 @@ public HttpResponseMessage getTaxonomy(
HttpStatus.BAD_REQUEST,
payload);
}

if(extension.equalsIgnoreCase("CSV")) {
byte[] taxonomyCsv = getTaxonomyCsv(logger);
HttpResponseMessage.Builder response = request.createResponseBuilder(HttpStatus.OK)
Expand All @@ -134,18 +131,17 @@ public HttpResponseMessage getTaxonomy(
return response.build();
}

TaxonomyJson taxonomyJson = getTaxonomy(logger);
String taxonomyJson = getTaxonomyList(logger, version);
TaxonomyMetadata taxonomyMetadata = getMetadata(logger);

Map<String, String> map = new LinkedHashMap<>();
map.put(AppConstant.RESPONSE_HEADER_UUID, taxonomyJson.getUuid());
map.put(AppConstant.RESPONSE_HEADER_CREATED, taxonomyJson.getCreated().toString());
map.put(AppConstant.RESPONSE_HEADER_UUID, taxonomyMetadata.getUuid());
map.put(AppConstant.RESPONSE_HEADER_CREATED, taxonomyMetadata.getCreated().toString());
map.put(AppConstant.RESPONSE_HEADER_VERSION, version);

String payload = generatePayload(logger, version, taxonomyJson);

return AppUtil.writeResponseWithHeaders(request,
HttpStatus.OK,
payload,
taxonomyJson,
map);

} catch (AppException e) {
Expand Down Expand Up @@ -197,20 +193,34 @@ public HttpResponseMessage getTaxonomy(
}
}

private static TaxonomyJson getTaxonomy(Logger logger) {
try {
msg = MessageFormat.format("Retrieving the json file from the blob storage at: [{0}]", Instant.now());
private static TaxonomyMetadata getMetadata(Logger logger) {
try {
msg = MessageFormat.format("Retrieving the metadata json file from the blob storage at: [{0}]", Instant.now());
logger.info(msg);
String content = getBlobContainerClientOutput()
.getBlobClient(JSON_NAME)
.downloadContent()
.toString();
return getObjectMapper().readValue(content, TaxonomyJson.class);
} catch (JsonProcessingException parsingException) {
.getBlobClient("taxonomy_metadata.json")
.downloadContent()
.toString();
return getObjectMapper().readValue(content, TaxonomyMetadata.class);
} catch (JsonProcessingException exception) {
logger.info("An AppException has occurred");
throw new AppException(parsingException, AppErrorCodeMessageEnum.JSON_PARSING_ERROR);
logger.info("Problem mapping Metadata Object from JSON file");
throw new AppException(exception, AppErrorCodeMessageEnum.JSON_PARSING_ERROR);
}
}
private static String getTaxonomyList(Logger logger, String version) {
final String STANDARD_JSON_NAME = JSON_NAME.split("\\.")[0]+"_standard.json";
final String TOPIC_JSON_NAME = JSON_NAME.split("\\.")[0]+"_topic.json";
msg = MessageFormat.format("Retrieving the {0} json file from the blob storage at: [{1}]", version, Instant.now());
logger.info(msg);
String jsonFile = STANDARD_JSON_NAME;
if(version.equalsIgnoreCase("topicflag"))
jsonFile = TOPIC_JSON_NAME;
return getBlobContainerClientOutput()
.getBlobClient(jsonFile)
.downloadContent()
.toString();
}
private static byte[] getTaxonomyCsv(Logger logger) {
try {
msg=MessageFormat.format("Retrieving the csv file from the blob storage at: [{0}]", Instant.now());
Expand All @@ -227,26 +237,4 @@ private static byte[] getTaxonomyCsv(Logger logger) {
}
}

private static String generatePayload(Logger logger, String version, TaxonomyJson taxonomyJson) {
String payload = null;
if (version.equalsIgnoreCase(VersionEnum.STANDARD.toString())) {
msg = MessageFormat.format("Versioning json id = [{0}] to the {1} version", taxonomyJson.getUuid(), VersionEnum.STANDARD);
logger.info(msg);
List<TaxonomyStandard> taxonomyList = getObjectMapper().convertValue(taxonomyJson.getTaxonomyList(), new TypeReference<>() {
});
payload = AppUtil.getPayload(getObjectMapper(), taxonomyList);
msg = MessageFormat.format("{0} taxonomy retrieved successfully", VersionEnum.STANDARD);
logger.info(msg);
} else if (version.equalsIgnoreCase(VersionEnum.TOPICFLAG.toString())) {
msg = MessageFormat.format("Versioning json id = [{0}] to the {1} version", taxonomyJson.getUuid(), VersionEnum.TOPICFLAG);
logger.info(msg);

List<TaxonomyTopicFlag> taxonomyList = getObjectMapper().convertValue(taxonomyJson.getTaxonomyList(), new TypeReference<>() {
});
payload = AppUtil.getPayload(getObjectMapper(), taxonomyList);
msg = MessageFormat.format("{0} taxonomy retrieved successfully", VersionEnum.TOPICFLAG);
logger.info(msg);
}
return payload;
}
}
33 changes: 24 additions & 9 deletions src/main/java/it/gov/pagopa/taxonomy/UpdateTaxonomy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
import it.gov.pagopa.taxonomy.exception.AppErrorCodeMessageEnum;
import it.gov.pagopa.taxonomy.exception.AppException;
import it.gov.pagopa.taxonomy.model.csv.TaxonomyCsv;
import it.gov.pagopa.taxonomy.model.json.TaxonomyJson;
import it.gov.pagopa.taxonomy.model.json.TaxonomyMetadata;
import it.gov.pagopa.taxonomy.model.json.TaxonomyStandard;
import it.gov.pagopa.taxonomy.model.json.TaxonomyTopicFlag;
import org.modelmapper.ModelMapper;

Expand Down Expand Up @@ -78,11 +79,13 @@ private static ModelMapper getModelMapper() {

public static void updateTaxonomy(Logger logger) {
try {
final String STANDARD_JSON_NAME = JSON_NAME.split("\\.")[0]+"_standard.json";
final String TOPIC_JSON_NAME = JSON_NAME.split("\\.")[0]+"_topic.json";
msg = MessageFormat.format("Download csv file [{0}] from blob at [{1}]", CSV_NAME, Instant.now());
logger.info(msg);
InputStreamReader inputStreamReader = new InputStreamReader(getBlobContainerClientInput().getBlobClient(CSV_NAME).downloadContent().toStream(), StandardCharsets.UTF_8);

msg = MessageFormat.format("Converting [{0}] into [{1}]", CSV_NAME, JSON_NAME);
msg = MessageFormat.format("Converting [{0}] into JSON", CSV_NAME);
logger.info(msg);
List<TaxonomyCsv> taxonomyCsvList = new CsvToBeanBuilder<TaxonomyCsv>(inputStreamReader)
.withSeparator(';')
Expand All @@ -94,19 +97,31 @@ public static void updateTaxonomy(Logger logger) {
Instant now = Instant.now();
String id = UUID.randomUUID().toString();

TaxonomyJson taxonomyJson = TaxonomyJson.builder()
TaxonomyMetadata taxonomyMetadata = TaxonomyMetadata.builder()
.uuid(id)
.created(now)
.taxonomyList(taxonomyCsvList.stream().map(taxonomyCsv ->
getModelMapper().map(taxonomyCsv, TaxonomyTopicFlag.class)
).collect(Collectors.toList()))
.build();

byte[] jsonBytes = getObjectMapper().writeValueAsBytes(taxonomyJson);
msg = MessageFormat.format("Uploading metadata json id = [{0}] created at: [{1}]", id, now);
logger.info(msg);
byte[] jsonBytesMetadata = getObjectMapper().writeValueAsBytes(taxonomyMetadata);
getBlobContainerClientOutput().getBlobClient("taxonomy_metadata.json").upload(BinaryData.fromBytes(jsonBytesMetadata), true);

msg = MessageFormat.format("Uploading standard json id = [{0}] created at: [{1}]", id, now);
List<TaxonomyStandard> taxonomyStandardJson = taxonomyCsvList
.stream().map(taxonomyCsv -> getModelMapper().map(taxonomyCsv, TaxonomyStandard.class)
).collect(Collectors.toList());
byte[] jsonBytesStandard = getObjectMapper().writeValueAsBytes(taxonomyStandardJson);
logger.info(msg);
getBlobContainerClientOutput().getBlobClient(STANDARD_JSON_NAME).upload(BinaryData.fromBytes(jsonBytesStandard), true);

msg = MessageFormat.format("Uploading json id = [{0}] created at: [{1}]", id, now);
msg = MessageFormat.format("Uploading topic json id = [{0}] created at: [{1}]", id, now);
List<TaxonomyTopicFlag> taxonomyTopicFlagJson = taxonomyCsvList
.stream().map(taxonomyCsv -> getModelMapper().map(taxonomyCsv, TaxonomyTopicFlag.class)
).collect(Collectors.toList());
byte[] jsonBytes = getObjectMapper().writeValueAsBytes(taxonomyTopicFlagJson);
logger.info(msg);
getBlobContainerClientOutput().getBlobClient(JSON_NAME).upload(BinaryData.fromBytes(jsonBytes), true);
getBlobContainerClientOutput().getBlobClient(TOPIC_JSON_NAME).upload(BinaryData.fromBytes(jsonBytes), true);

} catch (JsonProcessingException | IllegalStateException parsingException) {
logger.info("An AppException has occurred");
Expand Down
5 changes: 1 addition & 4 deletions ...opa/taxonomy/model/json/TaxonomyJson.java → ...taxonomy/model/json/TaxonomyMetadata.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,19 +5,16 @@
import lombok.*;

import java.time.Instant;
import java.util.List;

@Builder
@Getter
@Setter
@NoArgsConstructor
@AllArgsConstructor
@JsonInclude(JsonInclude.Include.NON_NULL)
public class TaxonomyJson {
public class TaxonomyMetadata {
private String uuid;

@JsonFormat(shape = JsonFormat.Shape.STRING, pattern = "yyyy-MM-dd'T'HH:mm:ss.SSSXXX", timezone = "UTC")
private Instant created;

private List<Taxonomy> taxonomyList;
}

0 comments on commit c05ba84

Please sign in to comment.