diff --git a/.env.local b/.env.local new file mode 100644 index 0000000..e8fb89c --- /dev/null +++ b/.env.local @@ -0,0 +1,29 @@ +ROOT_LOGGING_LEVEL=debug +APP_LOGGING_LEVEL=debug +WEB_LOGGING_LEVEL=debug + +MONGO_HOST="localhost" +MONGO_PORT=28017 +MONGO_USERNAME=admin +MONGO_PASSWORD=password +MONGO_SSL_ENABLED=false +MONGO_DB_NAME=wallet +MONGO_MIN_POOL_SIZE=0 +MONGO_MAX_POOL_SIZE=20 +MONGO_MAX_IDLE_TIMEOUT_MS=60000 +MONGO_CONNECTION_TIMEOUT_MS=1000 +MONGO_SOCKET_TIMEOUT_MS=10000 +MONGO_SERVER_SELECTION_TIMEOUT_MS=2000 +MONGO_WAITING_QUEUE_MS=2000 +MONGO_HEARTBEAT_FREQUENCY_MS=5000 + +REDIS_HOST="localhost" +REDIS_PORT=6379 +REDIS_PASSWORD="redis_password" +REDIS_SSL_ENABLED=false +WALLET_SESSION_TTL_SECONDS=10 + +EXPIRATION_QUEUE_NAME=pagopa-wallet-cdc-queue +EXPIRATION_QUEUE_TTL_SECONDS=3600 +EXPIRATION_QUEUE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;QueueEndpoint=http://localhost:10001/devstoreaccount1" +EXPIRATION_QUEUE_VISIBILITY_TIMEOUT_SEC=60 \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..b058e5d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,37 @@ +FROM amazoncorretto:21-alpine@sha256:6a98c4402708fe8d16e946b4b5bac396379ec5104c1661e2a27b2b45cf9e2d16 AS build +WORKDIR /workspace/app + +COPY gradlew . +COPY gradle gradle +COPY build.gradle.kts . +COPY settings.gradle.kts . + +COPY eclipse-style.xml eclipse-style.xml +COPY src src +RUN ./gradlew build -x test +RUN mkdir build/extracted && java -Djarmode=layertools -jar build/libs/*.jar extract --destination build/extracted + +FROM amazoncorretto:21-alpine@sha256:6a98c4402708fe8d16e946b4b5bac396379ec5104c1661e2a27b2b45cf9e2d16 + +RUN addgroup --system user && adduser --ingroup user --system user +USER user:user + +WORKDIR /app/ + +ARG EXTRACTED=/workspace/app/build/extracted + +#ELK Agent +ADD --chown=user https://github.com/open-telemetry/opentelemetry-java-instrumentation/releases/download/v2.2.0/opentelemetry-javaagent.jar . + +COPY --from=build --chown=user ${EXTRACTED}/dependencies/ ./ +RUN true +COPY --from=build --chown=user ${EXTRACTED}/spring-boot-loader/ ./ +RUN true +COPY --from=build --chown=user ${EXTRACTED}/snapshot-dependencies/ ./ +RUN true +COPY --from=build --chown=user ${EXTRACTED}/application/ ./ +RUN true + + +ENTRYPOINT ["java","-javaagent:opentelemetry-javaagent.jar", "--enable-preview","org.springframework.boot.loader.JarLauncher"] + diff --git a/README.md b/README.md index e09a5c9..1e8cfa7 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,255 @@ -# pagopa-payment-wallet-cdc-service +# PagoPA Wallet CDC Service + This repository is designed to manage and process events related to changes in wallet status within the PagoPA payment wallet ecosystem. This service listens to events that signal changes (such as wallet creation, modification, or deletion) and ensures that these updates are appropriately handled in a (near) real-time manner. + +- [PagoPA Wallet CDC Service](#pagopa-payment-wallet-cdc-service) + * [Api Documentation ๐Ÿ“–](#api-documentation-) + * [Technology Stack](#technology-stack) + * [Start Project Locally ๐Ÿš€](#start-project-locally-) + + [Prerequisites](#prerequisites) + + [Run docker container](#run-docker-container) + * [Develop Locally ๐Ÿ’ป](#develop-locally-) + + [Prerequisites](#prerequisites-1) + + [Run the project](#run-the-project) + + [Testing ๐Ÿงช](#testing-) + - [Unit testing](#unit-testing) + - [Integration testing](#integration-testing) + - [Performance testing](#performance-testing) + * [Dependency management ๐Ÿ”ง](#dependency-management-) + + [Dependency lock](#dependency-lock) + + [Dependency verification](#dependency-verification) + * [Contributors ๐Ÿ‘ฅ](#contributors-) + + [Maintainers](#maintainers) + +Table of contents generated with +markdown-toc + +--- + +## Api Documentation ๐Ÿ“– + +See +the [OpenAPI 3 here.](https://editor.swagger.io/?url=https://raw.githubusercontent.com/pagopa/pagopa-wallet-service/main/api-spec/wallet-api.yaml) + +--- + +## Technology Stack + +- Kotlin +- Spring Boot + +--- + +## Start Project Locally ๐Ÿš€ + +### Prerequisites + +- docker + +### Populate the environment + +The microservice needs a valid `.env` file in order to be run. + +If you want to start the application without too much hassle, you can just copy `.env.local` with + +```shell +$ cp .env.local .env +``` + +to get a good default configuration. + +If you want to customize the application environment, reference this table: + +| Variable name | Description | type | default | +|----------------------------------------------------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------------------|---------| +| MONGO_HOST | Host where MongoDB instance used to persist wallet data | hostname (string) | | +| MONGO_PORT | Port where MongoDB is bound to in MongoDB host | number | | +| MONGO_USERNAME | MongoDB username used to connect to the database | string | | +| MONGO_PASSWORD | MongoDB password used to connect to the database | string | | +| MONGO_SSL_ENABLED | Whether SSL is enabled while connecting to MongoDB | string | | +| MONGO_DB_NAME | Mongo database name | string | | +| MONGO_MIN_POOL_SIZE | Min amount of connections to be retained into connection pool. See docs * | string | | +| MONGO_MAX_POOL_SIZE | Max amount of connections to be retained into connection pool.See docs * | string | | +| MONGO_MAX_IDLE_TIMEOUT_MS | Max timeout after which an idle connection is killed in milliseconds. See docs * | string | | +| MONGO_CONNECTION_TIMEOUT_MS | Max time to wait for a connection to be opened. See docs * | string | | +| MONGO_SOCKET_TIMEOUT_MS | Max time to wait for a command send or receive before timing out. See docs * | string | | +| MONGO_SERVER_SELECTION_TIMEOUT_MS | Max time to wait for a server to be selected while performing a communication with Mongo in milliseconds. See docs * | string | | +| MONGO_WAITING_QUEUE_MS | Max time a thread has to wait for a connection to be available in milliseconds. See docs * | string | | +| MONGO_HEARTBEAT_FREQUENCY_MS | Hearth beat frequency in milliseconds. This is an hello command that is sent periodically on each active connection to perform an health check. See docs * | string | | +| ROOT_LOGGING_LEVEL | Application root logger level | string | INFO | +| APP_LOGGING_LEVEL | it.pagopa logger level | string | INFO | +| WEB_LOGGING_LEVEL | Web logger level | string | DEBUG | +| REDIS_HOST | Redis host name | string | | +| REDIS_PASSWORD | Redis password | string | | +| REDIS_PORT | Redis port | string | | +| REDIS_SSL_ENABLED | Whether SSL is enabled while connecting to Redis | string | | +| WALLET_SESSION_TTL_SECONDS | Wallet session TTL in seconds | int | | +| EXPIRATION_QUEUE_NAME | Name of expiration queue | string | | +| EXPIRATION_QUEUE_TTL_SECONDS | TTL in seconds for published message | string | | +| EXPIRATION_QUEUE_CONNECTION_STRING | Connection string to storage queue | string | | +| EXPIRATION_QUEUE_VISIBILITY_TIMEOUT_SEC | Visibility timeout in seconds for expired event | | | + +(*): for Mongo connection string options +see [docs](https://www.mongodb.com/docs/drivers/java/sync/v4.3/fundamentals/connection/connection-options/#connection-options) + +### Run docker container + +```shell +$ docker compose up --build +``` + +--- + +## Develop Locally ๐Ÿ’ป + +### Prerequisites + +- git +- gradle +- jdk-17 + +### Run the project + +```shell +$ export $(grep -v '^#' .env.local | xargs) +$ ./gradlew bootRun +``` + +### Testing ๐Ÿงช + +#### Unit testing + +To run the **Junit** tests: + +```shell +$ ./gradlew test +``` + +#### Integration testing + +TODO + +#### Performance testing + +install [k6](https://k6.io/) and then from `./performance-test/src` + +1. `k6 run --env VARS=local.environment.json --env TEST_TYPE=./test-types/load.json main_scenario.js` + +### Dependency management ๐Ÿ”ง + +For support reproducible build this project has the following gradle feature enabled: + +- [dependency lock](https://docs.gradle.org/8.1/userguide/dependency_locking.html) +- [dependency verification](https://docs.gradle.org/8.1/userguide/dependency_verification.html) + +#### Dependency lock + +This feature use the content of `gradle.lockfile` to check the declared dependencies against the locked one. + +If a transitive dependencies have been upgraded the build will fail because of the locked version mismatch. + +The following command can be used to upgrade dependency lockfile: + +```shell +./gradlew dependencies --write-locks +``` + +Running the above command will cause the `gradle.lockfile` to be updated against the current project dependency +configuration + +#### Dependency verification + +This feature is enabled by adding the gradle `./gradle/verification-metadata.xml` configuration file. + +Perform checksum comparison against dependency artifact (jar files, zip, ...) and metadata (pom.xml, gradle module +metadata, ...) used during build +and the ones stored into `verification-metadata.xml` file raising error during build in case of mismatch. + +The following command can be used to recalculate dependency checksum: + +```shell +./gradlew --write-verification-metadata sha256 clean spotlessApply build --no-build-cache --refresh-dependencies +``` + +In the above command the `clean`, `spotlessApply` `build` tasks where chosen to be run +in order to discover all transitive dependencies used during build and also the ones used during +spotless apply task used to format source code. + +The above command will upgrade the `verification-metadata.xml` adding all the newly discovered dependencies' checksum. +Those checksum should be checked against a trusted source to check for corrispondence with the library author published +checksum. + +`/gradlew --write-verification-metadata sha256` command appends all new dependencies to the verification files but does +not remove +entries for unused dependencies. + +This can make this file grow every time a dependency is upgraded. + +To detect and remove old dependencies make the following steps: + +1. Delete, if present, the `gradle/verification-metadata.dryrun.xml` +2. Run the gradle write-verification-metadata in dry-mode (this will generate a verification-metadata-dryrun.xml file + leaving untouched the original verification file) +3. Compare the verification-metadata file and the verification-metadata.dryrun one checking for differences and removing + old unused dependencies + +The 1-2 steps can be performed with the following commands + +```Shell +rm -f ./gradle/verification-metadata.dryrun.xml +./gradlew --write-verification-metadata sha256 clean spotlessApply build --dry-run +``` + +The resulting `verification-metadata.xml` modifications must be reviewed carefully checking the generated +dependencies checksum against official websites or other secure sources. + +If a dependency is not discovered during the above command execution it will lead to build errors. + +You can add those dependencies manually by modifying the `verification-metadata.xml` +file adding the following component: + +```xml + + + + + + + + + + + + + + + +``` + +Add those components at the end of the components list and then run the + +```shell +./gradlew --write-verification-metadata sha256 clean spotlessApply build --no-build-cache --refresh-dependencies +``` + +that will reorder the file with the added dependencies checksum in the expected order. + +Finally, you can add new dependencies both to gradle.lockfile writing verification metadata running + +```shell + ./gradlew dependencies --write-locks --write-verification-metadata sha256 --no-build-cache --refresh-dependencies +``` + +For more information read the +following [article](https://docs.gradle.org/8.1/userguide/dependency_verification.html#sec:checksum-verification) + +## Contributors ๐Ÿ‘ฅ + +Made with โค๏ธ by PagoPA S.p.A. + +### Maintainers + +See `CODEOWNERS` file + diff --git a/azurite/Dockerfile b/azurite/Dockerfile new file mode 100644 index 0000000..b2f04b7 --- /dev/null +++ b/azurite/Dockerfile @@ -0,0 +1,9 @@ +FROM mcr.microsoft.com/azure-cli:2.47.0@sha256:a9fc568020f977fbac633c1c8cb096029214d121e4a03531e74b6a685d161119 + +WORKDIR /usr/queues/script + +COPY ./create-queues.sh ./create-queues.sh + +RUN ["chmod" ,"+x" ,"create-queues.sh"] + +CMD ["sh","create-queues.sh"] \ No newline at end of file diff --git a/azurite/create-queues.sh b/azurite/create-queues.sh new file mode 100644 index 0000000..73915fa --- /dev/null +++ b/azurite/create-queues.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +echo "Analizing env variables" +queues=$(env | grep '^QUEUE_' | sed -n "s/^\(.*\)=.*/\1/p") +for queue in $queues + do + queue_value=$(eval "echo \$$queue") + echo "Found queue into env -> $queue_value, creating queue" + az storage queue create -n $queue_value --connection-string='DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;QueueEndpoint=http://storage:10001/devstoreaccount1' + done \ No newline at end of file diff --git a/build.gradle.kts b/build.gradle.kts index c053326..a3d1d89 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -46,8 +46,19 @@ dependencies { kotlin { compilerOptions { freeCompilerArgs.addAll("-Xjsr305=strict") } } +springBoot { + mainClass.set("it.pagopa.wallet.PagopaPaymentWalletCdcServiceApplicationKt") + buildInfo { + properties { + additional.set(mapOf("description" to (project.description ?: "Default description"))) + } + } +} + tasks.withType { useJUnitPlatform() } +tasks.named("jar") { enabled = false } + configure { kotlin { toggleOffOn() diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..f531cf7 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,89 @@ +services: + pagopa-cdc-wallet: + container_name: pagopa-cdc-wallet + build: + dockerfile: Dockerfile + context: . + env_file: + - ./.env + ports: + - "8200:8080" + depends_on: + - mongo + networks: + - pagopa-cdc-wallet-net + + mongo-express: + container_name: pagopa-cdc-wallet-mongo-express + image: mongo-express@sha256:1b23d7976f0210dbec74045c209e52fbb26d29b2e873d6c6fa3d3f0ae32c2a64 + env_file: ./docker/config/mongo/mongo-express.env + ports: + - "8201:8081" + networks: + - pagopa-cdc-wallet-net + depends_on: + - mongo + + mongo: + container_name: pagopa-cdc-wallet-mongo + image: mongo@sha256:7b225d4ae2df7cf98812f27f5d0d98054efaf9bfd4cc79c7e3e0bed1c1d3bf45 + env_file: ./docker/config/mongo/mongodb.env + ports: + - "28017:27017" + networks: + - pagopa-cdc-wallet-net + healthcheck: + test: echo "db.stats().ok" | mongo --quiet + interval: 10s + + redis: + container_name: wallet-cdc-redis + image: redis@sha256:e422889e156ebea83856b6ff973bfe0c86bce867d80def228044eeecf925592b + command: [ "sh", "-c", 'exec redis-server --requirepass "$$REDIS_PASSWORD"' ] + ports: + - "6379:6379" + networks: + - pagopa-cdc-wallet-net + + redis-insight: + container_name: wallet-redis-insight + image: redislabs/redisinsight@sha256:bbfe06e00282fc051bbfadd49286865e75b359baea62f0d56af0cb0d5684b942 + ports: + - "8001:8001" + networks: + - pagopa-cdc-wallet-net + + + storage: + image: mcr.microsoft.com/azure-storage/azurite:latest@sha256:d449d953a1c5bd1e26c800744a65c451e88f3387b696e02e3e2e79a995c3105b + container_name: pagopa-cdc-wallet-azurite + ports: + - "10000:10000" + - "10001:10001" + - "10002:10002" + volumes: + - $PWD/.data:/opt/azurite/folder + networks: + - pagopa-cdc-wallet-net + + azurite-init: + container_name: pagopa-cdc-wallet-azurite-init + build: + context: azurite/ + dockerfile: Dockerfile + depends_on: + - storage + environment: + QUEUE_EXPIRATION_WALLET: pagopa-cdc-wallet-expiration-queue + networks: + - pagopa-cdc-wallet-net + +networks: + pagopa-cdc-wallet-net: + driver: bridge + ipam: + config: + - subnet: 10.20.0.0/16 + gateway: 10.20.0.1 + + diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..b346f95 --- /dev/null +++ b/docker/README.md @@ -0,0 +1,21 @@ +# Docker Environment ๐Ÿณ + +`run_docker.sh` is a script to launch the image of this microservice and all the dependencies on +Docker. + +## How to use ๐Ÿ’ป + +You can use `local`, `dev`, `uat` or `prod` images + +`sh ./run_docker.sh ` + +You can skip to recreate the images of Docker with `--skip-recreate` + +--- + +โ„น๏ธ _Note_: for **PagoPa ACR** is **required** the login `az acr login -n ` + +โ„น๏ธ _Note_: If you run the script without the parameter, `local` is used as default. + +โ„น๏ธ _Note_: When you select `local`, a new image of this microservice is created from your branch, +but the `dev` dependencies are used. diff --git a/docker/config/mongo/mongo-express.env b/docker/config/mongo/mongo-express.env new file mode 100644 index 0000000..7b4c566 --- /dev/null +++ b/docker/config/mongo/mongo-express.env @@ -0,0 +1,3 @@ +MONGO_INITDB_ROOT_USERNAME=admin +MONGO_INITDB_ROOT_PASSWORD=password +ME_CONFIG_MONGODB_URL=mongodb://$MONGO_INITDB_ROOT_USERNAME:$MONGO_INITDB_ROOT_PASSWORD@pagopa-wallet-mongo:27017/ diff --git a/docker/config/mongo/mongodb.env b/docker/config/mongo/mongodb.env new file mode 100644 index 0000000..e64855e --- /dev/null +++ b/docker/config/mongo/mongodb.env @@ -0,0 +1,2 @@ +MONGO_INITDB_ROOT_USERNAME=admin +MONGO_INITDB_ROOT_PASSWORD=password diff --git a/docker/run_docker.sh b/docker/run_docker.sh new file mode 100644 index 0000000..add0aab --- /dev/null +++ b/docker/run_docker.sh @@ -0,0 +1,63 @@ +# sh ./run_docker.sh --skip-recreate + +ENV=$1 +RECREATE=$2 + +if [ -z "$ENV" ] +then + ENV="local" + echo "No environment specified: local is used." +fi + + +if [ "$ENV" = "local" ]; then + containerRegistry="pagopadcommonacr.azurecr.io" + image="service-local:latest" + echo "Running local image and dev dependencies" +else + + if [ "$ENV" = "dev" ]; then + containerRegistry="pagopadcommonacr.azurecr.io" + echo "Running all dev images" + elif [ "$ENV" = "uat" ]; then + containerRegistry="pagopaucommonacr.azurecr.io" + echo "Running all uat images" + elif [ "$ENV" = "prod" ]; then + containerRegistry="pagopapcommonacr.azurecr.io" + echo "Running all prod images" + else + echo "Error with parameter: use " + exit 1 + fi + + pip3 install yq + repository=$(yq -r '."microservice-chart".image.repository' ../helm/values-$ENV.yaml) + image="${repository}:latest" +fi + + +export containerRegistry=${containerRegistry} +export image=${image} + +stack_name=$(cd .. && basename "$PWD") +if [ "$RECREATE" = "--skip-recreate" ]; then + docker compose -p "${stack_name}" up -d + else + docker compose -p "${stack_name}" up -d --remove-orphans --force-recreate --build +fi + +# waiting the containers +printf 'Waiting for the service' +attempt_counter=0 +max_attempts=50 +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/actuator/info); do + if [ ${attempt_counter} -eq ${max_attempts} ];then + echo "Max attempts reached" + exit 1 + fi + + printf '.' + attempt_counter=$((attempt_counter+1)) + sleep 5 +done +echo 'Service Started'