From ba59c81dd097f2e79c7f370822c4a5a57c6bb5e3 Mon Sep 17 00:00:00 2001 From: asavershin Date: Mon, 6 May 2024 12:12:19 +0300 Subject: [PATCH 1/7] Add workers --- pom.xml | 11 ++ worker/.env | 17 ++ worker/Dockerfile-rotate | 7 + worker/docker-compose-rotate.yml | 52 +++++++ worker/pom.xml | 61 ++++++++ .../asavershin/images/BlackWhiteWorker.java | 71 +++++++++ .../com/github/asavershin/images/Main.java | 11 ++ .../github/asavershin/images/RotateImage.java | 73 +++++++++ .../com/github/asavershin/images/Task.java | 15 ++ .../com/github/asavershin/images/Worker.java | 8 + .../asavershin/images/WorkerManager.java | 5 + .../asavershin/images/WorkerManagerImpl.java | 44 ++++++ .../asavershin/images/config/KafkaConf.java | 140 +++++++++++++++++ .../asavershin/images/config/MinIOConf.java | 39 +++++ .../images/config/MinIOProperties.java | 40 +++++ .../images/config/ProducerProp.java | 19 +++ .../asavershin/images/config/RedisConf.java | 36 +++++ .../asavershin/images/domain/ImageId.java | 50 ++++++ .../asavershin/images/domain/RequestId.java | 53 +++++++ .../github/asavershin/images/in/Consumer.java | 53 +++++++ .../images/out/CacheRepository.java | 22 +++ .../images/out/CacheRepositoryIml.java | 36 +++++ .../asavershin/images/out/FileException.java | 12 ++ .../asavershin/images/out/MinioService.java | 11 ++ .../images/out/MinioServiceIml.java | 146 ++++++++++++++++++ .../asavershin/images/out/Producer.java | 29 ++++ .../main/resources/application-blackwhite.yml | 10 ++ .../src/main/resources/application-rotate.yml | 10 ++ worker/src/main/resources/application.yml | 36 +++++ 29 files changed, 1117 insertions(+) create mode 100644 worker/.env create mode 100644 worker/Dockerfile-rotate create mode 100644 worker/docker-compose-rotate.yml create mode 100644 worker/pom.xml create mode 100644 worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java create mode 100644 worker/src/main/java/com/github/asavershin/images/Main.java create mode 100644 worker/src/main/java/com/github/asavershin/images/RotateImage.java create mode 100644 worker/src/main/java/com/github/asavershin/images/Task.java create mode 100644 worker/src/main/java/com/github/asavershin/images/Worker.java create mode 100644 worker/src/main/java/com/github/asavershin/images/WorkerManager.java create mode 100644 worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java create mode 100644 worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java create mode 100644 worker/src/main/java/com/github/asavershin/images/config/MinIOConf.java create mode 100644 worker/src/main/java/com/github/asavershin/images/config/MinIOProperties.java create mode 100644 worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java create mode 100644 worker/src/main/java/com/github/asavershin/images/config/RedisConf.java create mode 100644 worker/src/main/java/com/github/asavershin/images/domain/ImageId.java create mode 100644 worker/src/main/java/com/github/asavershin/images/domain/RequestId.java create mode 100644 worker/src/main/java/com/github/asavershin/images/in/Consumer.java create mode 100644 worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java create mode 100644 worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java create mode 100644 worker/src/main/java/com/github/asavershin/images/out/FileException.java create mode 100644 worker/src/main/java/com/github/asavershin/images/out/MinioService.java create mode 100644 worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java create mode 100644 worker/src/main/java/com/github/asavershin/images/out/Producer.java create mode 100644 worker/src/main/resources/application-blackwhite.yml create mode 100644 worker/src/main/resources/application-rotate.yml create mode 100644 worker/src/main/resources/application.yml diff --git a/pom.xml b/pom.xml index 8a09635..cda65f6 100644 --- a/pom.xml +++ b/pom.xml @@ -16,6 +16,7 @@ api + worker @@ -45,6 +46,16 @@ ${org.projectlombok.version} true + + org.springframework.kafka + spring-kafka + + + + org.springframework.kafka + spring-kafka-test + test + diff --git a/worker/.env b/worker/.env new file mode 100644 index 0000000..4f33ad5 --- /dev/null +++ b/worker/.env @@ -0,0 +1,17 @@ +SERVER_PORT=8081 + +# MINIO +MINIO_BUCKET=files +MINIO_TTL_PREFIX=temporary/ +MINIO_EXPIRATION=1 +MINIO_URL=http://minio-api:9000 +MINIO_ROOT_USER=minioadmin +MINIO_ROOT_PASSWORD=minioadmin +MINIO_CONSOLE_PORT=9090 +MINIO_PORT=9000 + +# REDIS +REDIS_HOST=redis-rotate +REDIS_PORT=6379 +REDIS_PASSWORD=cGFzc3dvcmxk +REDIS_CACHE_TIME=86400000 diff --git a/worker/Dockerfile-rotate b/worker/Dockerfile-rotate new file mode 100644 index 0000000..623dc7e --- /dev/null +++ b/worker/Dockerfile-rotate @@ -0,0 +1,7 @@ +FROM openjdk:21 + +WORKDIR /app + +COPY target/*.jar app.jar + +CMD ["java", "-jar", "-Dspring.profiles.active=rotate", "app.jar"] \ No newline at end of file diff --git a/worker/docker-compose-rotate.yml b/worker/docker-compose-rotate.yml new file mode 100644 index 0000000..8688dab --- /dev/null +++ b/worker/docker-compose-rotate.yml @@ -0,0 +1,52 @@ +networks: + kafka-net: + name: kafka-net + driver: bridge + api-net: + name: api-net + driver: bridge + +services: + + backend-worker-rotate: + container_name: backend-worker-rotate + networks: + - kafka-net + - api-net + build: + context: . + dockerfile: Dockerfile-rotate + ports: + - 8081:8081 + env_file: + - .env + + redis-rotate: + networks: + - api-net + image: redis:7.2-rc-alpine + restart: always + container_name: redis-rotate + ports: + - '6380:6379' + command: redis-server --save 20 1 --loglevel debug --requirepass ${REDIS_PASSWORD} + volumes: + - redis-rotate-data:/data + + minio-api: + networks: + - api-net + image: minio/minio:RELEASE.2024-02-14T21-36-02Z + container_name: minio-api + env_file: + - .env + command: server ~/minio --console-address :9090 + ports: + - '9090:9090' + - '9000:9000' + volumes: + - minio-api-data:/minio + +volumes: + minio-api-data: + redis-rotate-data: diff --git a/worker/pom.xml b/worker/pom.xml new file mode 100644 index 0000000..d47d4f4 --- /dev/null +++ b/worker/pom.xml @@ -0,0 +1,61 @@ + + + 4.0.0 + + com.github.asavershin.images + images + 0.0.1-SNAPSHOT + + + worker + + + 21 + 21 + UTF-8 + 8.5.7 + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-validation + + + io.minio + minio + ${minio.version} + + + + org.springframework.boot + spring-boot-starter-data-redis + + + + + + + + + org.springframework.boot + spring-boot-maven-plugin + + + + org.projectlombok + lombok + + + + + + + + \ No newline at end of file diff --git a/worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java b/worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java new file mode 100644 index 0000000..6d4046f --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java @@ -0,0 +1,71 @@ +package com.github.asavershin.images; + +import com.github.asavershin.images.config.MinIOProperties; +import com.github.asavershin.images.out.MinioService; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; + +import javax.imageio.ImageIO; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.UUID; +@Service +@Profile("blackwhite") +@RequiredArgsConstructor +@Slf4j +public class BlackWhiteWorker implements Worker{ + private final MinioService imageStorage; + private final MinIOProperties props; + + @Override + public String doWork(final String imageUUID, final boolean lastWorker) { + InputStream is = imageStorage.getFile(imageUUID); + byte[] rotatedImageData = convertToGrayscale(is); + var id = UUID.randomUUID().toString(); + if (!lastWorker) { + id = props.getTtlprefix() + id; + } + imageStorage.saveFile(rotatedImageData, id); + return id; + } + + @Override + public String whoAmI() { + return "BLACKWHITE"; + } + + private byte[] convertToGrayscale(final InputStream imageData) { + try { + BufferedImage originalImage = ImageIO.read(imageData); + int width = originalImage.getWidth(); + int height = originalImage.getHeight(); + + BufferedImage grayscaleImage = new BufferedImage(width, height, BufferedImage.TYPE_BYTE_GRAY); + + // Проход по каждому пикселю и преобразование его в черно-белый + for (int y = 0; y < height; y++) { + for (int x = 0; x < width; x++) { + int rgb = originalImage.getRGB(x, y); + int r = (rgb >> 16) & 0xFF; + int g = (rgb >> 8) & 0xFF; + int b = rgb & 0xFF; + int gray = (int) (0.2126 * r + 0.7152 * g + 0.0722 * b); + int grayValue = (gray << 16) | (gray << 8) | gray; + grayscaleImage.setRGB(x, y, grayValue); + } + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageIO.write(grayscaleImage, "jpg", baos); + return baos.toByteArray(); + } catch (IOException e) { + log.error("Error converting image to grayscale: {}", e.getMessage()); + // Обработка ошибок, например, выброс исключения или возврат пустого массива + return new byte[0]; + } + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/Main.java b/worker/src/main/java/com/github/asavershin/images/Main.java new file mode 100644 index 0000000..6f94a28 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/Main.java @@ -0,0 +1,11 @@ +package com.github.asavershin.images; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class Main { + public static void main(final String[] args) { + SpringApplication.run(Main.class, args); + } +} \ No newline at end of file diff --git a/worker/src/main/java/com/github/asavershin/images/RotateImage.java b/worker/src/main/java/com/github/asavershin/images/RotateImage.java new file mode 100644 index 0000000..79f4d7f --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/RotateImage.java @@ -0,0 +1,73 @@ +package com.github.asavershin.images; + +import com.github.asavershin.images.config.MinIOProperties; +import com.github.asavershin.images.out.MinioService; +import io.minio.MinioProperties; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; +import org.springframework.stereotype.Service; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.geom.AffineTransform; +import java.awt.image.AffineTransformOp; +import java.awt.image.BufferedImage; +import java.awt.image.DataBufferByte; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.util.UUID; + +@Service +@Profile("rotate") +@RequiredArgsConstructor +@Slf4j +public class RotateImage implements Worker{ + private final MinioService imageStorage; + private final MinIOProperties props; + + @Override + public String doWork(final String imageUUID, final boolean lastWorker) { + InputStream is = imageStorage.getFile(imageUUID); + byte[] rotatedImageData = rotateImage90Degrees(is); + + var id = UUID.randomUUID().toString(); + if (!lastWorker) { + id = props.getTtlprefix() + id; + } + imageStorage.saveFile(rotatedImageData, id); + return id; + } + + @Override + public String whoAmI() { + return "ROTATE"; + } + + private byte[] rotateImage90Degrees(final InputStream imageData) { + try { + BufferedImage originalImage = ImageIO.read(imageData); + int width = originalImage.getWidth(); + int height = originalImage.getHeight(); + + BufferedImage rotatedImage = new BufferedImage(height, width, originalImage.getType()); + + for (int i = 0; i < width; i++) { + for (int j = 0; j < height; j++) { + rotatedImage.setRGB(height - 1 - j, i, originalImage.getRGB(i, j)); + } + } + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageIO.write(rotatedImage, "jpg", baos); + return baos.toByteArray(); + } catch (IOException e) { + log.error("Error rotating image: {}", e.getMessage()); + // Обработка ошибок, например, выброс исключения или возврат пустого массива + return new byte[0]; + } + } +} \ No newline at end of file diff --git a/worker/src/main/java/com/github/asavershin/images/Task.java b/worker/src/main/java/com/github/asavershin/images/Task.java new file mode 100644 index 0000000..761e6e4 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/Task.java @@ -0,0 +1,15 @@ +package com.github.asavershin.images; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +import java.util.List; +@Data +@AllArgsConstructor +@NoArgsConstructor +public class Task { + private String imageId; + private String requestId; + private List filters; +} diff --git a/worker/src/main/java/com/github/asavershin/images/Worker.java b/worker/src/main/java/com/github/asavershin/images/Worker.java new file mode 100644 index 0000000..77df9f8 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/Worker.java @@ -0,0 +1,8 @@ +package com.github.asavershin.images; + +import java.io.IOException; + +public interface Worker { + String doWork(String imageId, boolean lastWorker); + String whoAmI(); +} diff --git a/worker/src/main/java/com/github/asavershin/images/WorkerManager.java b/worker/src/main/java/com/github/asavershin/images/WorkerManager.java new file mode 100644 index 0000000..b09ac16 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/WorkerManager.java @@ -0,0 +1,5 @@ +package com.github.asavershin.images; + +public interface WorkerManager { + Task start(Task filters); +} diff --git a/worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java b/worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java new file mode 100644 index 0000000..97e8257 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java @@ -0,0 +1,44 @@ +package com.github.asavershin.images; + +import com.github.asavershin.images.out.CacheRepository; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Service; + +import java.util.Objects; + +@Service +@RequiredArgsConstructor +public class WorkerManagerImpl implements WorkerManager { + private final Worker worker; + private final CacheRepository cache; + @Value("${cache-expiration}") + @NotNull + private Long cacheExp; + + @Override + public Task start(final Task task) { + var cached = cache.getCache( + task.getRequestId() + task.getImageId()); + if (task.getFilters().isEmpty() + || !Objects.equals( + task.getFilters().get(0), + worker.whoAmI()) + || cached != null + ) { + return null; + } + + var imageId = worker.doWork(task.getImageId(), + task.getFilters().size() == 1 + ); + cache.addCache( + task.getRequestId() + imageId, + "", + cacheExp + ); + task.getFilters().remove(0); + return task; + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java b/worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java new file mode 100644 index 0000000..bb9f442 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java @@ -0,0 +1,140 @@ +package com.github.asavershin.images.config; + +import com.github.asavershin.images.Task; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.RoundRobinPartitioner; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; + +import java.util.Map; +import java.util.function.Consumer; + +@Configuration +@RequiredArgsConstructor +public class KafkaConf { + /** + * Static final string constant for the Kafka template bean name. + */ + public static final String ALL_ACKS_KAFKA_TEMPLATE = "allAcksKafkaTemplate"; + /** + * Injected producer properties. + */ + @NotNull(message = "KafkaConf is invalid: prodProp is null") + private final ProducerProp prodProp; + /** + * Injected Kafka properties. + */ + @NotNull(message = "KafkaConf is invalid: KafkaProperties is null") + private final KafkaProperties properties; + /** + * Creates a new Kafka topic. + * + * @param topic the name of the topic + * @param partitions the number of partitions + * @param replicas the number of replicas + * @return a new Kafka topic + */ + @Bean + public NewTopic wipTopic( + final @Value("${app.wiptopic}") String topic, + final @Value("${app.partitions}") int partitions, + final @Value("${app.replicas}") short replicas) { + + return new NewTopic(topic, partitions, replicas); + } + /** + * Creates a new Kafka topic with the specified name, + * number of partitions, and replicas. + * + * @param topic the name of the topic + * @param partitions the number of partitions for the topic + * @param replicas the number of replicas for the topic + * @return a new Kafka topic with the specified configuration + */ + @Bean + public NewTopic doneTopic( + final @Value("${app.donetopic}") String topic, + final @Value("${app.partitions}") int partitions, + final @Value("${app.replicas}") short replicas) { + + return new NewTopic(topic, partitions, replicas); + } + /** + * Creates a new Kafka template with all acknowledgments. + * + * @return a new Kafka template with all acknowledgments + */ + @Bean(ALL_ACKS_KAFKA_TEMPLATE) + public KafkaTemplate allAcksKafkaTemplate() { + return new KafkaTemplate<>(producerFactory( + props -> { + props.put(ProducerConfig.ACKS_CONFIG, "all"); + props.put(ProducerConfig.RETRIES_CONFIG, + prodProp.getRetries()); + }) + ); + } + + private ProducerFactory producerFactory( + final Consumer> enchanter + ) { + var props = properties.buildProducerProperties(null); + // Работаем со строками + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + JsonSerializer.class); + // Партиция одна, так что все равно как роутить + props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, + RoundRobinPartitioner.class); + // Отправляем сообщения сразу + props.put(ProducerConfig.LINGER_MS_CONFIG, 0); + // До-обогащаем конфигурацию + enchanter.accept(props); + return new DefaultKafkaProducerFactory<>(props); + } + /** + * Creates a Kafka listener container factory. + * + * @return a new Kafka listener container factory + */ + @Bean + public ConcurrentKafkaListenerContainerFactory + kafkaListenerContainerFactory() { + var props = properties.buildConsumerProperties(null); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); + ConcurrentKafkaListenerContainerFactory + factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory( + new DefaultKafkaConsumerFactory<>( + props, + new StringDeserializer(), + new JsonDeserializer<>(Task.class) + ) + ); + factory.getContainerProperties().setAckMode( + ContainerProperties.AckMode.MANUAL + ); + return factory; + } + + +} + diff --git a/worker/src/main/java/com/github/asavershin/images/config/MinIOConf.java b/worker/src/main/java/com/github/asavershin/images/config/MinIOConf.java new file mode 100644 index 0000000..6411c9a --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/config/MinIOConf.java @@ -0,0 +1,39 @@ +package com.github.asavershin.images.config; + +import io.minio.MinioClient; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * Configuration class for MinIO client. + * + * @author Asavershin + */ +@Configuration +@RequiredArgsConstructor +@Slf4j +public class MinIOConf { + + /** + * The MinIO properties. + */ + private final MinIOProperties minioProperties; + + /** + * Creates a MinioClient instance. + * + * @return A MinioClient instance configured with the provided + * MinIO properties. + */ + @Bean + public MinioClient minioClient() { + log.info(minioProperties.toString()); + return MinioClient.builder() + .endpoint(minioProperties.getUrl()) + .credentials(minioProperties.getUser(), + minioProperties.getPassword()) + .build(); + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/config/MinIOProperties.java b/worker/src/main/java/com/github/asavershin/images/config/MinIOProperties.java new file mode 100644 index 0000000..be79b2e --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/config/MinIOProperties.java @@ -0,0 +1,40 @@ +package com.github.asavershin.images.config; + +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.stereotype.Component; + +@Component +@Data +@ConfigurationProperties(prefix = "minio") +@NoArgsConstructor +public class MinIOProperties { + /** + * The name of the bucket in the MinIO service. + */ + private String bucket; + /** + * The ttl of the MinIO objects. + */ + private String expiration; + /** + * The name of the bucket with ttl. + */ + private String ttlprefix; + + /** + * The URL of the MinIO service. + */ + private String url; + + /** + * The username for the MinIO service. + */ + private String user; + + /** + * The password for the MinIO service. + */ + private String password; +} diff --git a/worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java b/worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java new file mode 100644 index 0000000..65351f9 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java @@ -0,0 +1,19 @@ +package com.github.asavershin.images.config; + +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +@NoArgsConstructor +public class ProducerProp { + /** + * Count of publisher tries. + */ + @Value("${app.retries}") + @NotNull(message = "Producer properties are invalid") + private String retries; +} diff --git a/worker/src/main/java/com/github/asavershin/images/config/RedisConf.java b/worker/src/main/java/com/github/asavershin/images/config/RedisConf.java new file mode 100644 index 0000000..681ba9f --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/config/RedisConf.java @@ -0,0 +1,36 @@ +package com.github.asavershin.images.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.GenericToStringSerializer; +import org.springframework.data.redis.serializer.StringRedisSerializer; + +/** + * Configuration class for setting up the RedisTemplate. + * + * @author Asavershin + */ +@Configuration +public class RedisConf { + + /** + * Creates a new instance of {@link RedisTemplate} with the provided + * {@link RedisConnectionFactory}. + * + * @param connectionFactory the RedisConnectionFactory to use + * @return a new instance of {@link RedisTemplate} + */ + @Bean + public RedisTemplate redisTemplate( + final RedisConnectionFactory connectionFactory) { + final RedisTemplate template = new RedisTemplate<>(); + template.setConnectionFactory(connectionFactory); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer( + new GenericToStringSerializer<>(Object.class) + ); + return template; + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/domain/ImageId.java b/worker/src/main/java/com/github/asavershin/images/domain/ImageId.java new file mode 100644 index 0000000..51baaaf --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/domain/ImageId.java @@ -0,0 +1,50 @@ +package com.github.asavershin.images.domain; + +import java.util.Objects; +import java.util.UUID; + +/** + * A value object representing an image ID. + * + * @param value The unique identifier for the image. + */ +public record ImageId(UUID value) { + /** + * Constructs an ImageId instance with the provided UUID. + * + * @param value The unique identifier for the image. + * @throws NullPointerException if the provided value is null. + */ + public ImageId { + Objects.requireNonNull(value, "Image ID must not be null"); + } + + /** + * Generates a new, unique ImageId. + * + * @return A new ImageId instance with a randomly generated UUID. + */ + public static ImageId nextIdentity() { + return new ImageId(UUID.randomUUID()); + } + + /** + * Parses a string into an ImageId. + * + * @param id The string representation of the ImageId. + * @return A new ImageId instance with the parsed UUID. + * @throws IllegalArgumentException if the provided string + * is not a valid UUID. + */ + public static ImageId fromString(final String id) { + Objects.requireNonNull(id, "ImageId must not be null"); + UUID uuid; + try { + uuid = UUID.fromString(id); + } catch (final IllegalArgumentException ex) { + throw new IllegalArgumentException("Invalid ImageId: " + + ex.getMessage()); + } + return new ImageId(uuid); + } +} \ No newline at end of file diff --git a/worker/src/main/java/com/github/asavershin/images/domain/RequestId.java b/worker/src/main/java/com/github/asavershin/images/domain/RequestId.java new file mode 100644 index 0000000..912c737 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/domain/RequestId.java @@ -0,0 +1,53 @@ +package com.github.asavershin.images.domain; + +import java.util.Objects; +import java.util.UUID; + +/** + * A value object representing an Image Processing Event ID. + * + * @param value The unique identifier for the Image Processing Event. + */ +public record RequestId(UUID value) { + /** + * Constructs an requestId instance with the provided UUID. + * + * @param value The unique identifier for the Image Processing Event. + * @throws NullPointerException if the provided value is null. + */ + public RequestId { + Objects.requireNonNull( + value, + "Image Processing Event ID must not be null" + ); + } + + /** + * Generates a new, unique requestId. + * + * @return A new requestId instance with a randomly generated UUID. + */ + public static RequestId nextIdentity() { + return new RequestId(UUID.randomUUID()); + } + + /** + * Parses a string into a {@link RequestId} instance. + * + * @param id The string representation of the {@link RequestId}. + * @return A new {@link RequestId} instance with the parsed UUID. + * @throws IllegalArgumentException if the provided string is not + * a valid UUID. + */ + public static RequestId fromString(final String id) { + Objects.requireNonNull(id, "requestId must not be null"); + UUID uuid; + try { + uuid = UUID.fromString(id); + } catch (final IllegalArgumentException ex) { + throw new IllegalArgumentException("Invalid requestId: " + + ex.getMessage()); + } + return new RequestId(uuid); + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/in/Consumer.java b/worker/src/main/java/com/github/asavershin/images/in/Consumer.java new file mode 100644 index 0000000..3c70d5a --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/in/Consumer.java @@ -0,0 +1,53 @@ +package com.github.asavershin.images.in; + +import com.github.asavershin.images.Task; +import com.github.asavershin.images.WorkerManager; +import com.github.asavershin.images.out.Producer; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.apache.kafka.clients.consumer.ConsumerRecord; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +@Component +@RequiredArgsConstructor +@Slf4j +public class Consumer { + private final WorkerManager manager; + private final Producer producer; + + @Value("${app.wiptopic}") + @NotNull(message = "Topic for KafkaProducer is null") + private String wip; + @Value("${app.donetopic}") + @NotNull(message = "Topic for KafkaProducer is null") + private String done; + + @KafkaListener( + topics = "${app.wiptopic}", + groupId = "${app.group-id}", + concurrency = "${app.replicas}", + containerFactory = "kafkaListenerContainerFactory" + ) + @Transactional + public void consume( + final ConsumerRecord record, + final Acknowledgment acknowledgment + ) { + log.info( + "Received message: {}", + record.value().toString() + ); + var task = manager.start(record.value()); + if (task.getFilters().isEmpty()) { + producer.produce(task, done); + } else { + producer.produce(task, wip); + } + acknowledgment.acknowledge(); + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java b/worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java new file mode 100644 index 0000000..126efd0 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java @@ -0,0 +1,22 @@ +package com.github.asavershin.images.out; + +public interface CacheRepository { + /** + * Adds a cache entry with the specified key, + * value, and expiration time. + * + * @param key the unique identifier for the cache entry + * @param value the value associated with the cache entry + * @param expiration the time in milliseconds after + * which the cache entry will expire + */ + void addCache(String key, String value, long expiration); + /** + * Retrieves the cache entry associated with the specified key. + * + * @param key the unique identifier for the cache entry + * @return the token associated with the cache entry, + * or null if the entry does not exist or has expired + */ + String getCache(String key); +} diff --git a/worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java b/worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java new file mode 100644 index 0000000..c604007 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java @@ -0,0 +1,36 @@ +package com.github.asavershin.images.out; + +import lombok.RequiredArgsConstructor; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Repository; + +import java.util.concurrent.TimeUnit; + +@Repository +@RequiredArgsConstructor +public class CacheRepositoryIml implements CacheRepository { + /** + * Redis template is used to interact with redis server. + */ + private final RedisTemplate redisTemplate; + + /** + * Not final to allow spring use proxy. + */ + @Override + public void addCache(final String key, + final String value, + final long expiration) { + redisTemplate.opsForValue().set( + key, value, expiration, TimeUnit.MILLISECONDS + ); + } + + /** + * Not final to allow spring use proxy. + */ + @Override + public String getCache(final String key) { + return (String) redisTemplate.opsForValue().get(key); + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/out/FileException.java b/worker/src/main/java/com/github/asavershin/images/out/FileException.java new file mode 100644 index 0000000..587ad7d --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/out/FileException.java @@ -0,0 +1,12 @@ +package com.github.asavershin.images.out; + +public class FileException extends RuntimeException { + /** + * Constructs a new FileException with the specified detail message. + * + * @param s the detail message + */ + public FileException(final String s) { + super(s); + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/out/MinioService.java b/worker/src/main/java/com/github/asavershin/images/out/MinioService.java new file mode 100644 index 0000000..f649fa8 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/out/MinioService.java @@ -0,0 +1,11 @@ +package com.github.asavershin.images.out; + +import org.springframework.web.multipart.MultipartFile; + +import java.io.InputStream; + +public interface MinioService { + void saveFile(byte[] image, String filename); + + InputStream getFile(String link); +} diff --git a/worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java b/worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java new file mode 100644 index 0000000..af1b8d3 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java @@ -0,0 +1,146 @@ +package com.github.asavershin.images.out; + + +import com.github.asavershin.images.config.MinIOProperties; +import io.minio.BucketExistsArgs; +import io.minio.GetObjectArgs; +import io.minio.MakeBucketArgs; +import io.minio.MinioClient; +import io.minio.PutObjectArgs; +import io.minio.SetBucketLifecycleArgs; +import io.minio.messages.Expiration; +import io.minio.messages.LifecycleConfiguration; +import io.minio.messages.LifecycleRule; +import io.minio.messages.RuleFilter; +import io.minio.messages.Status; +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.springframework.stereotype.Service; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.time.ZonedDateTime; +import java.util.LinkedList; +import java.util.List; + +@Service +@Slf4j +public class MinioServiceIml implements MinioService { + /** + * The MinioClient is used to interact with the MinIO server. + */ + private final MinioClient minioClient; + /** + * The MinIO properties from .yml. + */ + private final MinIOProperties minioProperties; + + /** + * Constructor for {@link MinioServiceIml}. + * + * @param aMinioClient The {@link MinioClient} + * instance to interact with the MinIO server. + * @param aMinioProperties The {@link MinIOProperties} + * instance containing the configuration + * for the MinIO server. + */ + public MinioServiceIml(final MinioClient aMinioClient, + final MinIOProperties aMinioProperties) { + this.minioClient = aMinioClient; + this.minioProperties = aMinioProperties; + createBuckets(); + } + + /** + * Not final to allow spring use proxy. + */ + @Override + public void saveFile(byte[] image, final String filename) { + + if (!bucketExists(minioProperties.getBucket())) { + throw new FileException( + "File upload failed: bucket does not exist" + ); + } + + if (image.length == 0) { + throw new FileException("File array could not be empty"); + } + try (var bais = new ByteArrayInputStream(image)) { + saveFile(bais, filename); + } catch (Exception e) { + throw new FileException("File upload failed: " + + e.getMessage()); + } + } + + /** + * Not final to allow spring use proxy. + */ + @Override + public InputStream getFile(final String link) { + if (link == null) { + throw new FileException("File download failed: link is nullable"); + } + try { + InputStream obj = minioClient.getObject(GetObjectArgs.builder() + .bucket(minioProperties.getBucket()) + .object(link) + .build()); + return obj; + } catch (Exception e) { + throw new FileException("File download failed: " + e.getMessage()); + } + } + + @SneakyThrows + private void saveFile( + final InputStream inputStream, + final String fileName + ) { + minioClient.putObject(PutObjectArgs.builder() + .stream(inputStream, inputStream.available(), -1) + .bucket(minioProperties.getBucket()) + .object(fileName) + .build()); + } + + @SneakyThrows + private void createBuckets() { + boolean found = bucketExists(minioProperties.getBucket()); + if (!found) { + List rules = new LinkedList<>(); + rules.add( + new LifecycleRule( + Status.ENABLED, + null, + new Expiration((ZonedDateTime) null, + Integer.valueOf( + minioProperties.getExpiration() + ), + null), + new RuleFilter(minioProperties.getTtlprefix()), + "rule1", + null, + null, + null)); + minioClient.makeBucket(MakeBucketArgs.builder() + .bucket(minioProperties.getBucket()) + .build()); + minioClient.setBucketLifecycle( + SetBucketLifecycleArgs.builder().bucket( + minioProperties.getBucket() + ) + .config( + new LifecycleConfiguration(rules) + ).build()); + } + } + + @SneakyThrows(Exception.class) + private boolean bucketExists(final String bucketName) { + return minioClient.bucketExists( + BucketExistsArgs.builder().bucket(bucketName).build() + ); + } +} diff --git a/worker/src/main/java/com/github/asavershin/images/out/Producer.java b/worker/src/main/java/com/github/asavershin/images/out/Producer.java new file mode 100644 index 0000000..3c6923d --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/images/out/Producer.java @@ -0,0 +1,29 @@ +package com.github.asavershin.images.out; + +import com.github.asavershin.images.Task; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import static com.github.asavershin.images.config.KafkaConf.ALL_ACKS_KAFKA_TEMPLATE; + +@Component +public class Producer { + private final KafkaTemplate producer; + + public Producer( + final @Qualifier(ALL_ACKS_KAFKA_TEMPLATE) + KafkaTemplate allAcksKafkaTemplate + ) { + producer = allAcksKafkaTemplate; + } + + @Transactional + public void produce(final Task event, final String topic) { + producer.send( + topic, + event + ); + } +} diff --git a/worker/src/main/resources/application-blackwhite.yml b/worker/src/main/resources/application-blackwhite.yml new file mode 100644 index 0000000..0032eba --- /dev/null +++ b/worker/src/main/resources/application-blackwhite.yml @@ -0,0 +1,10 @@ +server: + port: 8082 + +app: + wiptopic: ${TOPIC:images.wip} + retries: ${RETRIES:3} + donetopic: ${TOPIC:images.done} + partitions: ${PARTITIONS:1} + replicas: ${REPLICAS:3} + group-id: ${GROUP_ID:blackwhite} \ No newline at end of file diff --git a/worker/src/main/resources/application-rotate.yml b/worker/src/main/resources/application-rotate.yml new file mode 100644 index 0000000..68ced85 --- /dev/null +++ b/worker/src/main/resources/application-rotate.yml @@ -0,0 +1,10 @@ +server: + port: 8083 + +app: + wiptopic: ${TOPIC:images.wip} + retries: ${RETRIES:3} + donetopic: ${TOPIC:images.done} + partitions: ${PARTITIONS:1} + replicas: ${REPLICAS:3} + group-id: ${GROUP_ID:rotate} \ No newline at end of file diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml new file mode 100644 index 0000000..9a39d88 --- /dev/null +++ b/worker/src/main/resources/application.yml @@ -0,0 +1,36 @@ +spring: + servlet: + multipart: + max-file-size: 10MB + config: + import: optional:file:.env[.properties] + data: + redis: + host: ${REDIS_HOST} + port: ${REDIS_PORT} + password: ${REDIS_PASSWORD} + kafka: + listener.ack-mode: manual + bootstrap-servers: + - kafka-1:9092 + - kafka-2:9093 + - kafka-3:9094 + properties: + sasl: + jaas: + config: org.apache.kafka.common.security.plain.PlainLoginModule required username=${kafka_username:'kafka'} password=${kafka_password:'password123'}; + mechanism: PLAIN + security: + protocol: SASL_PLAINTEXT + +minio: + url: ${MINIO_URL} + user: ${MINIO_ROOT_USER} + password: ${MINIO_ROOT_PASSWORD} + bucket: ${MINIO_BUCKET} + ttlprefix: ${MINIO_TTL_PREFIX} + expiration: ${MINIO_EXPIRATION} + console-port: ${MINIO_CONSOLE_PORT} + port: ${MINIO_PORT} + +cache-expiration: ${REDIS_CACHE_TIME} \ No newline at end of file From bf31d666931a5744061897a6fffa3d411a278803 Mon Sep 17 00:00:00 2001 From: asavershin Date: Mon, 6 May 2024 22:19:51 +0300 Subject: [PATCH 2/7] Refactoring --- api/.env | 2 + api/pom.xml | 1 - .../asavershin/api/config/KafkaConf.java | 1 + .../config/properties/MinIOProperties.java | 11 +- .../asavershin/api/domain/filter/Filter.java | 9 +- .../out/storage/MinioServiceIml.java | 53 ++++-- api/src/main/resources/application.yml | 2 + .../asavershin/api/common/EventHelper.java | 7 +- .../api/domaintest/FilterEventTest.java | 8 +- .../api/integrations/EventLogicTest.java | 2 +- api/src/test/resources/application.properties | 1 + .../asavershin/images/config/KafkaConf.java | 140 ---------------- .../images/config/ProducerProp.java | 19 --- .../asavershin/images/domain/ImageId.java | 50 ------ .../asavershin/images/domain/RequestId.java | 53 ------ .../github/asavershin/images/in/Consumer.java | 53 ------ .../asavershin/{images => worker}/Main.java | 0 .../asavershin/worker/config/KafkaConf.java | 157 ++++++++++++++++++ .../{images => worker}/config/MinIOConf.java | 0 .../config/MinIOProperties.java | 0 .../worker/config/ProducerProp.java | 19 +++ .../{images => worker}/config/RedisConf.java | 0 .../{images => worker/dto}/Task.java | 2 +- .../github/asavershin/worker/in/Consumer.java | 77 +++++++++ .../out/CacheRepository.java | 2 + .../{images => worker}/out/FileException.java | 0 .../{images => worker}/out/MinioService.java | 0 .../out/impl}/CacheRepositoryIml.java | 10 +- .../out/impl}/MinioServiceIml.java | 4 +- .../out/producers/WipProducer.java} | 20 ++- .../services}/BlackWhiteWorker.java | 6 +- .../services}/RotateImage.java | 13 +- .../{images => worker/services}/Worker.java | 4 +- .../services}/WorkerManager.java | 2 +- .../services}/WorkerManagerImpl.java | 21 +-- 35 files changed, 365 insertions(+), 384 deletions(-) delete mode 100644 worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java delete mode 100644 worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java delete mode 100644 worker/src/main/java/com/github/asavershin/images/domain/ImageId.java delete mode 100644 worker/src/main/java/com/github/asavershin/images/domain/RequestId.java delete mode 100644 worker/src/main/java/com/github/asavershin/images/in/Consumer.java rename worker/src/main/java/com/github/asavershin/{images => worker}/Main.java (100%) create mode 100644 worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java rename worker/src/main/java/com/github/asavershin/{images => worker}/config/MinIOConf.java (100%) rename worker/src/main/java/com/github/asavershin/{images => worker}/config/MinIOProperties.java (100%) create mode 100644 worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java rename worker/src/main/java/com/github/asavershin/{images => worker}/config/RedisConf.java (100%) rename worker/src/main/java/com/github/asavershin/{images => worker/dto}/Task.java (87%) create mode 100644 worker/src/main/java/com/github/asavershin/worker/in/Consumer.java rename worker/src/main/java/com/github/asavershin/{images => worker}/out/CacheRepository.java (95%) rename worker/src/main/java/com/github/asavershin/{images => worker}/out/FileException.java (100%) rename worker/src/main/java/com/github/asavershin/{images => worker}/out/MinioService.java (100%) rename worker/src/main/java/com/github/asavershin/{images/out => worker/out/impl}/CacheRepositoryIml.java (82%) rename worker/src/main/java/com/github/asavershin/{images/out => worker/out/impl}/MinioServiceIml.java (97%) rename worker/src/main/java/com/github/asavershin/{images/out/Producer.java => worker/out/producers/WipProducer.java} (51%) rename worker/src/main/java/com/github/asavershin/{images => worker/services}/BlackWhiteWorker.java (94%) rename worker/src/main/java/com/github/asavershin/{images => worker/services}/RotateImage.java (84%) rename worker/src/main/java/com/github/asavershin/{images => worker/services}/Worker.java (61%) rename worker/src/main/java/com/github/asavershin/{images => worker/services}/WorkerManager.java (63%) rename worker/src/main/java/com/github/asavershin/{images => worker/services}/WorkerManagerImpl.java (78%) diff --git a/api/.env b/api/.env index c2d0dce..03d5aa4 100644 --- a/api/.env +++ b/api/.env @@ -17,11 +17,13 @@ REDIS_CACHE_TIME=86400000 # MINIO MINIO_BUCKET=files +MINIO_EXPIRATION=1 MINIO_URL=http://minio-api:9000 MINIO_ROOT_USER=minioadmin MINIO_ROOT_PASSWORD=minioadmin MINIO_CONSOLE_PORT=9090 MINIO_PORT=9000 +MINIO_TTL_PREFIX=temporary/ # MONGO MONGO_INITDB_ROOT_USERNAME=admin diff --git a/api/pom.xml b/api/pom.xml index 3802b62..2d73337 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -19,7 +19,6 @@ 2.1.0 0.11.5 8.5.7 - 3.7.0 1.19.1 diff --git a/api/src/main/java/com/github/asavershin/api/config/KafkaConf.java b/api/src/main/java/com/github/asavershin/api/config/KafkaConf.java index ff150ec..1b9953d 100644 --- a/api/src/main/java/com/github/asavershin/api/config/KafkaConf.java +++ b/api/src/main/java/com/github/asavershin/api/config/KafkaConf.java @@ -102,6 +102,7 @@ private ProducerFactory producerFactory( StringSerializer.class); props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, JsonSerializer.class); + props.put(JsonSerializer.ADD_TYPE_INFO_HEADERS, false); // Партиция одна, так что все равно как роутить props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, RoundRobinPartitioner.class); diff --git a/api/src/main/java/com/github/asavershin/api/config/properties/MinIOProperties.java b/api/src/main/java/com/github/asavershin/api/config/properties/MinIOProperties.java index 805e730..2d10506 100644 --- a/api/src/main/java/com/github/asavershin/api/config/properties/MinIOProperties.java +++ b/api/src/main/java/com/github/asavershin/api/config/properties/MinIOProperties.java @@ -14,19 +14,24 @@ public class MinIOProperties { * The name of the bucket in the MinIO service. */ private String bucket; - /** * The URL of the MinIO service. */ private String url; - /** * The username for the MinIO service. */ private String user; - /** * The password for the MinIO service. */ private String password; + /** + * The ttl of the MinIO objects. + */ + private String expiration; + /** + * The name of the bucket with ttl. + */ + private String ttlprefix; } diff --git a/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java b/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java index e4d32dd..b9c760f 100644 --- a/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java +++ b/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java @@ -10,17 +10,12 @@ public enum Filter { /** * Represents the REVERS_COLORS filter. */ - REVERS_COLORS, + ROTATE, /** * Represents the CROP filter. */ - CROP, - - /** - * Represents the REMOVE_BACKGROUND filter. - */ - REMOVE_BACKGROUND; + BLACKWHITE; /** * Converts a string representation of a filter name string diff --git a/api/src/main/java/com/github/asavershin/api/infrastructure/out/storage/MinioServiceIml.java b/api/src/main/java/com/github/asavershin/api/infrastructure/out/storage/MinioServiceIml.java index 055f46b..eebe7d3 100644 --- a/api/src/main/java/com/github/asavershin/api/infrastructure/out/storage/MinioServiceIml.java +++ b/api/src/main/java/com/github/asavershin/api/infrastructure/out/storage/MinioServiceIml.java @@ -9,12 +9,20 @@ import io.minio.MinioClient; import io.minio.PutObjectArgs; import io.minio.RemoveObjectArgs; +import io.minio.SetBucketLifecycleArgs; +import io.minio.messages.Expiration; +import io.minio.messages.LifecycleConfiguration; +import io.minio.messages.LifecycleRule; +import io.minio.messages.RuleFilter; +import io.minio.messages.Status; import lombok.SneakyThrows; import org.apache.commons.compress.utils.IOUtils; import org.springframework.stereotype.Service; import org.springframework.web.multipart.MultipartFile; import java.io.InputStream; +import java.time.ZonedDateTime; +import java.util.LinkedList; import java.util.List; @Service @@ -27,14 +35,15 @@ public class MinioServiceIml implements MinioService { * The MinIO properties from .yml. */ private final MinIOProperties minioProperties; + /** * Constructor for {@link MinioServiceIml}. * - * @param aMinioClient The {@link MinioClient} - * instance to interact with the MinIO server. + * @param aMinioClient The {@link MinioClient} + * instance to interact with the MinIO server. * @param aMinioProperties The {@link MinIOProperties} - * instance containing the configuration - * for the MinIO server. + * instance containing the configuration + * for the MinIO server. */ public MinioServiceIml(final MinioClient aMinioClient, final MinIOProperties aMinioProperties) { @@ -42,6 +51,7 @@ public MinioServiceIml(final MinioClient aMinioClient, this.minioProperties = aMinioProperties; createBucket(); } + /** * Not final to allow spring use proxy. */ @@ -66,6 +76,7 @@ public void saveFile(final MultipartFile image, final String filename) { } saveFile(inputStream, filename); } + /** * Not final to allow spring use proxy. */ @@ -77,13 +88,14 @@ public byte[] getFile(final String link) { try { return IOUtils.toByteArray( minioClient.getObject(GetObjectArgs.builder() - .bucket(minioProperties.getBucket()) - .object(link) - .build())); + .bucket(minioProperties.getBucket()) + .object(link) + .build())); } catch (Exception e) { throw new FileException("File download failed: " + e.getMessage()); } } + /** * Not final to allow spring use proxy. */ @@ -110,14 +122,35 @@ public void deleteFiles(final List links) { @SneakyThrows private void createBucket() { - boolean found = minioClient.bucketExists(BucketExistsArgs.builder() - .bucket(minioProperties.getBucket()) - .build()); + boolean found = bucketExists(minioProperties.getBucket()); if (!found) { + List rules = new LinkedList<>(); + rules.add( + new LifecycleRule( + Status.ENABLED, + null, + new Expiration((ZonedDateTime) null, + Integer.valueOf( + minioProperties.getExpiration() + ), + null), + new RuleFilter(minioProperties.getTtlprefix()), + "rule1", + null, + null, + null)); minioClient.makeBucket(MakeBucketArgs.builder() .bucket(minioProperties.getBucket()) .build()); + minioClient.setBucketLifecycle( + SetBucketLifecycleArgs.builder().bucket( + minioProperties.getBucket() + ) + .config( + new LifecycleConfiguration(rules) + ).build()); } + } @SneakyThrows diff --git a/api/src/main/resources/application.yml b/api/src/main/resources/application.yml index d737f2b..625f99f 100644 --- a/api/src/main/resources/application.yml +++ b/api/src/main/resources/application.yml @@ -54,6 +54,8 @@ minio: url: ${MINIO_URL} user: ${MINIO_ROOT_USER} password: ${MINIO_ROOT_PASSWORD} + expiration: ${MINIO_EXPIRATION} + ttlprefix: ${MINIO_TTL_PREFIX} bucket: ${MINIO_BUCKET} console-port: ${MINIO_CONSOLE_PORT} port: ${MINIO_PORT} diff --git a/api/src/test/java/com/github/asavershin/api/common/EventHelper.java b/api/src/test/java/com/github/asavershin/api/common/EventHelper.java index fc2c97a..d73c4b0 100644 --- a/api/src/test/java/com/github/asavershin/api/common/EventHelper.java +++ b/api/src/test/java/com/github/asavershin/api/common/EventHelper.java @@ -21,15 +21,14 @@ public class EventHelper { public static List hugeList() { var result = new ArrayList(); for (int i = 0; i < 32780; i++) { - result.add(Filter.REVERS_COLORS); + result.add(Filter.ROTATE); } return result; } public static List filters() { - return List.of(Filter.REVERS_COLORS, - Filter.CROP, - Filter.REMOVE_BACKGROUND); + return List.of(Filter.ROTATE, + Filter.BLACKWHITE); } public static ImageProcessingStarted started(final ImageId id) { diff --git a/api/src/test/java/com/github/asavershin/api/domaintest/FilterEventTest.java b/api/src/test/java/com/github/asavershin/api/domaintest/FilterEventTest.java index d6f472b..854067e 100644 --- a/api/src/test/java/com/github/asavershin/api/domaintest/FilterEventTest.java +++ b/api/src/test/java/com/github/asavershin/api/domaintest/FilterEventTest.java @@ -23,7 +23,7 @@ public void testImageProcessingStartedConstructor() { // Given ImageId imageId = ImageId.nextIdentity(); List filters = new ArrayList<>(); - filters.add(Filter.CROP); + filters.add(Filter.BLACKWHITE); // When var event = new ImageProcessingStarted(filters, imageId); @@ -40,7 +40,7 @@ public void testNullOriginalImageIdToNewEvent() { // Given var lotsOfFilters = hugeList(); var zeroFilters = new ArrayList(); - var normalFilters = List.of(Filter.CROP); + var normalFilters = List.of(Filter.BLACKWHITE); // When var nullIdEx = assertThrows(NullPointerException.class, @@ -68,7 +68,7 @@ public void testZeroFiltersToNewEvent() { void testPublish() { // Arrange var originalImageId = ImageId.nextIdentity(); - List filters = List.of(Filter.CROP, Filter.REMOVE_BACKGROUND); + List filters = List.of(Filter.BLACKWHITE, Filter.ROTATE); ImageProcessingStarted imageProcessingStarted = new ImageProcessingStarted(filters, originalImageId); // Act @@ -109,7 +109,7 @@ public void testImageProcessingResultConstructor() { private List createFiltersWithMaxCount() { List filters = new ArrayList<>(); for (int i = 0; i < 32780; i++) { - filters.add(Filter.CROP); + filters.add(Filter.BLACKWHITE); } return filters; } diff --git a/api/src/test/java/com/github/asavershin/api/integrations/EventLogicTest.java b/api/src/test/java/com/github/asavershin/api/integrations/EventLogicTest.java index b13d603..6c4ca9f 100644 --- a/api/src/test/java/com/github/asavershin/api/integrations/EventLogicTest.java +++ b/api/src/test/java/com/github/asavershin/api/integrations/EventLogicTest.java @@ -92,7 +92,7 @@ public void testStartedEvent() { startedEvent.store(newEvent, testUser.userId()); // Then var storedEvent = getEventFromDB(dsl, newEvent.requestId()); - assertEquals(storedEvent.size(), 3); + assertEquals(storedEvent.size(), 2); assertNotNull(storedEvent); assertNotNull(storedEvent.get(0).getValue(IMAGE_PROCESSING_EVENT.IMAGE_PROCESSING_EVENT_ID)); assertEquals(storedEvent.get(0).getValue(IMAGE_PROCESSING_EVENT.IMAGE_PROCESSING_EVENT_ID), diff --git a/api/src/test/resources/application.properties b/api/src/test/resources/application.properties index 9e264eb..6f267fc 100644 --- a/api/src/test/resources/application.properties +++ b/api/src/test/resources/application.properties @@ -23,6 +23,7 @@ minio.password=minioadmin minio.bucket=files minio.console-port=9090 minio.port=9000 +minio.expiration=1 testconf=test diff --git a/worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java b/worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java deleted file mode 100644 index bb9f442..0000000 --- a/worker/src/main/java/com/github/asavershin/images/config/KafkaConf.java +++ /dev/null @@ -1,140 +0,0 @@ -package com.github.asavershin.images.config; - -import com.github.asavershin.images.Task; -import jakarta.validation.constraints.NotNull; -import lombok.RequiredArgsConstructor; -import org.apache.kafka.clients.admin.NewTopic; -import org.apache.kafka.clients.consumer.ConsumerConfig; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.RoundRobinPartitioner; -import org.apache.kafka.common.serialization.StringDeserializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -import org.springframework.context.annotation.Bean; -import org.springframework.context.annotation.Configuration; -import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -import org.springframework.kafka.core.DefaultKafkaProducerFactory; -import org.springframework.kafka.core.KafkaTemplate; -import org.springframework.kafka.core.ProducerFactory; -import org.springframework.kafka.listener.ContainerProperties; -import org.springframework.kafka.support.serializer.JsonDeserializer; -import org.springframework.kafka.support.serializer.JsonSerializer; - -import java.util.Map; -import java.util.function.Consumer; - -@Configuration -@RequiredArgsConstructor -public class KafkaConf { - /** - * Static final string constant for the Kafka template bean name. - */ - public static final String ALL_ACKS_KAFKA_TEMPLATE = "allAcksKafkaTemplate"; - /** - * Injected producer properties. - */ - @NotNull(message = "KafkaConf is invalid: prodProp is null") - private final ProducerProp prodProp; - /** - * Injected Kafka properties. - */ - @NotNull(message = "KafkaConf is invalid: KafkaProperties is null") - private final KafkaProperties properties; - /** - * Creates a new Kafka topic. - * - * @param topic the name of the topic - * @param partitions the number of partitions - * @param replicas the number of replicas - * @return a new Kafka topic - */ - @Bean - public NewTopic wipTopic( - final @Value("${app.wiptopic}") String topic, - final @Value("${app.partitions}") int partitions, - final @Value("${app.replicas}") short replicas) { - - return new NewTopic(topic, partitions, replicas); - } - /** - * Creates a new Kafka topic with the specified name, - * number of partitions, and replicas. - * - * @param topic the name of the topic - * @param partitions the number of partitions for the topic - * @param replicas the number of replicas for the topic - * @return a new Kafka topic with the specified configuration - */ - @Bean - public NewTopic doneTopic( - final @Value("${app.donetopic}") String topic, - final @Value("${app.partitions}") int partitions, - final @Value("${app.replicas}") short replicas) { - - return new NewTopic(topic, partitions, replicas); - } - /** - * Creates a new Kafka template with all acknowledgments. - * - * @return a new Kafka template with all acknowledgments - */ - @Bean(ALL_ACKS_KAFKA_TEMPLATE) - public KafkaTemplate allAcksKafkaTemplate() { - return new KafkaTemplate<>(producerFactory( - props -> { - props.put(ProducerConfig.ACKS_CONFIG, "all"); - props.put(ProducerConfig.RETRIES_CONFIG, - prodProp.getRetries()); - }) - ); - } - - private ProducerFactory producerFactory( - final Consumer> enchanter - ) { - var props = properties.buildProducerProperties(null); - // Работаем со строками - props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, - StringSerializer.class); - props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, - JsonSerializer.class); - // Партиция одна, так что все равно как роутить - props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, - RoundRobinPartitioner.class); - // Отправляем сообщения сразу - props.put(ProducerConfig.LINGER_MS_CONFIG, 0); - // До-обогащаем конфигурацию - enchanter.accept(props); - return new DefaultKafkaProducerFactory<>(props); - } - /** - * Creates a Kafka listener container factory. - * - * @return a new Kafka listener container factory - */ - @Bean - public ConcurrentKafkaListenerContainerFactory - kafkaListenerContainerFactory() { - var props = properties.buildConsumerProperties(null); - props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); - props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); - ConcurrentKafkaListenerContainerFactory - factory = new ConcurrentKafkaListenerContainerFactory<>(); - factory.setConsumerFactory( - new DefaultKafkaConsumerFactory<>( - props, - new StringDeserializer(), - new JsonDeserializer<>(Task.class) - ) - ); - factory.getContainerProperties().setAckMode( - ContainerProperties.AckMode.MANUAL - ); - return factory; - } - - -} - diff --git a/worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java b/worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java deleted file mode 100644 index 65351f9..0000000 --- a/worker/src/main/java/com/github/asavershin/images/config/ProducerProp.java +++ /dev/null @@ -1,19 +0,0 @@ -package com.github.asavershin.images.config; - -import jakarta.validation.constraints.NotNull; -import lombok.Data; -import lombok.NoArgsConstructor; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.stereotype.Component; - -@Data -@Component -@NoArgsConstructor -public class ProducerProp { - /** - * Count of publisher tries. - */ - @Value("${app.retries}") - @NotNull(message = "Producer properties are invalid") - private String retries; -} diff --git a/worker/src/main/java/com/github/asavershin/images/domain/ImageId.java b/worker/src/main/java/com/github/asavershin/images/domain/ImageId.java deleted file mode 100644 index 51baaaf..0000000 --- a/worker/src/main/java/com/github/asavershin/images/domain/ImageId.java +++ /dev/null @@ -1,50 +0,0 @@ -package com.github.asavershin.images.domain; - -import java.util.Objects; -import java.util.UUID; - -/** - * A value object representing an image ID. - * - * @param value The unique identifier for the image. - */ -public record ImageId(UUID value) { - /** - * Constructs an ImageId instance with the provided UUID. - * - * @param value The unique identifier for the image. - * @throws NullPointerException if the provided value is null. - */ - public ImageId { - Objects.requireNonNull(value, "Image ID must not be null"); - } - - /** - * Generates a new, unique ImageId. - * - * @return A new ImageId instance with a randomly generated UUID. - */ - public static ImageId nextIdentity() { - return new ImageId(UUID.randomUUID()); - } - - /** - * Parses a string into an ImageId. - * - * @param id The string representation of the ImageId. - * @return A new ImageId instance with the parsed UUID. - * @throws IllegalArgumentException if the provided string - * is not a valid UUID. - */ - public static ImageId fromString(final String id) { - Objects.requireNonNull(id, "ImageId must not be null"); - UUID uuid; - try { - uuid = UUID.fromString(id); - } catch (final IllegalArgumentException ex) { - throw new IllegalArgumentException("Invalid ImageId: " - + ex.getMessage()); - } - return new ImageId(uuid); - } -} \ No newline at end of file diff --git a/worker/src/main/java/com/github/asavershin/images/domain/RequestId.java b/worker/src/main/java/com/github/asavershin/images/domain/RequestId.java deleted file mode 100644 index 912c737..0000000 --- a/worker/src/main/java/com/github/asavershin/images/domain/RequestId.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.github.asavershin.images.domain; - -import java.util.Objects; -import java.util.UUID; - -/** - * A value object representing an Image Processing Event ID. - * - * @param value The unique identifier for the Image Processing Event. - */ -public record RequestId(UUID value) { - /** - * Constructs an requestId instance with the provided UUID. - * - * @param value The unique identifier for the Image Processing Event. - * @throws NullPointerException if the provided value is null. - */ - public RequestId { - Objects.requireNonNull( - value, - "Image Processing Event ID must not be null" - ); - } - - /** - * Generates a new, unique requestId. - * - * @return A new requestId instance with a randomly generated UUID. - */ - public static RequestId nextIdentity() { - return new RequestId(UUID.randomUUID()); - } - - /** - * Parses a string into a {@link RequestId} instance. - * - * @param id The string representation of the {@link RequestId}. - * @return A new {@link RequestId} instance with the parsed UUID. - * @throws IllegalArgumentException if the provided string is not - * a valid UUID. - */ - public static RequestId fromString(final String id) { - Objects.requireNonNull(id, "requestId must not be null"); - UUID uuid; - try { - uuid = UUID.fromString(id); - } catch (final IllegalArgumentException ex) { - throw new IllegalArgumentException("Invalid requestId: " - + ex.getMessage()); - } - return new RequestId(uuid); - } -} diff --git a/worker/src/main/java/com/github/asavershin/images/in/Consumer.java b/worker/src/main/java/com/github/asavershin/images/in/Consumer.java deleted file mode 100644 index 3c70d5a..0000000 --- a/worker/src/main/java/com/github/asavershin/images/in/Consumer.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.github.asavershin.images.in; - -import com.github.asavershin.images.Task; -import com.github.asavershin.images.WorkerManager; -import com.github.asavershin.images.out.Producer; -import jakarta.validation.constraints.NotNull; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import org.apache.kafka.clients.consumer.ConsumerRecord; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.kafka.annotation.KafkaListener; -import org.springframework.kafka.support.Acknowledgment; -import org.springframework.stereotype.Component; -import org.springframework.transaction.annotation.Transactional; - -@Component -@RequiredArgsConstructor -@Slf4j -public class Consumer { - private final WorkerManager manager; - private final Producer producer; - - @Value("${app.wiptopic}") - @NotNull(message = "Topic for KafkaProducer is null") - private String wip; - @Value("${app.donetopic}") - @NotNull(message = "Topic for KafkaProducer is null") - private String done; - - @KafkaListener( - topics = "${app.wiptopic}", - groupId = "${app.group-id}", - concurrency = "${app.replicas}", - containerFactory = "kafkaListenerContainerFactory" - ) - @Transactional - public void consume( - final ConsumerRecord record, - final Acknowledgment acknowledgment - ) { - log.info( - "Received message: {}", - record.value().toString() - ); - var task = manager.start(record.value()); - if (task.getFilters().isEmpty()) { - producer.produce(task, done); - } else { - producer.produce(task, wip); - } - acknowledgment.acknowledge(); - } -} diff --git a/worker/src/main/java/com/github/asavershin/images/Main.java b/worker/src/main/java/com/github/asavershin/worker/Main.java similarity index 100% rename from worker/src/main/java/com/github/asavershin/images/Main.java rename to worker/src/main/java/com/github/asavershin/worker/Main.java diff --git a/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java b/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java new file mode 100644 index 0000000..61d4eaf --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java @@ -0,0 +1,157 @@ +//package com.github.asavershin.images.config; +// +//import com.github.asavershin.images.Task; +//import jakarta.validation.constraints.NotNull; +//import lombok.RequiredArgsConstructor; +//import org.apache.kafka.clients.admin.NewTopic; +//import org.apache.kafka.clients.consumer.ConsumerConfig; +//import org.apache.kafka.clients.producer.ProducerConfig; +//import org.apache.kafka.clients.producer.RoundRobinPartitioner; +//import org.apache.kafka.common.serialization.StringDeserializer; +//import org.apache.kafka.common.serialization.StringSerializer; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +//import org.springframework.context.annotation.Bean; +//import org.springframework.context.annotation.Configuration; +//import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +//import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +//import org.springframework.kafka.core.DefaultKafkaProducerFactory; +//import org.springframework.kafka.core.KafkaTemplate; +//import org.springframework.kafka.core.ProducerFactory; +//import org.springframework.kafka.support.serializer.JsonDeserializer; +//import org.springframework.kafka.support.serializer.JsonSerializer; +// +//import java.util.Map; +//import java.util.function.Consumer; +// +//@Configuration +//@RequiredArgsConstructor +//public class KafkaConf { +// /** +// * Static final string constant for the Kafka template bean name. +// */ +// public static final String WIP_PRODUCER = "wipProducer"; +// /** +// * Static final string constant for the Kafka template bean name. +// */ +// public static final String DONE_PRODUCER = "doneProducer"; +// /** +// * Injected producer properties. +// */ +// @NotNull(message = "KafkaConf is invalid: prodProp is null") +// private final ProducerProp prodProp; +// /** +// * Injected Kafka properties. +// */ +// @NotNull(message = "KafkaConf is invalid: KafkaProperties is null") +// private final KafkaProperties properties; +// /** +// * Creates a new Kafka topic. +// * +// * @param topic the name of the topic +// * @param partitions the number of partitions +// * @param replicas the number of replicas +// * @return a new Kafka topic +// */ +// @Bean +// public NewTopic wipTopic( +// final @Value("${app.wiptopic}") String topic, +// final @Value("${app.partitions}") int partitions, +// final @Value("${app.replicas}") short replicas) { +// +// return new NewTopic(topic, partitions, replicas); +// } +// /** +// * Creates a new Kafka topic with the specified name, +// * number of partitions, and replicas. +// * +// * @param topic the name of the topic +// * @param partitions the number of partitions for the topic +// * @param replicas the number of replicas for the topic +// * @return a new Kafka topic with the specified configuration +// */ +// @Bean +// public NewTopic doneTopic( +// final @Value("${app.donetopic}") String topic, +// final @Value("${app.partitions}") int partitions, +// final @Value("${app.replicas}") short replicas) { +// +// return new NewTopic(topic, partitions, replicas); +// } +// /** +// * Creates a new Kafka template with all acknowledgments. +// * +// * @return a new Kafka template with all acknowledgments +// */ +// @Bean(WIP_PRODUCER) +// public KafkaTemplate wipProducer() { +// return new KafkaTemplate<>(producerFactory( +// props -> { +// props.put(ProducerConfig.ACKS_CONFIG, "all"); +// props.put(ProducerConfig.RETRIES_CONFIG, +// prodProp.getRetries()); +// }) +// ); +// } +// /** +// * Creates a new Kafka template with all acknowledgments. +// * +// * @return a new Kafka template with all acknowledgments +// */ +// @Bean(DONE_PRODUCER) +// public KafkaTemplate doneProducer() { +// return new KafkaTemplate<>(producerFactory( +// props -> { +// props.put(ProducerConfig.ACKS_CONFIG, "all"); +// props.put(ProducerConfig.RETRIES_CONFIG, +// prodProp.getRetries()); +// }) +// ); +// } +// +// private ProducerFactory producerFactory( +// final Consumer> enchanter +// ) { +// var props = properties.buildProducerProperties(null); +// // Работаем со строками +// props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, +// StringSerializer.class); +// props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, +// JsonSerializer.class); +// // Партиция одна, так что все равно как роутить +// props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, +// RoundRobinPartitioner.class); +// // Отправляем сообщения сразу +// props.put(ProducerConfig.LINGER_MS_CONFIG, 0); +// // До-обогащаем конфигурацию +// enchanter.accept(props); +// return new DefaultKafkaProducerFactory<>(props); +// } +// /** +// * Creates a Kafka listener container factory. +// * +// * @return a new Kafka listener container factory +// */ +// @Bean +// public ConcurrentKafkaListenerContainerFactory +// kafkaListenerContainerFactory() { +// var props = properties.buildConsumerProperties(null); +// props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); +// props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); +// props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "5"); +// props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); +// ConcurrentKafkaListenerContainerFactory +// factory = new ConcurrentKafkaListenerContainerFactory<>(); +// factory.setConsumerFactory( +// new DefaultKafkaConsumerFactory<>( +// props, +// new StringDeserializer(), +// new JsonDeserializer<>(Task.class) +// ) +// ); +// return factory; +// } +// +// +//} + diff --git a/worker/src/main/java/com/github/asavershin/images/config/MinIOConf.java b/worker/src/main/java/com/github/asavershin/worker/config/MinIOConf.java similarity index 100% rename from worker/src/main/java/com/github/asavershin/images/config/MinIOConf.java rename to worker/src/main/java/com/github/asavershin/worker/config/MinIOConf.java diff --git a/worker/src/main/java/com/github/asavershin/images/config/MinIOProperties.java b/worker/src/main/java/com/github/asavershin/worker/config/MinIOProperties.java similarity index 100% rename from worker/src/main/java/com/github/asavershin/images/config/MinIOProperties.java rename to worker/src/main/java/com/github/asavershin/worker/config/MinIOProperties.java diff --git a/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java b/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java new file mode 100644 index 0000000..47407fb --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java @@ -0,0 +1,19 @@ +//package com.github.asavershin.images.config; +// +//import jakarta.validation.constraints.NotNull; +//import lombok.Data; +//import lombok.NoArgsConstructor; +//import org.springframework.beans.factory.annotation.Value; +//import org.springframework.stereotype.Component; +// +//@Data +//@Component +//@NoArgsConstructor +//public class ProducerProp { +// /** +// * Count of publisher tries. +// */ +// @Value("${app.retries}") +// @NotNull(message = "Producer properties are invalid") +// private String retries; +//} diff --git a/worker/src/main/java/com/github/asavershin/images/config/RedisConf.java b/worker/src/main/java/com/github/asavershin/worker/config/RedisConf.java similarity index 100% rename from worker/src/main/java/com/github/asavershin/images/config/RedisConf.java rename to worker/src/main/java/com/github/asavershin/worker/config/RedisConf.java diff --git a/worker/src/main/java/com/github/asavershin/images/Task.java b/worker/src/main/java/com/github/asavershin/worker/dto/Task.java similarity index 87% rename from worker/src/main/java/com/github/asavershin/images/Task.java rename to worker/src/main/java/com/github/asavershin/worker/dto/Task.java index 761e6e4..cfe7217 100644 --- a/worker/src/main/java/com/github/asavershin/images/Task.java +++ b/worker/src/main/java/com/github/asavershin/worker/dto/Task.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images; +package com.github.asavershin.worker; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java b/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java new file mode 100644 index 0000000..7fbf9ff --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java @@ -0,0 +1,77 @@ +//package com.github.asavershin.images.in; +// +//import com.github.asavershin.images.DoneTask; +//import com.github.asavershin.images.Task; +//import com.github.asavershin.images.WorkerManager; +//import com.github.asavershin.images.out.CacheRepository; +//import com.github.asavershin.images.out.DoneProducer; +//import com.github.asavershin.images.out.WipProducer; +//import lombok.RequiredArgsConstructor; +//import lombok.extern.slf4j.Slf4j; +//import org.springframework.kafka.annotation.KafkaListener; +//import org.springframework.messaging.handler.annotation.Payload; +//import org.springframework.stereotype.Component; +//import org.springframework.transaction.annotation.Transactional; +// +//import java.util.List; +//import java.util.concurrent.ExecutorService; +//import java.util.concurrent.Executors; +// +//@Component +//@RequiredArgsConstructor +//@Slf4j +//public class Consumer { +// private final WorkerManager manager; +// private final WipProducer wipProducer; +// private final DoneProducer doneProducer; +// private final CacheRepository cacheRepository; +// +// private final ExecutorService executorService = Executors.newFixedThreadPool(5); +// +// @KafkaListener( +// topics = "${app.wiptopic}", +// groupId = "${app.group-id}", +// concurrency = "${app.replicas}", +// containerFactory = "kafkaListenerContainerFactory" +// ) +// @Transactional +// public void consume( +// final @Payload List records +// ) { +// log.info("Consumed: {}", records.size()); +// for (var record : records) { +// executorService.execute(() -> processRecord(record)); +// } +// } +// +// private void processRecord( +// final Task record +// ) { +// try { +// log.info("Received message: {}", record.toString()); +// var task = manager.start(record); +// if (task == null) { +// return; +// } +// if (task.getFilters().isEmpty()) { +// doneProducer.produce(new DoneTask(task.getImageId(), task.getRequestId())); +// } else { +// wipProducer.produce(task); +// } +// } catch (Exception ex) { +// log.info(ex.getMessage()); +// cacheRepository.deleteCache( +// record.getRequestId() + record.getImageId() +// ); +// wipProducer.produce(record); +// } +// } +//} +// +////{ +//// "imageId": "example", +//// "requestId": "987654", +//// "filters": [ +//// "ROTATE" +//// ] +////} diff --git a/worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java b/worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java similarity index 95% rename from worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java rename to worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java index 126efd0..5f6a850 100644 --- a/worker/src/main/java/com/github/asavershin/images/out/CacheRepository.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java @@ -19,4 +19,6 @@ public interface CacheRepository { * or null if the entry does not exist or has expired */ String getCache(String key); + + void deleteCache(String key); } diff --git a/worker/src/main/java/com/github/asavershin/images/out/FileException.java b/worker/src/main/java/com/github/asavershin/worker/out/FileException.java similarity index 100% rename from worker/src/main/java/com/github/asavershin/images/out/FileException.java rename to worker/src/main/java/com/github/asavershin/worker/out/FileException.java diff --git a/worker/src/main/java/com/github/asavershin/images/out/MinioService.java b/worker/src/main/java/com/github/asavershin/worker/out/MinioService.java similarity index 100% rename from worker/src/main/java/com/github/asavershin/images/out/MinioService.java rename to worker/src/main/java/com/github/asavershin/worker/out/MinioService.java diff --git a/worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java b/worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java similarity index 82% rename from worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java rename to worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java index c604007..89b15c0 100644 --- a/worker/src/main/java/com/github/asavershin/images/out/CacheRepositoryIml.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images.out; +package com.github.asavershin.worker.out; import lombok.RequiredArgsConstructor; import org.springframework.data.redis.core.RedisTemplate; @@ -33,4 +33,12 @@ public void addCache(final String key, public String getCache(final String key) { return (String) redisTemplate.opsForValue().get(key); } + + /** + * Not final to allow spring use proxy. + */ + @Override + public void deleteCache(final String key) { + redisTemplate.delete(key); + } } diff --git a/worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java b/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java similarity index 97% rename from worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java rename to worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java index af1b8d3..9f01e52 100644 --- a/worker/src/main/java/com/github/asavershin/images/out/MinioServiceIml.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java @@ -1,7 +1,7 @@ -package com.github.asavershin.images.out; +package com.github.asavershin.worker.out; -import com.github.asavershin.images.config.MinIOProperties; +import com.github.asavershin.worker.config.MinIOProperties; import io.minio.BucketExistsArgs; import io.minio.GetObjectArgs; import io.minio.MakeBucketArgs; diff --git a/worker/src/main/java/com/github/asavershin/images/out/Producer.java b/worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java similarity index 51% rename from worker/src/main/java/com/github/asavershin/images/out/Producer.java rename to worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java index 3c6923d..7c747a2 100644 --- a/worker/src/main/java/com/github/asavershin/images/out/Producer.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java @@ -1,26 +1,30 @@ -package com.github.asavershin.images.out; +package com.github.asavershin.worker.out; -import com.github.asavershin.images.Task; +import com.github.asavershin.worker.Task; +import jakarta.validation.constraints.NotNull; import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; import org.springframework.kafka.core.KafkaTemplate; import org.springframework.stereotype.Component; import org.springframework.transaction.annotation.Transactional; -import static com.github.asavershin.images.config.KafkaConf.ALL_ACKS_KAFKA_TEMPLATE; +import static com.github.asavershin.worker.config.KafkaConf.WIP_PRODUCER; @Component -public class Producer { +public class WipProducer { private final KafkaTemplate producer; - - public Producer( - final @Qualifier(ALL_ACKS_KAFKA_TEMPLATE) + @Value("${app.wiptopic}") + @NotNull(message = "Topic for KafkaProducer is null") + private String topic; + public WipProducer( + final @Qualifier(WIP_PRODUCER) KafkaTemplate allAcksKafkaTemplate ) { producer = allAcksKafkaTemplate; } @Transactional - public void produce(final Task event, final String topic) { + public void produce(final Task event) { producer.send( topic, event diff --git a/worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java b/worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java similarity index 94% rename from worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java rename to worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java index 6d4046f..6cc2e0f 100644 --- a/worker/src/main/java/com/github/asavershin/images/BlackWhiteWorker.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java @@ -1,7 +1,7 @@ -package com.github.asavershin.images; +package com.github.asavershin.worker; -import com.github.asavershin.images.config.MinIOProperties; -import com.github.asavershin.images.out.MinioService; +import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.out.MinioService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Profile; diff --git a/worker/src/main/java/com/github/asavershin/images/RotateImage.java b/worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java similarity index 84% rename from worker/src/main/java/com/github/asavershin/images/RotateImage.java rename to worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java index 79f4d7f..8cddc62 100644 --- a/worker/src/main/java/com/github/asavershin/images/RotateImage.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java @@ -1,22 +1,15 @@ -package com.github.asavershin.images; +package com.github.asavershin.worker; -import com.github.asavershin.images.config.MinIOProperties; -import com.github.asavershin.images.out.MinioService; -import io.minio.MinioProperties; +import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.out.MinioService; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; import org.springframework.context.annotation.Profile; import org.springframework.stereotype.Service; import javax.imageio.ImageIO; -import java.awt.*; -import java.awt.geom.AffineTransform; -import java.awt.image.AffineTransformOp; import java.awt.image.BufferedImage; -import java.awt.image.DataBufferByte; -import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; -import java.io.File; import java.io.IOException; import java.io.InputStream; import java.util.UUID; diff --git a/worker/src/main/java/com/github/asavershin/images/Worker.java b/worker/src/main/java/com/github/asavershin/worker/services/Worker.java similarity index 61% rename from worker/src/main/java/com/github/asavershin/images/Worker.java rename to worker/src/main/java/com/github/asavershin/worker/services/Worker.java index 77df9f8..48467db 100644 --- a/worker/src/main/java/com/github/asavershin/images/Worker.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/Worker.java @@ -1,6 +1,4 @@ -package com.github.asavershin.images; - -import java.io.IOException; +package com.github.asavershin.worker; public interface Worker { String doWork(String imageId, boolean lastWorker); diff --git a/worker/src/main/java/com/github/asavershin/images/WorkerManager.java b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java similarity index 63% rename from worker/src/main/java/com/github/asavershin/images/WorkerManager.java rename to worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java index b09ac16..bc884ca 100644 --- a/worker/src/main/java/com/github/asavershin/images/WorkerManager.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images; +package com.github.asavershin.worker; public interface WorkerManager { Task start(Task filters); diff --git a/worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java similarity index 78% rename from worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java rename to worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java index 97e8257..d27a260 100644 --- a/worker/src/main/java/com/github/asavershin/images/WorkerManagerImpl.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java @@ -1,6 +1,6 @@ -package com.github.asavershin.images; +package com.github.asavershin.worker; -import com.github.asavershin.images.out.CacheRepository; +import com.github.asavershin.worker.out.CacheRepository; import jakarta.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; import org.springframework.beans.factory.annotation.Value; @@ -19,25 +19,26 @@ public class WorkerManagerImpl implements WorkerManager { @Override public Task start(final Task task) { + if (!Objects.equals( + task.getFilters().get(0), + worker.whoAmI())) { + throw new RuntimeException("Invalid worker"); + } var cached = cache.getCache( task.getRequestId() + task.getImageId()); if (task.getFilters().isEmpty() - || !Objects.equals( - task.getFilters().get(0), - worker.whoAmI()) || cached != null ) { return null; } - - var imageId = worker.doWork(task.getImageId(), - task.getFilters().size() == 1 - ); cache.addCache( - task.getRequestId() + imageId, + task.getRequestId() + task.getImageId(), "", cacheExp ); + var imageId = worker.doWork(task.getImageId(), + task.getFilters().size() == 1 + ); task.getFilters().remove(0); return task; } From f44f575f8f66f04fece257fb5a9a3a38a4f43c88 Mon Sep 17 00:00:00 2001 From: asavershin Date: Mon, 6 May 2024 22:20:25 +0300 Subject: [PATCH 3/7] Add tests --- .../github/asavershin/worker/ImageHelper.java | 42 +++++++++++++ .../worker/config/CacheRedisConfig.java | 44 ++++++++++++++ .../asavershin/worker/config/MinioConfig.java | 52 ++++++++++++++++ .../integrations/BlackWhiteFilterTest.java | 60 +++++++++++++++++++ .../worker/integrations/BlackWhiteTest.java | 15 +++++ .../worker/integrations/RotateFilterTest.java | 60 +++++++++++++++++++ .../worker/integrations/RotateTest.java | 15 +++++ .../src/test/resources/application.properties | 13 ++++ 8 files changed, 301 insertions(+) create mode 100644 worker/src/test/java/com/github/asavershin/worker/ImageHelper.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/config/CacheRedisConfig.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/config/MinioConfig.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteFilterTest.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteTest.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/integrations/RotateFilterTest.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/integrations/RotateTest.java create mode 100644 worker/src/test/resources/application.properties diff --git a/worker/src/test/java/com/github/asavershin/worker/ImageHelper.java b/worker/src/test/java/com/github/asavershin/worker/ImageHelper.java new file mode 100644 index 0000000..32f77b9 --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/ImageHelper.java @@ -0,0 +1,42 @@ +package com.github.asavershin.worker; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.mock.web.MockMultipartFile; +import org.springframework.web.multipart.MultipartFile; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.awt.image.BufferedImage; +import java.io.ByteArrayOutputStream; + +@Slf4j +public class ImageHelper { + public static byte[] createTestImage(int width, int height) { + try { + // Создаем пустое изображение с заданными размерами + BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB); + + // Задаем цвета для полос на изображении + Color[] colors = {Color.RED, Color.GREEN, Color.BLUE, Color.YELLOW, Color.ORANGE}; + + // Рисуем полосы разных цветов + Graphics2D graphics = image.createGraphics(); + for (int i = 0; i < colors.length; i++) { + graphics.setColor(colors[i]); + graphics.fillRect(0, i * height / colors.length, width, (i + 1) * height / colors.length); + } + graphics.dispose(); + + // Конвертируем изображение в массив байтов + ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + ImageIO.write(image, "png", outputStream); + byte[] imageData = outputStream.toByteArray(); + outputStream.close(); + + return imageData; + } catch (Exception e) { + log.error(e.getMessage()); + throw new RuntimeException("Error with creating test image"); + } + } +} diff --git a/worker/src/test/java/com/github/asavershin/worker/config/CacheRedisConfig.java b/worker/src/test/java/com/github/asavershin/worker/config/CacheRedisConfig.java new file mode 100644 index 0000000..43fed6a --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/config/CacheRedisConfig.java @@ -0,0 +1,44 @@ +package com.github.asavershin.worker.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.stereotype.Component; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.utility.DockerImageName; + +import static java.util.Objects.isNull; + +@Slf4j +public class CacheRedisConfig { + + private static volatile GenericContainer redisContainer = null; + + private static GenericContainer getRedisContainer() { + var instance = redisContainer; + if (isNull(redisContainer)) { + synchronized (GenericContainer.class) { + redisContainer = instance = new GenericContainer<>( + DockerImageName.parse("redis:7.2-rc-alpine")) + .withExposedPorts(6379); + redisContainer.start(); + } + } + return instance; + } + + @Component("RedisInitializer") + public static class Initializer implements ApplicationContextInitializer { + @Override + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + var redisContainer = getRedisContainer(); + TestPropertyValues.of( + "spring.data.redis.host=" + redisContainer.getHost(), + "spring.data.redis.port=" + redisContainer.getMappedPort(6379) + ).applyTo(configurableApplicationContext.getEnvironment()); + } + } + +} + diff --git a/worker/src/test/java/com/github/asavershin/worker/config/MinioConfig.java b/worker/src/test/java/com/github/asavershin/worker/config/MinioConfig.java new file mode 100644 index 0000000..4620442 --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/config/MinioConfig.java @@ -0,0 +1,52 @@ +package com.github.asavershin.worker.config; + +import lombok.extern.slf4j.Slf4j; +import org.springframework.boot.test.util.TestPropertyValues; +import org.springframework.context.ApplicationContextInitializer; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.stereotype.Component; +import org.testcontainers.containers.MinIOContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; + +import java.time.Duration; + +import static java.util.Objects.isNull; + +@Slf4j +public class MinioConfig { + + private static volatile MinIOContainer minioContainer = null; + + + private static MinIOContainer getMinioContainer() { + MinIOContainer instance = minioContainer; + if (isNull(minioContainer)) { + synchronized (MinIOContainer.class) { + minioContainer = instance = + new MinIOContainer("minio/minio:latest") + .withReuse(true) + .withLogConsumer(new Slf4jLogConsumer(log)) + .withExposedPorts(9000) + .withUserName("minioadmin") + .withPassword("minioadmin") + .withStartupTimeout(Duration.ofSeconds(60)); + minioContainer.start(); + } + } + return instance; + } + + @Component("MinioInitializer") + public static class Initializer implements ApplicationContextInitializer { + @Override + public void initialize(ConfigurableApplicationContext configurableApplicationContext) { + var minioContainer = getMinioContainer(); + TestPropertyValues.of( + "minio.url=" + minioContainer.getS3URL(), + "minio.access-key=" + minioContainer.getUserName(), + "minio.secret-key=" + minioContainer.getPassword() + ).applyTo(configurableApplicationContext.getEnvironment()); + } + } + +} diff --git a/worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteFilterTest.java b/worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteFilterTest.java new file mode 100644 index 0000000..37fca8e --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteFilterTest.java @@ -0,0 +1,60 @@ +package com.github.asavershin.worker.integrations; + +import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.out.MinioService; +import com.github.asavershin.worker.services.Worker; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.messages.Item; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.atomic.AtomicInteger; + +import static com.github.asavershin.worker.ImageHelper.createTestImage; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class BlackWhiteFilterTest extends BlackWhiteTest{ + @Autowired + private MinioService minioService; + @Autowired + private Worker worker; + @Autowired + private MinioClient minioClient; + @Autowired + private MinIOProperties minioProperties; + + @BeforeEach + public void clearDB() { + try { + Iterable> files = minioClient.listObjects(ListObjectsArgs.builder().bucket(minioProperties.getBucket()).build()); + + for (var file : files) { + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minioProperties.getBucket()) + .object(file.get().objectName()) + .build()); + } + } catch (Exception e) { + log.info("Clean bucket fail"); + e.printStackTrace(); + } + } + + @Test + public void rotateWorkerTest() { + minioService.saveFile(createTestImage(100, 200), "example"); + worker.doWork("example", true); + + var objectsInMinio = minioClient.listObjects(ListObjectsArgs.builder().bucket("files").build()); + AtomicInteger countObjectsInMinio = new AtomicInteger(); + objectsInMinio.forEach( it -> countObjectsInMinio.addAndGet(1)); + assertEquals(2, countObjectsInMinio.get()); + } +} diff --git a/worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteTest.java b/worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteTest.java new file mode 100644 index 0000000..e6c9d48 --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/integrations/BlackWhiteTest.java @@ -0,0 +1,15 @@ +package com.github.asavershin.worker.integrations; + +import com.github.asavershin.worker.config.CacheRedisConfig; +import com.github.asavershin.worker.config.MinioConfig; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; + +@SpringBootTest +@EmbeddedKafka +@ContextConfiguration(initializers = {MinioConfig.Initializer.class, CacheRedisConfig.Initializer.class}) +@ActiveProfiles("blackwhite") +public abstract class BlackWhiteTest { +} diff --git a/worker/src/test/java/com/github/asavershin/worker/integrations/RotateFilterTest.java b/worker/src/test/java/com/github/asavershin/worker/integrations/RotateFilterTest.java new file mode 100644 index 0000000..f641c32 --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/integrations/RotateFilterTest.java @@ -0,0 +1,60 @@ +package com.github.asavershin.worker.integrations; + +import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.services.Worker; +import com.github.asavershin.worker.out.MinioService; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.MinioProperties; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.messages.Item; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import java.util.concurrent.atomic.AtomicInteger; + +import static com.github.asavershin.worker.ImageHelper.createTestImage; +import static org.junit.jupiter.api.Assertions.assertEquals; +@Slf4j +public class RotateFilterTest extends RotateTest{ + @Autowired + private MinioService minioService; + @Autowired + private Worker worker; + @Autowired + private MinioClient minioClient; + @Autowired + private MinIOProperties minioProperties; + + @BeforeEach + public void clearDB() { + try { + Iterable> files = minioClient.listObjects(ListObjectsArgs.builder().bucket(minioProperties.getBucket()).build()); + + for (var file : files) { + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minioProperties.getBucket()) + .object(file.get().objectName()) + .build()); + } + } catch (Exception e) { + log.info("Clean bucket fail"); + e.printStackTrace(); + } + } + + @Test + public void rotateWorkerTest() { + minioService.saveFile(createTestImage(100, 200), "example"); + worker.doWork("example", true); + + var objectsInMinio = minioClient.listObjects(ListObjectsArgs.builder().bucket("files").build()); + AtomicInteger countObjectsInMinio = new AtomicInteger(); + objectsInMinio.forEach( it -> countObjectsInMinio.addAndGet(1)); + assertEquals(2, countObjectsInMinio.get()); + } +} diff --git a/worker/src/test/java/com/github/asavershin/worker/integrations/RotateTest.java b/worker/src/test/java/com/github/asavershin/worker/integrations/RotateTest.java new file mode 100644 index 0000000..b4ed2be --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/integrations/RotateTest.java @@ -0,0 +1,15 @@ +package com.github.asavershin.worker.integrations; + +import com.github.asavershin.worker.config.CacheRedisConfig; +import com.github.asavershin.worker.config.MinioConfig; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; + +@SpringBootTest +@EmbeddedKafka +@ContextConfiguration(initializers = {MinioConfig.Initializer.class, CacheRedisConfig.Initializer.class}) +@ActiveProfiles("rotate") +public abstract class RotateTest { +} diff --git a/worker/src/test/resources/application.properties b/worker/src/test/resources/application.properties new file mode 100644 index 0000000..749d4ed --- /dev/null +++ b/worker/src/test/resources/application.properties @@ -0,0 +1,13 @@ +spring.data.redis.host=redis-rotate +spring.data.redis.port=6379 +spring.cache.type=redis +spring.cache.cache-names=redis-cache + +cache-expiration=86400000 + +minio.url=http://minio-api:9000 +minio.user=minioadmin +minio.password=minioadmin +minio.bucket=files +minio.console-port=9090 +minio.port=9000 From 24cc8efac016a00a496cfd90d6e658c23ef69bca Mon Sep 17 00:00:00 2001 From: asavershin Date: Mon, 6 May 2024 22:20:39 +0300 Subject: [PATCH 4/7] Add workers --- README.md | 3 + pom.xml | 10 - worker/.env | 3 +- worker/Dockerfile-blackwhite | 7 + worker/docker-compose-blackwhite.yml | 52 +++ worker/docker-compose-rotate.yml | 30 +- worker/pom.xml | 60 ++++ .../asavershin/worker/InvalidWorker.java | 7 + .../com/github/asavershin/worker/Main.java | 2 +- .../asavershin/worker/config/KafkaConf.java | 316 +++++++++--------- .../asavershin/worker/config/MinIOConf.java | 2 +- .../worker/config/MinIOProperties.java | 2 +- .../worker/config/ProducerProp.java | 38 +-- .../asavershin/worker/config/RedisConf.java | 2 +- .../asavershin/worker/dto/DoneTask.java | 13 + .../github/asavershin/worker/dto/Task.java | 2 +- .../github/asavershin/worker/in/Consumer.java | 138 ++++---- .../worker/out/CacheRepository.java | 2 +- .../asavershin/worker/out/FileException.java | 2 +- .../asavershin/worker/out/MinioService.java | 4 +- .../worker/out/impl/CacheRepositoryIml.java | 3 +- .../worker/out/impl/MinioServiceIml.java | 6 +- .../worker/out/producers/DoneProducer.java | 33 ++ .../worker/out/producers/WipProducer.java | 4 +- .../worker/services/BlackWhiteWorker.java | 4 +- .../worker/services/RotateImage.java | 4 +- .../asavershin/worker/services/Worker.java | 2 +- .../worker/services/WorkerManager.java | 4 +- .../worker/services/WorkerManagerImpl.java | 7 +- .../main/resources/application-blackwhite.yml | 25 ++ .../src/main/resources/application-rotate.yml | 25 ++ worker/src/main/resources/application.yml | 25 -- 32 files changed, 511 insertions(+), 326 deletions(-) create mode 100644 worker/Dockerfile-blackwhite create mode 100644 worker/docker-compose-blackwhite.yml create mode 100644 worker/src/main/java/com/github/asavershin/worker/InvalidWorker.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/dto/DoneTask.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/out/producers/DoneProducer.java diff --git a/README.md b/README.md index c071ffc..5a2bd40 100644 --- a/README.md +++ b/README.md @@ -8,5 +8,8 @@ docker compose -f kafka/docker-compose.yml up --build -d docker compose -f api/docker-compose.yml up --build -d +docker compose -f worker/docker-compose-blackwhite.yml up --build -d +docker compose -f worker/docker-compose-rotate.yml up --build -d + diff --git a/pom.xml b/pom.xml index cda65f6..c94b1eb 100644 --- a/pom.xml +++ b/pom.xml @@ -46,16 +46,6 @@ ${org.projectlombok.version} true - - org.springframework.kafka - spring-kafka - - - - org.springframework.kafka - spring-kafka-test - test - diff --git a/worker/.env b/worker/.env index 4f33ad5..ebf2e6c 100644 --- a/worker/.env +++ b/worker/.env @@ -11,7 +11,8 @@ MINIO_CONSOLE_PORT=9090 MINIO_PORT=9000 # REDIS -REDIS_HOST=redis-rotate +REDIS_HOST_ROTATE=redis-rotate +REDIS_HOST_BLACKWHITE=redis-blackwhite REDIS_PORT=6379 REDIS_PASSWORD=cGFzc3dvcmxk REDIS_CACHE_TIME=86400000 diff --git a/worker/Dockerfile-blackwhite b/worker/Dockerfile-blackwhite new file mode 100644 index 0000000..d71a660 --- /dev/null +++ b/worker/Dockerfile-blackwhite @@ -0,0 +1,7 @@ +FROM openjdk:21 + +WORKDIR /app + +COPY target/*.jar app.jar + +CMD ["java", "-jar", "-Dspring.profiles.active=blackwhite", "app.jar"] \ No newline at end of file diff --git a/worker/docker-compose-blackwhite.yml b/worker/docker-compose-blackwhite.yml new file mode 100644 index 0000000..ff8c902 --- /dev/null +++ b/worker/docker-compose-blackwhite.yml @@ -0,0 +1,52 @@ +networks: + kafka-net: + name: kafka-net + driver: bridge + api-net: + name: api-net + driver: bridge + +services: + + backend-worker-blackwhite: + container_name: backend-worker-blackwhite + networks: + - kafka-net + - api-net + build: + context: . + dockerfile: Dockerfile-blackwhite + ports: + - 8083:8081 + env_file: + - .env + + redis-blackwhite: + networks: + - api-net + image: redis:7.2-rc-alpine + restart: always + container_name: redis-blackwhite + ports: + - '6381:6379' + command: redis-server --save 20 1 --loglevel debug --requirepass ${REDIS_PASSWORD} + volumes: + - redis-rotate-data:/data + +# minio-api: +# networks: +# - api-net +# image: minio/minio:RELEASE.2024-02-14T21-36-02Z +# container_name: minio-api +# env_file: +# - .env +# command: server ~/minio --console-address :9090 +# ports: +# - '9090:9090' +# - '9000:9000' +# volumes: +# - minio-api-data:/minio + +volumes: +# minio-api-data: + redis-rotate-data: diff --git a/worker/docker-compose-rotate.yml b/worker/docker-compose-rotate.yml index 8688dab..d7d4d88 100644 --- a/worker/docker-compose-rotate.yml +++ b/worker/docker-compose-rotate.yml @@ -17,7 +17,7 @@ services: context: . dockerfile: Dockerfile-rotate ports: - - 8081:8081 + - 8082:8081 env_file: - .env @@ -33,20 +33,20 @@ services: volumes: - redis-rotate-data:/data - minio-api: - networks: - - api-net - image: minio/minio:RELEASE.2024-02-14T21-36-02Z - container_name: minio-api - env_file: - - .env - command: server ~/minio --console-address :9090 - ports: - - '9090:9090' - - '9000:9000' - volumes: - - minio-api-data:/minio +# minio-api: +# networks: +# - api-net +# image: minio/minio:RELEASE.2024-02-14T21-36-02Z +# container_name: minio-api +# env_file: +# - .env +# command: server ~/minio --console-address :9090 +# ports: +# - '9090:9090' +# - '9000:9000' +# volumes: +# - minio-api-data:/minio volumes: - minio-api-data: +# minio-api-data: redis-rotate-data: diff --git a/worker/pom.xml b/worker/pom.xml index d47d4f4..89f6ddf 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -16,6 +16,7 @@ 21 UTF-8 8.5.7 + 1.19.1 @@ -38,6 +39,37 @@ spring-boot-starter-data-redis + + + + org.testcontainers + minio + ${containers.version} + test + + + + org.testcontainers + testcontainers + ${containers.version} + test + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.kafka + spring-kafka + + + + org.springframework.kafka + spring-kafka-test + test + + @@ -55,6 +87,34 @@ + + + org.jacoco + jacoco-maven-plugin + 0.8.10 + + + + prepare-agent + + + + report + test + + report + + + + + + + com/github/asavershin/worker/services/** + com/github/asavershin/worker/out/producers/** + com/github/asavershin/worker/in/** + + + diff --git a/worker/src/main/java/com/github/asavershin/worker/InvalidWorker.java b/worker/src/main/java/com/github/asavershin/worker/InvalidWorker.java new file mode 100644 index 0000000..9180854 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/InvalidWorker.java @@ -0,0 +1,7 @@ +package com.github.asavershin.worker; + +public class InvalidWorker extends RuntimeException { + public InvalidWorker(final String invalidWorker) { + super(invalidWorker); + } +} diff --git a/worker/src/main/java/com/github/asavershin/worker/Main.java b/worker/src/main/java/com/github/asavershin/worker/Main.java index 6f94a28..40ec739 100644 --- a/worker/src/main/java/com/github/asavershin/worker/Main.java +++ b/worker/src/main/java/com/github/asavershin/worker/Main.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images; +package com.github.asavershin.worker; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; diff --git a/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java b/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java index 61d4eaf..581afe0 100644 --- a/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java +++ b/worker/src/main/java/com/github/asavershin/worker/config/KafkaConf.java @@ -1,157 +1,161 @@ -//package com.github.asavershin.images.config; -// -//import com.github.asavershin.images.Task; -//import jakarta.validation.constraints.NotNull; -//import lombok.RequiredArgsConstructor; -//import org.apache.kafka.clients.admin.NewTopic; -//import org.apache.kafka.clients.consumer.ConsumerConfig; -//import org.apache.kafka.clients.producer.ProducerConfig; -//import org.apache.kafka.clients.producer.RoundRobinPartitioner; -//import org.apache.kafka.common.serialization.StringDeserializer; -//import org.apache.kafka.common.serialization.StringSerializer; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.boot.autoconfigure.kafka.KafkaProperties; -//import org.springframework.context.annotation.Bean; -//import org.springframework.context.annotation.Configuration; -//import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; -//import org.springframework.kafka.core.DefaultKafkaConsumerFactory; -//import org.springframework.kafka.core.DefaultKafkaProducerFactory; -//import org.springframework.kafka.core.KafkaTemplate; -//import org.springframework.kafka.core.ProducerFactory; -//import org.springframework.kafka.support.serializer.JsonDeserializer; -//import org.springframework.kafka.support.serializer.JsonSerializer; -// -//import java.util.Map; -//import java.util.function.Consumer; -// -//@Configuration -//@RequiredArgsConstructor -//public class KafkaConf { -// /** -// * Static final string constant for the Kafka template bean name. -// */ -// public static final String WIP_PRODUCER = "wipProducer"; -// /** -// * Static final string constant for the Kafka template bean name. -// */ -// public static final String DONE_PRODUCER = "doneProducer"; -// /** -// * Injected producer properties. -// */ -// @NotNull(message = "KafkaConf is invalid: prodProp is null") -// private final ProducerProp prodProp; -// /** -// * Injected Kafka properties. -// */ -// @NotNull(message = "KafkaConf is invalid: KafkaProperties is null") -// private final KafkaProperties properties; -// /** -// * Creates a new Kafka topic. -// * -// * @param topic the name of the topic -// * @param partitions the number of partitions -// * @param replicas the number of replicas -// * @return a new Kafka topic -// */ -// @Bean -// public NewTopic wipTopic( -// final @Value("${app.wiptopic}") String topic, -// final @Value("${app.partitions}") int partitions, -// final @Value("${app.replicas}") short replicas) { -// -// return new NewTopic(topic, partitions, replicas); -// } -// /** -// * Creates a new Kafka topic with the specified name, -// * number of partitions, and replicas. -// * -// * @param topic the name of the topic -// * @param partitions the number of partitions for the topic -// * @param replicas the number of replicas for the topic -// * @return a new Kafka topic with the specified configuration -// */ -// @Bean -// public NewTopic doneTopic( -// final @Value("${app.donetopic}") String topic, -// final @Value("${app.partitions}") int partitions, -// final @Value("${app.replicas}") short replicas) { -// -// return new NewTopic(topic, partitions, replicas); -// } -// /** -// * Creates a new Kafka template with all acknowledgments. -// * -// * @return a new Kafka template with all acknowledgments -// */ -// @Bean(WIP_PRODUCER) -// public KafkaTemplate wipProducer() { -// return new KafkaTemplate<>(producerFactory( -// props -> { -// props.put(ProducerConfig.ACKS_CONFIG, "all"); -// props.put(ProducerConfig.RETRIES_CONFIG, -// prodProp.getRetries()); -// }) -// ); -// } -// /** -// * Creates a new Kafka template with all acknowledgments. -// * -// * @return a new Kafka template with all acknowledgments -// */ -// @Bean(DONE_PRODUCER) -// public KafkaTemplate doneProducer() { -// return new KafkaTemplate<>(producerFactory( -// props -> { -// props.put(ProducerConfig.ACKS_CONFIG, "all"); -// props.put(ProducerConfig.RETRIES_CONFIG, -// prodProp.getRetries()); -// }) -// ); -// } -// -// private ProducerFactory producerFactory( -// final Consumer> enchanter -// ) { -// var props = properties.buildProducerProperties(null); -// // Работаем со строками -// props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, -// StringSerializer.class); -// props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, -// JsonSerializer.class); -// // Партиция одна, так что все равно как роутить -// props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, -// RoundRobinPartitioner.class); -// // Отправляем сообщения сразу -// props.put(ProducerConfig.LINGER_MS_CONFIG, 0); -// // До-обогащаем конфигурацию -// enchanter.accept(props); -// return new DefaultKafkaProducerFactory<>(props); -// } -// /** -// * Creates a Kafka listener container factory. -// * -// * @return a new Kafka listener container factory -// */ -// @Bean -// public ConcurrentKafkaListenerContainerFactory -// kafkaListenerContainerFactory() { -// var props = properties.buildConsumerProperties(null); -// props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); -// props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); -// props.put(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, "5"); -// props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); -// ConcurrentKafkaListenerContainerFactory -// factory = new ConcurrentKafkaListenerContainerFactory<>(); -// factory.setConsumerFactory( -// new DefaultKafkaConsumerFactory<>( -// props, -// new StringDeserializer(), -// new JsonDeserializer<>(Task.class) -// ) -// ); -// return factory; -// } -// -// -//} +package com.github.asavershin.worker.config; + +import com.github.asavershin.worker.dto.DoneTask; +import com.github.asavershin.worker.dto.Task; +import jakarta.validation.constraints.NotNull; +import lombok.RequiredArgsConstructor; +import org.apache.kafka.clients.admin.NewTopic; +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.clients.producer.RoundRobinPartitioner; +import org.apache.kafka.common.serialization.StringDeserializer; +import org.apache.kafka.common.serialization.StringSerializer; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.kafka.KafkaProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.kafka.config.ConcurrentKafkaListenerContainerFactory; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.ContainerProperties; +import org.springframework.kafka.support.serializer.JsonDeserializer; +import org.springframework.kafka.support.serializer.JsonSerializer; + +import java.util.Map; +import java.util.function.Consumer; + +@Configuration +@RequiredArgsConstructor +public class KafkaConf { + /** + * Static final string constant for the Kafka template bean name. + */ + public static final String WIP_PRODUCER = "wipProducerTemplate"; + /** + * Static final string constant for the Kafka template bean name. + */ + public static final String DONE_PRODUCER = "doneProducerTemplate"; + /** + * Injected producer properties. + */ + @NotNull(message = "KafkaConf is invalid: prodProp is null") + private final ProducerProp prodProp; + /** + * Injected Kafka properties. + */ + @NotNull(message = "KafkaConf is invalid: KafkaProperties is null") + private final KafkaProperties properties; + /** + * Creates a new Kafka topic. + * + * @param topic the name of the topic + * @param partitions the number of partitions + * @param replicas the number of replicas + * @return a new Kafka topic + */ + @Bean + public NewTopic wipTopic( + final @Value("${app.wiptopic}") String topic, + final @Value("${app.partitions}") int partitions, + final @Value("${app.replicas}") short replicas) { + + return new NewTopic(topic, partitions, replicas); + } + /** + * Creates a new Kafka topic with the specified name, + * number of partitions, and replicas. + * + * @param topic the name of the topic + * @param partitions the number of partitions for the topic + * @param replicas the number of replicas for the topic + * @return a new Kafka topic with the specified configuration + */ + @Bean + public NewTopic doneTopic( + final @Value("${app.donetopic}") String topic, + final @Value("${app.partitions}") int partitions, + final @Value("${app.replicas}") short replicas) { + + return new NewTopic(topic, partitions, replicas); + } + /** + * Creates a new Kafka template with all acknowledgments. + * + * @return a new Kafka template with all acknowledgments + */ + @Bean(WIP_PRODUCER) + public KafkaTemplate wipProducerTemplate() { + return new KafkaTemplate<>(producerFactory( + props -> { + props.put(ProducerConfig.ACKS_CONFIG, "all"); + props.put(ProducerConfig.RETRIES_CONFIG, + prodProp.getRetries()); + }) + ); + } + /** + * Creates a new Kafka template with all acknowledgments. + * + * @return a new Kafka template with all acknowledgments + */ + @Bean(DONE_PRODUCER) + public KafkaTemplate doneProducerTemplate() { + return new KafkaTemplate<>(producerFactory( + props -> { + props.put(ProducerConfig.ACKS_CONFIG, "all"); + props.put(ProducerConfig.RETRIES_CONFIG, + prodProp.getRetries()); + }) + ); + } + + private ProducerFactory producerFactory( + final Consumer> enchanter + ) { + var props = properties.buildProducerProperties(null); + // Работаем со строками + props.put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, + StringSerializer.class); + props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, + JsonSerializer.class); + props.put(JsonSerializer.ADD_TYPE_INFO_HEADERS, false); + // Партиция одна, так что все равно как роутить + props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG, + RoundRobinPartitioner.class); + // Отправляем сообщения сразу + props.put(ProducerConfig.LINGER_MS_CONFIG, 0); + // До-обогащаем конфигурацию + enchanter.accept(props); + return new DefaultKafkaProducerFactory<>(props); + } + /** + * Creates a Kafka listener container factory. + * + * @return a new Kafka listener container factory + */ + @Bean + public ConcurrentKafkaListenerContainerFactory + kafkaListenerContainerFactory() { + var props = properties.buildConsumerProperties(null); + props.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false"); + props.put(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed"); + ConcurrentKafkaListenerContainerFactory + factory = new ConcurrentKafkaListenerContainerFactory<>(); + factory.setConsumerFactory( + new DefaultKafkaConsumerFactory<>( + props, + new StringDeserializer(), + new JsonDeserializer<>(Task.class) + ) + ); + factory.getContainerProperties().setAckMode( + ContainerProperties.AckMode.MANUAL + ); + return factory; + } + + +} diff --git a/worker/src/main/java/com/github/asavershin/worker/config/MinIOConf.java b/worker/src/main/java/com/github/asavershin/worker/config/MinIOConf.java index 6411c9a..60cb9c4 100644 --- a/worker/src/main/java/com/github/asavershin/worker/config/MinIOConf.java +++ b/worker/src/main/java/com/github/asavershin/worker/config/MinIOConf.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images.config; +package com.github.asavershin.worker.config; import io.minio.MinioClient; import lombok.RequiredArgsConstructor; diff --git a/worker/src/main/java/com/github/asavershin/worker/config/MinIOProperties.java b/worker/src/main/java/com/github/asavershin/worker/config/MinIOProperties.java index be79b2e..1b926f3 100644 --- a/worker/src/main/java/com/github/asavershin/worker/config/MinIOProperties.java +++ b/worker/src/main/java/com/github/asavershin/worker/config/MinIOProperties.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images.config; +package com.github.asavershin.worker.config; import lombok.Data; import lombok.NoArgsConstructor; diff --git a/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java b/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java index 47407fb..7dca52f 100644 --- a/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java +++ b/worker/src/main/java/com/github/asavershin/worker/config/ProducerProp.java @@ -1,19 +1,19 @@ -//package com.github.asavershin.images.config; -// -//import jakarta.validation.constraints.NotNull; -//import lombok.Data; -//import lombok.NoArgsConstructor; -//import org.springframework.beans.factory.annotation.Value; -//import org.springframework.stereotype.Component; -// -//@Data -//@Component -//@NoArgsConstructor -//public class ProducerProp { -// /** -// * Count of publisher tries. -// */ -// @Value("${app.retries}") -// @NotNull(message = "Producer properties are invalid") -// private String retries; -//} +package com.github.asavershin.worker.config; + +import jakarta.validation.constraints.NotNull; +import lombok.Data; +import lombok.NoArgsConstructor; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.stereotype.Component; + +@Data +@Component +@NoArgsConstructor +public class ProducerProp { + /** + * Count of publisher tries. + */ + @Value("${app.retries}") + @NotNull(message = "Producer properties are invalid") + private String retries; +} diff --git a/worker/src/main/java/com/github/asavershin/worker/config/RedisConf.java b/worker/src/main/java/com/github/asavershin/worker/config/RedisConf.java index 681ba9f..e8cc16a 100644 --- a/worker/src/main/java/com/github/asavershin/worker/config/RedisConf.java +++ b/worker/src/main/java/com/github/asavershin/worker/config/RedisConf.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images.config; +package com.github.asavershin.worker.config; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; diff --git a/worker/src/main/java/com/github/asavershin/worker/dto/DoneTask.java b/worker/src/main/java/com/github/asavershin/worker/dto/DoneTask.java new file mode 100644 index 0000000..0086b65 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/dto/DoneTask.java @@ -0,0 +1,13 @@ +package com.github.asavershin.worker.dto; + +import lombok.AllArgsConstructor; +import lombok.Data; +import lombok.NoArgsConstructor; + +@Data +@AllArgsConstructor +@NoArgsConstructor +public class DoneTask { + private String imageId; + private String requestId; +} diff --git a/worker/src/main/java/com/github/asavershin/worker/dto/Task.java b/worker/src/main/java/com/github/asavershin/worker/dto/Task.java index cfe7217..1c210f8 100644 --- a/worker/src/main/java/com/github/asavershin/worker/dto/Task.java +++ b/worker/src/main/java/com/github/asavershin/worker/dto/Task.java @@ -1,4 +1,4 @@ -package com.github.asavershin.worker; +package com.github.asavershin.worker.dto; import lombok.AllArgsConstructor; import lombok.Data; diff --git a/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java b/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java index 7fbf9ff..450d1c6 100644 --- a/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java +++ b/worker/src/main/java/com/github/asavershin/worker/in/Consumer.java @@ -1,77 +1,61 @@ -//package com.github.asavershin.images.in; -// -//import com.github.asavershin.images.DoneTask; -//import com.github.asavershin.images.Task; -//import com.github.asavershin.images.WorkerManager; -//import com.github.asavershin.images.out.CacheRepository; -//import com.github.asavershin.images.out.DoneProducer; -//import com.github.asavershin.images.out.WipProducer; -//import lombok.RequiredArgsConstructor; -//import lombok.extern.slf4j.Slf4j; -//import org.springframework.kafka.annotation.KafkaListener; -//import org.springframework.messaging.handler.annotation.Payload; -//import org.springframework.stereotype.Component; -//import org.springframework.transaction.annotation.Transactional; -// -//import java.util.List; -//import java.util.concurrent.ExecutorService; -//import java.util.concurrent.Executors; -// -//@Component -//@RequiredArgsConstructor -//@Slf4j -//public class Consumer { -// private final WorkerManager manager; -// private final WipProducer wipProducer; -// private final DoneProducer doneProducer; -// private final CacheRepository cacheRepository; -// -// private final ExecutorService executorService = Executors.newFixedThreadPool(5); -// -// @KafkaListener( -// topics = "${app.wiptopic}", -// groupId = "${app.group-id}", -// concurrency = "${app.replicas}", -// containerFactory = "kafkaListenerContainerFactory" -// ) -// @Transactional -// public void consume( -// final @Payload List records -// ) { -// log.info("Consumed: {}", records.size()); -// for (var record : records) { -// executorService.execute(() -> processRecord(record)); -// } -// } -// -// private void processRecord( -// final Task record -// ) { -// try { -// log.info("Received message: {}", record.toString()); -// var task = manager.start(record); -// if (task == null) { -// return; -// } -// if (task.getFilters().isEmpty()) { -// doneProducer.produce(new DoneTask(task.getImageId(), task.getRequestId())); -// } else { -// wipProducer.produce(task); -// } -// } catch (Exception ex) { -// log.info(ex.getMessage()); -// cacheRepository.deleteCache( -// record.getRequestId() + record.getImageId() -// ); -// wipProducer.produce(record); -// } -// } -//} -// -////{ -//// "imageId": "example", -//// "requestId": "987654", -//// "filters": [ -//// "ROTATE" -//// ] -////} +package com.github.asavershin.worker.in; + +import com.github.asavershin.worker.InvalidWorker; +import com.github.asavershin.worker.dto.DoneTask; +import com.github.asavershin.worker.dto.Task; +import com.github.asavershin.worker.services.WorkerManager; +import com.github.asavershin.worker.out.CacheRepository; +import com.github.asavershin.worker.out.producers.DoneProducer; +import com.github.asavershin.worker.out.producers.WipProducer; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.kafka.annotation.KafkaListener; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.messaging.handler.annotation.Payload; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + + +@Component +@RequiredArgsConstructor +@Slf4j +public class Consumer { + private final WorkerManager manager; + private final WipProducer wipProducer; + private final DoneProducer doneProducer; + private final CacheRepository cacheRepository; + + @KafkaListener( + topics = "${app.wiptopic}", + groupId = "${app.group-id}", + concurrency = "${app.replicas}", + containerFactory = "kafkaListenerContainerFactory" + ) + @Transactional + public void consume( + final @Payload Task record, + final Acknowledgment acknowledgment + ) { + log.info( + "Received message: {}", + record.toString() + ); + Task task; + try { + task = manager.start(record); + } catch (final InvalidWorker e) { + log.info(e.getMessage()); + return; + } + if (task == null) { + acknowledgment.acknowledge(); + return; + } + if (task.getFilters().isEmpty()) { + doneProducer.produce(new DoneTask(task.getImageId(), task.getRequestId())); + } else { + wipProducer.produce(task); + } + acknowledgment.acknowledge(); + } +} diff --git a/worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java b/worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java index 5f6a850..678057d 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/CacheRepository.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images.out; +package com.github.asavershin.worker.out; public interface CacheRepository { /** diff --git a/worker/src/main/java/com/github/asavershin/worker/out/FileException.java b/worker/src/main/java/com/github/asavershin/worker/out/FileException.java index 587ad7d..c6dcefc 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/FileException.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/FileException.java @@ -1,4 +1,4 @@ -package com.github.asavershin.images.out; +package com.github.asavershin.worker.out; public class FileException extends RuntimeException { /** diff --git a/worker/src/main/java/com/github/asavershin/worker/out/MinioService.java b/worker/src/main/java/com/github/asavershin/worker/out/MinioService.java index f649fa8..dba9612 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/MinioService.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/MinioService.java @@ -1,6 +1,4 @@ -package com.github.asavershin.images.out; - -import org.springframework.web.multipart.MultipartFile; +package com.github.asavershin.worker.out; import java.io.InputStream; diff --git a/worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java b/worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java index 89b15c0..441ceef 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/impl/CacheRepositoryIml.java @@ -1,5 +1,6 @@ -package com.github.asavershin.worker.out; +package com.github.asavershin.worker.out.impl; +import com.github.asavershin.worker.out.CacheRepository; import lombok.RequiredArgsConstructor; import org.springframework.data.redis.core.RedisTemplate; import org.springframework.stereotype.Repository; diff --git a/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java b/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java index 9f01e52..871ee77 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java @@ -1,7 +1,9 @@ -package com.github.asavershin.worker.out; +package com.github.asavershin.worker.out.impl; import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.out.FileException; +import com.github.asavershin.worker.out.MinioService; import io.minio.BucketExistsArgs; import io.minio.GetObjectArgs; import io.minio.MakeBucketArgs; @@ -89,7 +91,7 @@ public InputStream getFile(final String link) { .build()); return obj; } catch (Exception e) { - throw new FileException("File download failed: " + e.getMessage()); + throw new FileException("File " + link + " download failed: " + e.getMessage()); } } diff --git a/worker/src/main/java/com/github/asavershin/worker/out/producers/DoneProducer.java b/worker/src/main/java/com/github/asavershin/worker/out/producers/DoneProducer.java new file mode 100644 index 0000000..b687f6c --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/out/producers/DoneProducer.java @@ -0,0 +1,33 @@ +package com.github.asavershin.worker.out.producers; + +import com.github.asavershin.worker.dto.DoneTask; +import jakarta.validation.constraints.NotNull; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.stereotype.Component; +import org.springframework.transaction.annotation.Transactional; + +import static com.github.asavershin.worker.config.KafkaConf.DONE_PRODUCER; + +@Component +public class DoneProducer { + private final KafkaTemplate producer; + @Value("${app.donetopic}") + @NotNull(message = "Topic for KafkaProducer is null") + private String topic; + public DoneProducer( + final @Qualifier(DONE_PRODUCER) + KafkaTemplate allAcksKafkaTemplate + ) { + producer = allAcksKafkaTemplate; + } + + @Transactional + public void produce(final DoneTask event) { + producer.send( + topic, + event + ); + } +} diff --git a/worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java b/worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java index 7c747a2..4a30355 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/producers/WipProducer.java @@ -1,6 +1,6 @@ -package com.github.asavershin.worker.out; +package com.github.asavershin.worker.out.producers; -import com.github.asavershin.worker.Task; +import com.github.asavershin.worker.dto.Task; import jakarta.validation.constraints.NotNull; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; diff --git a/worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java b/worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java index 6cc2e0f..265f0fd 100644 --- a/worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/BlackWhiteWorker.java @@ -1,4 +1,4 @@ -package com.github.asavershin.worker; +package com.github.asavershin.worker.services; import com.github.asavershin.worker.config.MinIOProperties; import com.github.asavershin.worker.out.MinioService; @@ -17,7 +17,7 @@ @Profile("blackwhite") @RequiredArgsConstructor @Slf4j -public class BlackWhiteWorker implements Worker{ +public class BlackWhiteWorker implements Worker { private final MinioService imageStorage; private final MinIOProperties props; diff --git a/worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java b/worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java index 8cddc62..7977c33 100644 --- a/worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/RotateImage.java @@ -1,4 +1,4 @@ -package com.github.asavershin.worker; +package com.github.asavershin.worker.services; import com.github.asavershin.worker.config.MinIOProperties; import com.github.asavershin.worker.out.MinioService; @@ -18,7 +18,7 @@ @Profile("rotate") @RequiredArgsConstructor @Slf4j -public class RotateImage implements Worker{ +public class RotateImage implements Worker { private final MinioService imageStorage; private final MinIOProperties props; diff --git a/worker/src/main/java/com/github/asavershin/worker/services/Worker.java b/worker/src/main/java/com/github/asavershin/worker/services/Worker.java index 48467db..ef069a6 100644 --- a/worker/src/main/java/com/github/asavershin/worker/services/Worker.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/Worker.java @@ -1,4 +1,4 @@ -package com.github.asavershin.worker; +package com.github.asavershin.worker.services; public interface Worker { String doWork(String imageId, boolean lastWorker); diff --git a/worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java index bc884ca..1eea1fa 100644 --- a/worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManager.java @@ -1,4 +1,6 @@ -package com.github.asavershin.worker; +package com.github.asavershin.worker.services; + +import com.github.asavershin.worker.dto.Task; public interface WorkerManager { Task start(Task filters); diff --git a/worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java index d27a260..b9f3757 100644 --- a/worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java +++ b/worker/src/main/java/com/github/asavershin/worker/services/WorkerManagerImpl.java @@ -1,5 +1,7 @@ -package com.github.asavershin.worker; +package com.github.asavershin.worker.services; +import com.github.asavershin.worker.InvalidWorker; +import com.github.asavershin.worker.dto.Task; import com.github.asavershin.worker.out.CacheRepository; import jakarta.validation.constraints.NotNull; import lombok.RequiredArgsConstructor; @@ -22,7 +24,7 @@ public Task start(final Task task) { if (!Objects.equals( task.getFilters().get(0), worker.whoAmI())) { - throw new RuntimeException("Invalid worker"); + throw new InvalidWorker("Invalid worker"); } var cached = cache.getCache( task.getRequestId() + task.getImageId()); @@ -39,6 +41,7 @@ public Task start(final Task task) { var imageId = worker.doWork(task.getImageId(), task.getFilters().size() == 1 ); + task.setImageId(imageId); task.getFilters().remove(0); return task; } diff --git a/worker/src/main/resources/application-blackwhite.yml b/worker/src/main/resources/application-blackwhite.yml index 0032eba..816310c 100644 --- a/worker/src/main/resources/application-blackwhite.yml +++ b/worker/src/main/resources/application-blackwhite.yml @@ -1,6 +1,31 @@ server: port: 8082 +spring: + servlet: + multipart: + max-file-size: 10MB + config: + import: optional:file:.env[.properties] + data: + redis: + host: ${REDIS_HOST_BLACKWHITE} + port: ${REDIS_PORT} + password: ${REDIS_PASSWORD} + kafka: + listener.ack-mode: manual + bootstrap-servers: + - kafka-1:9092 + - kafka-2:9093 + - kafka-3:9094 + properties: + sasl: + jaas: + config: org.apache.kafka.common.security.plain.PlainLoginModule required username=${kafka_username:'kafka'} password=${kafka_password:'password123'}; + mechanism: PLAIN + security: + protocol: SASL_PLAINTEXT + app: wiptopic: ${TOPIC:images.wip} retries: ${RETRIES:3} diff --git a/worker/src/main/resources/application-rotate.yml b/worker/src/main/resources/application-rotate.yml index 68ced85..50c8bf3 100644 --- a/worker/src/main/resources/application-rotate.yml +++ b/worker/src/main/resources/application-rotate.yml @@ -1,6 +1,31 @@ server: port: 8083 +spring: + servlet: + multipart: + max-file-size: 10MB + config: + import: optional:file:.env[.properties] + data: + redis: + host: ${REDIS_HOST_ROTATE} + port: ${REDIS_PORT} + password: ${REDIS_PASSWORD} + kafka: + listener.ack-mode: manual + bootstrap-servers: + - kafka-1:9092 + - kafka-2:9093 + - kafka-3:9094 + properties: + sasl: + jaas: + config: org.apache.kafka.common.security.plain.PlainLoginModule required username=${kafka_username:'kafka'} password=${kafka_password:'password123'}; + mechanism: PLAIN + security: + protocol: SASL_PLAINTEXT + app: wiptopic: ${TOPIC:images.wip} retries: ${RETRIES:3} diff --git a/worker/src/main/resources/application.yml b/worker/src/main/resources/application.yml index 9a39d88..2b832a8 100644 --- a/worker/src/main/resources/application.yml +++ b/worker/src/main/resources/application.yml @@ -1,28 +1,3 @@ -spring: - servlet: - multipart: - max-file-size: 10MB - config: - import: optional:file:.env[.properties] - data: - redis: - host: ${REDIS_HOST} - port: ${REDIS_PORT} - password: ${REDIS_PASSWORD} - kafka: - listener.ack-mode: manual - bootstrap-servers: - - kafka-1:9092 - - kafka-2:9093 - - kafka-3:9094 - properties: - sasl: - jaas: - config: org.apache.kafka.common.security.plain.PlainLoginModule required username=${kafka_username:'kafka'} password=${kafka_password:'password123'}; - mechanism: PLAIN - security: - protocol: SASL_PLAINTEXT - minio: url: ${MINIO_URL} user: ${MINIO_ROOT_USER} From 6580a8874cb40a3b0f3862a8c6348aea0d5d64be Mon Sep 17 00:00:00 2001 From: asavershin Date: Fri, 31 May 2024 19:32:23 +0300 Subject: [PATCH 5/7] Add new filter and rate limiting --- api/pom.xml | 13 +++++ .../asavershin/api/config/Bucket4jConfig.java | 43 +++++++++++++++ .../asavershin/api/domain/filter/Filter.java | 6 ++- .../in/controllers/ImageController.java | 53 +++++++++++++++++-- 4 files changed, 111 insertions(+), 4 deletions(-) create mode 100644 api/src/main/java/com/github/asavershin/api/config/Bucket4jConfig.java diff --git a/api/pom.xml b/api/pom.xml index 2d73337..d690a37 100644 --- a/api/pom.xml +++ b/api/pom.xml @@ -27,6 +27,19 @@ + + + com.bucket4j + bucket4j-core + 8.10.1 + + + + com.bucket4j + bucket4j-redis + 8.10.1 + + org.springframework.boot spring-boot-starter-web diff --git a/api/src/main/java/com/github/asavershin/api/config/Bucket4jConfig.java b/api/src/main/java/com/github/asavershin/api/config/Bucket4jConfig.java new file mode 100644 index 0000000..36ef49a --- /dev/null +++ b/api/src/main/java/com/github/asavershin/api/config/Bucket4jConfig.java @@ -0,0 +1,43 @@ +package com.github.asavershin.api.config; + +import io.github.bucket4j.Bandwidth; +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy; +import io.github.bucket4j.distributed.proxy.ClientSideConfig; +import io.github.bucket4j.distributed.proxy.ProxyManager; +import io.github.bucket4j.redis.lettuce.cas.LettuceBasedProxyManager; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.codec.RedisCodec; +import io.lettuce.core.codec.StringCodec; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.UserDetails; + +import java.time.Duration; +import java.util.function.Supplier; + +@Configuration +public class Bucket4jConfig { + @Bean + public RedisClient redisClient(final RedisProperties redisProperties) { + return RedisClient.create( + RedisURI.Builder.redis(redisProperties.getHost(), redisProperties.getPort()) + .withPassword(redisProperties.getPassword().toCharArray()).build()); + } + + @Bean + public ProxyManager lettuceBasedProxyManager(final RedisClient redisClient) { + StatefulRedisConnection redisConnection = redisClient + .connect(RedisCodec.of(StringCodec.UTF8, ByteArrayCodec.INSTANCE)); + + return LettuceBasedProxyManager + .builderFor(redisConnection) + .build(); + } +} diff --git a/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java b/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java index b9c760f..494a20e 100644 --- a/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java +++ b/api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java @@ -15,7 +15,11 @@ public enum Filter { /** * Represents the CROP filter. */ - BLACKWHITE; + BLACKWHITE, + /** + * Filter for external api imagga. + */ + IMAGGA; /** * Converts a string representation of a filter name string diff --git a/api/src/main/java/com/github/asavershin/api/infrastructure/in/controllers/ImageController.java b/api/src/main/java/com/github/asavershin/api/infrastructure/in/controllers/ImageController.java index ef47436..2235721 100644 --- a/api/src/main/java/com/github/asavershin/api/infrastructure/in/controllers/ImageController.java +++ b/api/src/main/java/com/github/asavershin/api/infrastructure/in/controllers/ImageController.java @@ -15,10 +15,17 @@ import com.github.asavershin.api.infrastructure.in.controllers.dto.image.UploadImageResponse; import com.github.asavershin.api.infrastructure.in.security.CustomUserDetails; import com.github.asavershin.api.infrastructure.out.producers.KafkaProducer; +import io.github.bucket4j.Bandwidth; +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.distributed.proxy.ProxyManager; import io.swagger.v3.oas.annotations.Operation; import io.swagger.v3.oas.annotations.tags.Tag; import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Bean; import org.springframework.security.core.annotation.AuthenticationPrincipal; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.web.bind.annotation.DeleteMapping; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; @@ -29,13 +36,16 @@ import org.springframework.web.bind.annotation.RestController; import org.springframework.web.multipart.MultipartFile; +import java.time.Duration; import java.util.List; import java.util.UUID; +import java.util.function.Supplier; @RestController @RequestMapping("/api/v1/image") @Tag(name = "image", description = "Работа с изображениями") @RequiredArgsConstructor +@Slf4j public class ImageController { /** * Domain query that allows you take images of specific user. @@ -54,6 +64,10 @@ public class ImageController { * Service for tracking the status of image processing events. */ private final GetStatusEvent statusEvent; + /** + * Using for limiting requests to imagga filter. + */ + private final ProxyManager pm; /** * Not final to allows Spring use proxy. @@ -144,11 +158,12 @@ public byte[] downloadImage( user.authenticatedUser().userId() ); } + /** * This start image processing event by authenticated user to his image. * - * @param user Is param that injects by spring and contains - * current authenticated spring user. + * @param user Is param that injects by spring and contains + * current authenticated spring user. * @param imageId The ID of the image that will be processed. * @param filters The list of filters that will be applied to the image. * @return request id @@ -165,7 +180,24 @@ public ApplyImageFiltersResponse applyImageFilters( .toList(), ImageId.fromString(imageId) ); - producer.send(event, user.authenticatedUser().userId()); + if (event.filters().contains(Filter.IMAGGA)) { + var b = getBucket(user); + var tokens = b.getAvailableTokens(); + log.info("Available tokens {} for user {} ", + tokens, + user.getUsername() + ); + if (tokens > 0) { + producer.send(event, user.authenticatedUser().userId()); + b.tryConsume(1); + } else { + throw new RuntimeException( + "You have reached your limit on \"imagga\"" + ); + } + } else { + producer.send(event, user.authenticatedUser().userId()); + } return new ApplyImageFiltersResponse( event.requestId().value().toString() ); @@ -198,4 +230,19 @@ public ApplyImageFiltersResponse applyImageFilters( ) ); } + + private Bucket getBucket(final UserDetails userDetails) { + var username = userDetails.getUsername(); + return pm.builder().build( + "bucket: " + username, + () -> BucketConfiguration.builder() + .addLimit(Bandwidth.builder().capacity(1L) + .refillIntervally(1, + Duration.ofDays(1L) + ) + .build() + ) + .build() + ); + } } From 581acf78dd09a9b578115298083232f32d11a764 Mon Sep 17 00:00:00 2001 From: asavershin Date: Fri, 31 May 2024 19:34:55 +0300 Subject: [PATCH 6/7] Add new worker | retry | rate limiting --- worker/.env | 1 + worker/Dockerfile-imagga | 7 + worker/docker-compose-imagga.yml | 52 +++++ worker/pom.xml | 19 ++ .../worker/BadImaggaCodeException.java | 7 + .../worker/config/ImaggaConfig.java | 68 ++++++ .../worker/config/ImaggaProperties.java | 16 ++ .../asavershin/worker/dto/ImagaStatus.java | 8 + .../worker/dto/ImaggaTagsResponse.java | 32 +++ .../worker/dto/ImaggaUploadResponse.java | 15 ++ .../worker/out/impl/MinioServiceIml.java | 2 +- .../worker/services/ImaggaWorker.java | 211 ++++++++++++++++++ .../src/main/resources/application-imagga.yml | 39 ++++ .../integrations/ImaggaIntegrationTest.java | 80 +++++++ .../worker/integrations/ImaggaTest.java | 15 ++ 15 files changed, 571 insertions(+), 1 deletion(-) create mode 100644 worker/Dockerfile-imagga create mode 100644 worker/docker-compose-imagga.yml create mode 100644 worker/src/main/java/com/github/asavershin/worker/BadImaggaCodeException.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/config/ImaggaConfig.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/config/ImaggaProperties.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/dto/ImagaStatus.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/dto/ImaggaTagsResponse.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/dto/ImaggaUploadResponse.java create mode 100644 worker/src/main/java/com/github/asavershin/worker/services/ImaggaWorker.java create mode 100644 worker/src/main/resources/application-imagga.yml create mode 100644 worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaIntegrationTest.java create mode 100644 worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaTest.java diff --git a/worker/.env b/worker/.env index ebf2e6c..f0498d7 100644 --- a/worker/.env +++ b/worker/.env @@ -11,6 +11,7 @@ MINIO_CONSOLE_PORT=9090 MINIO_PORT=9000 # REDIS +REDIS_HOST_IMAGGA=redis-imagga REDIS_HOST_ROTATE=redis-rotate REDIS_HOST_BLACKWHITE=redis-blackwhite REDIS_PORT=6379 diff --git a/worker/Dockerfile-imagga b/worker/Dockerfile-imagga new file mode 100644 index 0000000..9439abc --- /dev/null +++ b/worker/Dockerfile-imagga @@ -0,0 +1,7 @@ +FROM openjdk:21 + +WORKDIR /app + +COPY target/*.jar app.jar + +CMD ["java", "-jar", "-Dspring.profiles.active=imagga", "app.jar"] \ No newline at end of file diff --git a/worker/docker-compose-imagga.yml b/worker/docker-compose-imagga.yml new file mode 100644 index 0000000..942061b --- /dev/null +++ b/worker/docker-compose-imagga.yml @@ -0,0 +1,52 @@ +networks: + kafka-net: + name: kafka-net + driver: bridge + api-net: + name: api-net + driver: bridge + +services: + + backend-worker-imagga: + container_name: backend-worker-imagga + networks: + - kafka-net + - api-net + build: + context: . + dockerfile: Dockerfile-imagga + ports: + - 8084:8081 + env_file: + - .env + + redis-imagga: + networks: + - api-net + image: redis:7.2-rc-alpine + restart: always + container_name: redis-imagga + ports: + - '6381:6379' + command: redis-server --save 20 1 --loglevel debug --requirepass ${REDIS_PASSWORD} + volumes: + - redis-rotate-data:/data + +# minio-api: +# networks: +# - api-net +# image: minio/minio:RELEASE.2024-02-14T21-36-02Z +# container_name: minio-api +# env_file: +# - .env +# command: server ~/minio --console-address :9090 +# ports: +# - '9090:9090' +# - '9000:9000' +# volumes: +# - minio-api-data:/minio + +volumes: +# minio-api-data: + redis-rotate-data: diff --git a/worker/pom.xml b/worker/pom.xml index 89f6ddf..16c0929 100644 --- a/worker/pom.xml +++ b/worker/pom.xml @@ -20,6 +20,25 @@ + + org.springframework.retry + spring-retry + + + org.springframework + spring-aspects + + + com.bucket4j + bucket4j-core + 8.10.1 + + + + com.bucket4j + bucket4j-redis + 8.10.1 + org.springframework.boot spring-boot-starter-web diff --git a/worker/src/main/java/com/github/asavershin/worker/BadImaggaCodeException.java b/worker/src/main/java/com/github/asavershin/worker/BadImaggaCodeException.java new file mode 100644 index 0000000..90a5ef2 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/BadImaggaCodeException.java @@ -0,0 +1,7 @@ +package com.github.asavershin.worker; + +public class BadImaggaCodeException extends RuntimeException { + public BadImaggaCodeException(final String s) { + super(s); + } +} diff --git a/worker/src/main/java/com/github/asavershin/worker/config/ImaggaConfig.java b/worker/src/main/java/com/github/asavershin/worker/config/ImaggaConfig.java new file mode 100644 index 0000000..5dc05d7 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/config/ImaggaConfig.java @@ -0,0 +1,68 @@ +package com.github.asavershin.worker.config; + +import io.github.bucket4j.Bandwidth; +import io.github.bucket4j.Bucket; +import io.github.bucket4j.BucketConfiguration; +import io.github.bucket4j.distributed.ExpirationAfterWriteStrategy; +import io.github.bucket4j.distributed.proxy.ClientSideConfig; +import io.github.bucket4j.distributed.proxy.ProxyManager; +import io.github.bucket4j.redis.lettuce.cas.LettuceBasedProxyManager; +import io.lettuce.core.RedisClient; +import io.lettuce.core.RedisURI; +import io.lettuce.core.api.StatefulRedisConnection; +import io.lettuce.core.codec.ByteArrayCodec; +import io.lettuce.core.codec.RedisCodec; +import io.lettuce.core.codec.StringCodec; +import org.springframework.boot.autoconfigure.data.redis.RedisProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.Profile; +import org.springframework.retry.annotation.EnableRetry; +import org.springframework.web.client.RestTemplate; + +import java.time.Duration; + +@Configuration +@EnableRetry +@Profile("imagga") +public class ImaggaConfig { + @Bean + public RestTemplate restTemplate() { + return new RestTemplate(); + } + + @Bean + public RedisClient redisClient(final RedisProperties redisProperties) { + return RedisClient.create( + RedisURI.Builder.redis(redisProperties.getHost(), redisProperties.getPort()) + .withPassword(redisProperties.getPassword().toCharArray()).build()); + } + + @Bean + public ProxyManager lettuceBasedProxyManager(final RedisClient redisClient) { + StatefulRedisConnection redisConnection = redisClient + .connect(RedisCodec.of(StringCodec.UTF8, ByteArrayCodec.INSTANCE)); + + return LettuceBasedProxyManager + .builderFor(redisConnection) + .build(); + } + + @Bean + public Bucket bucketConfiguration(final ProxyManager proxyManager) { + return proxyManager.builder().build( + "imagga", + () -> BucketConfiguration.builder() + .addLimit( + Bandwidth.builder() + .capacity(33) + .refillIntervally( + 33, + Duration.ofDays(1L) + ) + .build() + ) + .build() + ); + } +} diff --git a/worker/src/main/java/com/github/asavershin/worker/config/ImaggaProperties.java b/worker/src/main/java/com/github/asavershin/worker/config/ImaggaProperties.java new file mode 100644 index 0000000..8935878 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/config/ImaggaProperties.java @@ -0,0 +1,16 @@ +package com.github.asavershin.worker.config; + +import lombok.Getter; +import lombok.Setter; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.context.annotation.Configuration; + +@Configuration +@ConfigurationProperties(prefix = "imagga") +@Getter +@Setter +public class ImaggaProperties { + private String key; + private String secret; + private String ttlprefix; +} diff --git a/worker/src/main/java/com/github/asavershin/worker/dto/ImagaStatus.java b/worker/src/main/java/com/github/asavershin/worker/dto/ImagaStatus.java new file mode 100644 index 0000000..0ac09f6 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/dto/ImagaStatus.java @@ -0,0 +1,8 @@ +package com.github.asavershin.worker.dto; + +import lombok.Data; +@Data +public class ImagaStatus { + private String text; + private String type; +} diff --git a/worker/src/main/java/com/github/asavershin/worker/dto/ImaggaTagsResponse.java b/worker/src/main/java/com/github/asavershin/worker/dto/ImaggaTagsResponse.java new file mode 100644 index 0000000..34c687d --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/dto/ImaggaTagsResponse.java @@ -0,0 +1,32 @@ +package com.github.asavershin.worker.dto; + +import lombok.Data; +import org.springframework.context.annotation.Profile; + +import java.util.List; + +@Data +@Profile("imagga") +public class ImaggaTagsResponse { + private Result result; + + @Data + public static class Result { + private List tags; + } + + @Data + public static class Tag { + private double confidence; + private TagInfo tag; + } + + @Data + public static class TagInfo { + private String en; + } + + private ImagaStatus status; + +} + diff --git a/worker/src/main/java/com/github/asavershin/worker/dto/ImaggaUploadResponse.java b/worker/src/main/java/com/github/asavershin/worker/dto/ImaggaUploadResponse.java new file mode 100644 index 0000000..cd9aa91 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/dto/ImaggaUploadResponse.java @@ -0,0 +1,15 @@ +package com.github.asavershin.worker.dto; + +import com.fasterxml.jackson.annotation.JsonProperty; +import lombok.Data; + +@Data +public class ImaggaUploadResponse { + private Result result; + private ImagaStatus status; + @Data + public static class Result { + @JsonProperty("upload_id") + private String uploadId; + } +} diff --git a/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java b/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java index 871ee77..d3780b5 100644 --- a/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java +++ b/worker/src/main/java/com/github/asavershin/worker/out/impl/MinioServiceIml.java @@ -57,7 +57,7 @@ public MinioServiceIml(final MinioClient aMinioClient, * Not final to allow spring use proxy. */ @Override - public void saveFile(byte[] image, final String filename) { + public void saveFile(final byte[] image, final String filename) { if (!bucketExists(minioProperties.getBucket())) { throw new FileException( diff --git a/worker/src/main/java/com/github/asavershin/worker/services/ImaggaWorker.java b/worker/src/main/java/com/github/asavershin/worker/services/ImaggaWorker.java new file mode 100644 index 0000000..078e8b1 --- /dev/null +++ b/worker/src/main/java/com/github/asavershin/worker/services/ImaggaWorker.java @@ -0,0 +1,211 @@ +package com.github.asavershin.worker.services; + +import com.github.asavershin.worker.BadImaggaCodeException; +import com.github.asavershin.worker.config.ImaggaProperties; +import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.dto.ImaggaTagsResponse; +import com.github.asavershin.worker.dto.ImaggaUploadResponse; +import com.github.asavershin.worker.out.MinioService; +import io.github.bucket4j.Bucket; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import org.springframework.context.annotation.Profile; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpHeaders; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; +import org.springframework.retry.annotation.Backoff; +import org.springframework.retry.annotation.Recover; +import org.springframework.retry.annotation.Retryable; +import org.springframework.stereotype.Service; +import org.springframework.util.LinkedMultiValueMap; +import org.springframework.util.MultiValueMap; +import org.springframework.web.client.HttpStatusCodeException; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; + +import javax.imageio.ImageIO; +import java.awt.*; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.List; +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +@Service +@Profile("imagga") +@RequiredArgsConstructor +@Slf4j +public class ImaggaWorker implements Worker { + private final MinioService imageStorage; + private final MinIOProperties minioProps; + private final ImaggaProperties imaggaProps; + private final RestTemplate restTemplate; + private static String ENDPOINT_UPLOAD = "https://api.imagga.com/v2/uploads"; + private static final String ENDPOINT_TAGS = "https://api.imagga.com/v2/tags"; + private final Bucket bucket; + + + @Override + @Retryable( + retryFor = {BadImaggaCodeException.class}, + maxAttempts = 3, + backoff = @Backoff(delay = 1000) + ) + public String doWork(final String imageUUID, final boolean lastWorker) { + if (!bucket.tryConsume(1)) { + throw new RuntimeException("Rate limit exceeded"); + } + log.info("Start Work"); + InputStream is = imageStorage.getFile(imageUUID); + log.info("End get file"); + var uploadId = requestToImagga(is); + ImaggaTagsResponse tagsResponse = getTags(uploadId.orElseThrow()).orElseThrow(); + log.info("Start sending to minio"); + // Add tags to image + byte[] modifiedImage = addTagsToImage(imageStorage.getFile(imageUUID), tagsResponse.getResult().getTags()); + + // Generate new UUID for the modified image + String id = UUID.randomUUID().toString(); + if (!lastWorker) { + id = minioProps.getTtlprefix() + id; + } + // Save the modified image to MinIO + imageStorage.saveFile(modifiedImage, id); + log.info("End imagga worker"); + return id; + } + @Recover + public String recoverAfterRetryLimitReached( + final BadImaggaCodeException ex, + final String imageUUID, + final boolean lastWorker + ){ + log.error("Imagga api is not responding"); + return imageUUID; + } + + @Override + public String whoAmI() { + return "ROTATE"; + } + + private Optional requestToImagga(final InputStream is) { + log.info("start send request"); + MultiValueMap body = new LinkedMultiValueMap<>(); + log.info("Add body"); + ByteArrayResource byteArrayResource; + try { + byteArrayResource = new ByteArrayResource(toByteArray(is)) { + @Override + public String getFilename() { + return "upload.jpg"; // You can set the filename if needed + } + }; + } catch (IOException e) { + log.error("Error reading InputStream", e); + return Optional.empty(); + } + + body.add("image", byteArrayResource); + log.info("Add headers"); + + + HttpHeaders headers = new HttpHeaders(); + headers.setBasicAuth(imaggaProps.getKey(), imaggaProps.getSecret()); + headers.setContentType(MediaType.MULTIPART_FORM_DATA); + log.info("create req entity"); + HttpEntity> requestEntity = new HttpEntity<>(body, headers); + log.info("Send to imagga"); + ResponseEntity response = restTemplate.exchange(ENDPOINT_UPLOAD, HttpMethod.POST, requestEntity, new ParameterizedTypeReference<>() { + }); + log.info("end send request"); + if (!checkResponseCode(response)) { + return Optional.empty(); + } + return Optional.of(Objects.requireNonNull(response.getBody()).getResult().getUploadId()); + } + + private byte[] toByteArray(final InputStream is) throws IOException { + ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + int nRead; + byte[] data = new byte[1024]; + while ((nRead = is.read(data, 0, data.length)) != -1) { + buffer.write(data, 0, nRead); + } + buffer.flush(); + return buffer.toByteArray(); + } + + private Optional getTags(final String uploadId) { + log.info("Start getTags"); + + HttpHeaders headers = new HttpHeaders(); + headers.setBasicAuth(imaggaProps.getKey(), imaggaProps.getSecret()); + HttpEntity requestEntity = new HttpEntity<>(headers); + + var builder = UriComponentsBuilder.fromHttpUrl(ENDPOINT_TAGS) + .queryParam("image_upload_id", uploadId) + .queryParam("limit", 3); + var url = builder.build().encode().toUriString(); + + log.info("Sending GET request to: {}", url); + ResponseEntity response = restTemplate.exchange(url, HttpMethod.GET, requestEntity, new ParameterizedTypeReference<>() { + }); + + if (checkResponseCode(response)) { + log.info("Get tags successful"); + return Optional.ofNullable(response.getBody()); + } else { + log.error("Get tags failed with status code: " + response.getStatusCode()); + return Optional.empty(); + } + } + + private byte[] addTagsToImage( + final InputStream is, + final List tags + ) { + // Load image with ImageJ + try { + var image = ImageIO.read(is); + var font = new Font("Arial", Font.BOLD, 50); + // Add tags to the image + int y = 0; // Starting Y position for text + Graphics g = image.getGraphics(); + g.setFont(font); + g.setColor(Color.GREEN); + for (ImaggaTagsResponse.Tag tag : tags) { + g.drawString(tag.getTag().getEn(), 250, y); + y += 50; // Move to next line + } + // Convert ImagePlus to byte array + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + ImageIO.write(image, "jpg", baos); + g.dispose(); + return baos.toByteArray(); + } catch (IOException e) { + throw new RuntimeException(e.getMessage()); + } + } + + private boolean checkResponseCode(final ResponseEntity response) { + if (response.getStatusCode() == HttpStatus.OK) { + log.info("Upload successful"); + return true; + } else if (response.getStatusCode() == HttpStatus.TOO_MANY_REQUESTS + || response.getStatusCode().is5xxServerError()) { + log.error("Upload failed with status code: " + response.getStatusCode()); + throw new BadImaggaCodeException("Upload failed with status code: " + response.getStatusCode()); + } else { + log.error("Upload failed with status code: " + response.getStatusCode()); + return false; + } + } +} diff --git a/worker/src/main/resources/application-imagga.yml b/worker/src/main/resources/application-imagga.yml new file mode 100644 index 0000000..d1d7c04 --- /dev/null +++ b/worker/src/main/resources/application-imagga.yml @@ -0,0 +1,39 @@ +server: + port: 8082 + +spring: + servlet: + multipart: + max-file-size: 10MB + config: + import: optional:file:.env[.properties] + data: + redis: + host: ${REDIS_HOST_BLACKWHITE} + port: ${REDIS_PORT} + password: ${REDIS_PASSWORD} + kafka: + listener.ack-mode: manual + bootstrap-servers: + - kafka-1:9092 + - kafka-2:9093 + - kafka-3:9094 + properties: + sasl: + jaas: + config: org.apache.kafka.common.security.plain.PlainLoginModule required username=${kafka_username:'kafka'} password=${kafka_password:'password123'}; + mechanism: PLAIN + security: + protocol: SASL_PLAINTEXT + +app: + wiptopic: ${TOPIC:images.wip} + retries: ${RETRIES:3} + donetopic: ${TOPIC:images.done} + partitions: ${PARTITIONS:1} + replicas: ${REPLICAS:3} + group-id: ${GROUP_ID:blackwhite} + +imagga: + key: ${IMAGGA_KEY:acc_0a66897eb48ffb1} + secret: ${IMAGGA_SECRET:4b2c3cc9d36f0c91cd99da09d9781ef9} \ No newline at end of file diff --git a/worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaIntegrationTest.java b/worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaIntegrationTest.java new file mode 100644 index 0000000..47c8ed3 --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaIntegrationTest.java @@ -0,0 +1,80 @@ +package com.github.asavershin.worker.integrations; + +import com.github.asavershin.worker.config.MinIOProperties; +import com.github.asavershin.worker.out.MinioService; +import com.github.asavershin.worker.services.Worker; +import io.minio.ListObjectsArgs; +import io.minio.MinioClient; +import io.minio.RemoveObjectArgs; +import io.minio.Result; +import io.minio.messages.Item; +import lombok.extern.slf4j.Slf4j; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpMethod; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.test.web.client.match.MockRestRequestMatchers; +import org.springframework.test.web.client.response.MockRestResponseCreators; +import org.springframework.web.client.RestTemplate; + +import java.util.concurrent.atomic.AtomicInteger; + +import static com.github.asavershin.worker.ImageHelper.createTestImage; +import static org.junit.Assert.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Slf4j +public class ImaggaIntegrationTest extends ImaggaTest{ + @Autowired + private MinioService minioService; + @Autowired + private Worker worker; + @Autowired + private MinioClient minioClient; + @Autowired + private MinIOProperties minioProperties; + @Autowired + private RestTemplate restTemplate; + + @BeforeEach + public void clearDB() { + try { + Iterable> files = minioClient.listObjects(ListObjectsArgs.builder().bucket(minioProperties.getBucket()).build()); + + for (var file : files) { + minioClient.removeObject( + RemoveObjectArgs.builder() + .bucket(minioProperties.getBucket()) + .object(file.get().objectName()) + .build()); + } + } catch (Exception e) { + log.info("Clean bucket fail"); + e.printStackTrace(); + } + } + + @Test + public void testImaggaWorker() { + minioService.saveFile(createTestImage(2000, 2000), "example"); + MockRestServiceServer mockServer = MockRestServiceServer.createServer(restTemplate); + mockServer.expect(MockRestRequestMatchers.requestTo("https://api.imagga.com/v2/uploads")) + .andExpect(MockRestRequestMatchers.method(HttpMethod.POST)) + .andRespond(MockRestResponseCreators.withSuccess("{ \"result\": { \"upload_id\": \"testUploadId\" }, \"status\": { \"text\": \"Success\", \"type\": \"success\" } }", MediaType.APPLICATION_JSON)); + + // Мокируем запрос на получение тегов от Imagga + mockServer.expect(MockRestRequestMatchers.requestTo("https://api.imagga.com/v2/tags?image_upload_id=testUploadId&limit=3")) + .andExpect(MockRestRequestMatchers.method(HttpMethod.GET)) + .andRespond(MockRestResponseCreators.withSuccess("{ \"result\": { \"tags\": [ { \"confidence\": 0.99, \"tag\": { \"en\": \"testTag\" } } ] }, \"status\": { \"text\": \"Success\", \"type\": \"success\" } }", MediaType.APPLICATION_JSON)); + var id = worker.doWork("example", true); + + Assertions.assertNotNull(id); + var objectsInMinio = minioClient.listObjects(ListObjectsArgs.builder().bucket("files").build()); + AtomicInteger countObjectsInMinio = new AtomicInteger(); + objectsInMinio.forEach( it -> countObjectsInMinio.addAndGet(1)); + assertEquals(2, countObjectsInMinio.get()); + } +} diff --git a/worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaTest.java b/worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaTest.java new file mode 100644 index 0000000..076aaf6 --- /dev/null +++ b/worker/src/test/java/com/github/asavershin/worker/integrations/ImaggaTest.java @@ -0,0 +1,15 @@ +package com.github.asavershin.worker.integrations; + +import com.github.asavershin.worker.config.CacheRedisConfig; +import com.github.asavershin.worker.config.MinioConfig; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.kafka.test.context.EmbeddedKafka; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.ContextConfiguration; + +@SpringBootTest +@EmbeddedKafka +@ContextConfiguration(initializers = {MinioConfig.Initializer.class, CacheRedisConfig.Initializer.class}) +@ActiveProfiles("imagga") +public class ImaggaTest { +} From eb359488ebe1350cafa4d99ac97c8eb2adf78fa1 Mon Sep 17 00:00:00 2001 From: asavershin Date: Fri, 31 May 2024 19:35:29 +0300 Subject: [PATCH 7/7] Update README --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 5a2bd40..6cff16b 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ docker compose -f api/docker-compose.yml up --build -d docker compose -f worker/docker-compose-blackwhite.yml up --build -d docker compose -f worker/docker-compose-rotate.yml up --build -d +docker compose -f worker/docker-compose-imagga.yml up --build -d