Add workers

README.md

@@ -8,5 +8,8 @@ docker compose -f kafka/docker-compose.yml up --build -d
 
 docker compose -f api/docker-compose.yml up --build -d  
 
+docker compose -f worker/docker-compose-blackwhite.yml up --build -d
+docker compose -f worker/docker-compose-rotate.yml up --build -d
+
 
 

api/.env

@@ -17,11 +17,13 @@ REDIS_CACHE_TIME=86400000
 
 # MINIO
 MINIO_BUCKET=files
+MINIO_EXPIRATION=1
 MINIO_URL=http://minio-api:9000
 MINIO_ROOT_USER=minioadmin
 MINIO_ROOT_PASSWORD=minioadmin
 MINIO_CONSOLE_PORT=9090
 MINIO_PORT=9000
+MINIO_TTL_PREFIX=temporary/
 
 # MONGO
 MONGO_INITDB_ROOT_USERNAME=admin

api/pom.xml

@@ -19,7 +19,6 @@
         <springdoc.starter.version>2.1.0</springdoc.starter.version>
         <jsonwebtoken.version>0.11.5</jsonwebtoken.version>
         <minio.version>8.5.7</minio.version>
-        <kafka-clients.version>3.7.0</kafka-clients.version>
 
 <!--        TESTS-->
         <containers.version>1.19.1</containers.version>

api/src/main/java/com/github/asavershin/api/config/KafkaConf.java

@@ -102,6 +102,7 @@ private ProducerFactory<String, FiltersForPublisher> producerFactory(
                 StringSerializer.class);
         props.put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG,
                 JsonSerializer.class);
+        props.put(JsonSerializer.ADD_TYPE_INFO_HEADERS, false);
         // Партиция одна, так что все равно как роутить
         props.put(ProducerConfig.PARTITIONER_CLASS_CONFIG,
                 RoundRobinPartitioner.class);

api/src/main/java/com/github/asavershin/api/config/properties/MinIOProperties.java

@@ -14,19 +14,24 @@ public class MinIOProperties {
      * The name of the bucket in the MinIO service.
      */
     private String bucket;
-
     /**
      * The URL of the MinIO service.
      */
     private String url;
-
     /**
      * The username for the MinIO service.
      */
     private String user;
-
     /**
      * The password for the MinIO service.
      */
     private String password;
+    /**
+     * The ttl of the MinIO objects.
+     */
+    private String expiration;
+    /**
+     * The name of the bucket with ttl.
+     */
+    private String ttlprefix;
 }

api/src/main/java/com/github/asavershin/api/domain/filter/Filter.java

@@ -10,17 +10,12 @@ public enum Filter {
     /**
      * Represents the REVERS_COLORS filter.
      */
-    REVERS_COLORS,
+    ROTATE,
 
     /**
      * Represents the CROP filter.
      */
-    CROP,
-
-    /**
-     * Represents the REMOVE_BACKGROUND filter.
-     */
-    REMOVE_BACKGROUND;
+    BLACKWHITE;
 
     /**
      * Converts a string representation of a filter name string

api/src/main/java/com/github/asavershin/api/infrastructure/out/storage/MinioServiceIml.java

@@ -9,12 +9,20 @@
 import io.minio.MinioClient;
 import io.minio.PutObjectArgs;
 import io.minio.RemoveObjectArgs;
+import io.minio.SetBucketLifecycleArgs;
+import io.minio.messages.Expiration;
+import io.minio.messages.LifecycleConfiguration;
+import io.minio.messages.LifecycleRule;
+import io.minio.messages.RuleFilter;
+import io.minio.messages.Status;
 import lombok.SneakyThrows;
 import org.apache.commons.compress.utils.IOUtils;
 import org.springframework.stereotype.Service;
 import org.springframework.web.multipart.MultipartFile;
 
 import java.io.InputStream;
+import java.time.ZonedDateTime;
+import java.util.LinkedList;
 import java.util.List;
 
 @Service
@@ -27,21 +35,23 @@ public class MinioServiceIml implements MinioService {
      * The MinIO properties from .yml.
      */
     private final MinIOProperties minioProperties;
+
     /**
      * Constructor for {@link MinioServiceIml}.
      *
-     * @param aMinioClient The {@link MinioClient}
-     *                    instance to interact with the MinIO server.
+     * @param aMinioClient     The {@link MinioClient}
+     *                         instance to interact with the MinIO server.
      * @param aMinioProperties The {@link MinIOProperties}
-     *                        instance containing the configuration
-     *                        for the MinIO server.
+     *                         instance containing the configuration
+     *                         for the MinIO server.
      */
     public MinioServiceIml(final MinioClient aMinioClient,
                            final MinIOProperties aMinioProperties) {
         this.minioClient = aMinioClient;
         this.minioProperties = aMinioProperties;
         createBucket();
     }
+
     /**
      * Not final to allow spring use proxy.
      */
@@ -66,6 +76,7 @@ public void saveFile(final MultipartFile image, final String filename) {
         }
         saveFile(inputStream, filename);
     }
+
     /**
      * Not final to allow spring use proxy.
      */
@@ -77,13 +88,14 @@ public byte[] getFile(final String link) {
         try {
             return IOUtils.toByteArray(
                     minioClient.getObject(GetObjectArgs.builder()
-                    .bucket(minioProperties.getBucket())
-                    .object(link)
-                    .build()));
+                            .bucket(minioProperties.getBucket())
+                            .object(link)
+                            .build()));
         } catch (Exception e) {
             throw new FileException("File download failed: " + e.getMessage());
         }
     }
+
     /**
      * Not final to allow spring use proxy.
      */
@@ -110,14 +122,35 @@ public void deleteFiles(final List<String> links) {
 
     @SneakyThrows
     private void createBucket() {
-        boolean found = minioClient.bucketExists(BucketExistsArgs.builder()
-                .bucket(minioProperties.getBucket())
-                .build());
+        boolean found = bucketExists(minioProperties.getBucket());
         if (!found) {
+            List<LifecycleRule> rules = new LinkedList<>();
+            rules.add(
+                    new LifecycleRule(
+                            Status.ENABLED,
+                            null,
+                            new Expiration((ZonedDateTime) null,
+                                    Integer.valueOf(
+                                            minioProperties.getExpiration()
+                                    ),
+                                    null),
+                            new RuleFilter(minioProperties.getTtlprefix()),
+                            "rule1",
+                            null,
+                            null,
+                            null));
             minioClient.makeBucket(MakeBucketArgs.builder()
                     .bucket(minioProperties.getBucket())
                     .build());
+            minioClient.setBucketLifecycle(
+                    SetBucketLifecycleArgs.builder().bucket(
+                                    minioProperties.getBucket()
+                            )
+                            .config(
+                                    new LifecycleConfiguration(rules)
+                            ).build());
         }
+
     }
 
     @SneakyThrows

api/src/main/resources/application.yml

@@ -54,6 +54,8 @@ minio:
   url: ${MINIO_URL}
   user: ${MINIO_ROOT_USER}
   password: ${MINIO_ROOT_PASSWORD}
+  expiration: ${MINIO_EXPIRATION}
+  ttlprefix: ${MINIO_TTL_PREFIX}
   bucket: ${MINIO_BUCKET}
   console-port: ${MINIO_CONSOLE_PORT}
   port: ${MINIO_PORT}

api/src/test/java/com/github/asavershin/api/common/EventHelper.java

@@ -21,15 +21,14 @@ public class EventHelper {
     public static List<Filter> hugeList() {
         var result = new ArrayList<Filter>();
         for (int i = 0; i < 32780; i++) {
-            result.add(Filter.REVERS_COLORS);
+            result.add(Filter.ROTATE);
         }
         return result;
     }
 
     public static List<Filter> filters() {
-        return List.of(Filter.REVERS_COLORS,
-                Filter.CROP,
-                Filter.REMOVE_BACKGROUND);
+        return List.of(Filter.ROTATE,
+                Filter.BLACKWHITE);
     }
 
     public static ImageProcessingStarted started(final ImageId id) {

api/src/test/java/com/github/asavershin/api/domaintest/FilterEventTest.java

@@ -23,7 +23,7 @@ public void testImageProcessingStartedConstructor() {
         // Given
         ImageId imageId = ImageId.nextIdentity();
         List<Filter> filters = new ArrayList<>();
-        filters.add(Filter.CROP);
+        filters.add(Filter.BLACKWHITE);
         // When
         var event = new ImageProcessingStarted(filters, imageId);
 
@@ -40,7 +40,7 @@ public void testNullOriginalImageIdToNewEvent() {
         // Given
         var lotsOfFilters = hugeList();
         var zeroFilters = new ArrayList<Filter>();
-        var normalFilters = List.of(Filter.CROP);
+        var normalFilters = List.of(Filter.BLACKWHITE);
 
         // When
         var nullIdEx = assertThrows(NullPointerException.class,
@@ -68,7 +68,7 @@ public void testZeroFiltersToNewEvent() {
     void testPublish() {
         // Arrange
         var originalImageId = ImageId.nextIdentity();
-        List<Filter> filters = List.of(Filter.CROP, Filter.REMOVE_BACKGROUND);
+        List<Filter> filters = List.of(Filter.BLACKWHITE, Filter.ROTATE);
         ImageProcessingStarted imageProcessingStarted = new ImageProcessingStarted(filters, originalImageId);
 
         // Act
@@ -109,7 +109,7 @@ public void testImageProcessingResultConstructor() {
     private List<Filter> createFiltersWithMaxCount() {
         List<Filter> filters = new ArrayList<>();
         for (int i = 0; i < 32780; i++) {
-            filters.add(Filter.CROP);
+            filters.add(Filter.BLACKWHITE);
         }
         return filters;
     }

api/src/test/java/com/github/asavershin/api/integrations/EventLogicTest.java

@@ -92,7 +92,7 @@ public void testStartedEvent() {
         startedEvent.store(newEvent, testUser.userId());
         // Then
         var storedEvent = getEventFromDB(dsl, newEvent.requestId());
-        assertEquals(storedEvent.size(), 3);
+        assertEquals(storedEvent.size(), 2);
         assertNotNull(storedEvent);
         assertNotNull(storedEvent.get(0).getValue(IMAGE_PROCESSING_EVENT.IMAGE_PROCESSING_EVENT_ID));
         assertEquals(storedEvent.get(0).getValue(IMAGE_PROCESSING_EVENT.IMAGE_PROCESSING_EVENT_ID),

api/src/test/resources/application.properties

@@ -23,6 +23,7 @@ minio.password=minioadmin
 minio.bucket=files
 minio.console-port=9090
 minio.port=9000
+minio.expiration=1
 
 testconf=test
 

pom.xml

@@ -16,6 +16,7 @@
 
 	<modules>
 		<module>api</module>
+		<module>worker</module>
 	</modules>
 
 	<properties>

worker/.env

@@ -0,0 +1,18 @@
+SERVER_PORT=8081
+
+# MINIO
+MINIO_BUCKET=files
+MINIO_TTL_PREFIX=temporary/
+MINIO_EXPIRATION=1
+MINIO_URL=http://minio-api:9000
+MINIO_ROOT_USER=minioadmin
+MINIO_ROOT_PASSWORD=minioadmin
+MINIO_CONSOLE_PORT=9090
+MINIO_PORT=9000
+
+# REDIS
+REDIS_HOST_ROTATE=redis-rotate
+REDIS_HOST_BLACKWHITE=redis-blackwhite
+REDIS_PORT=6379
+REDIS_PASSWORD=cGFzc3dvcmxk
+REDIS_CACHE_TIME=86400000

worker/Dockerfile-blackwhite

@@ -0,0 +1,7 @@
+FROM openjdk:21
+
+WORKDIR /app
+
+COPY target/*.jar app.jar
+
+CMD ["java", "-jar", "-Dspring.profiles.active=blackwhite", "app.jar"]

worker/Dockerfile-rotate

@@ -0,0 +1,7 @@
+FROM openjdk:21
+
+WORKDIR /app
+
+COPY target/*.jar app.jar
+
+CMD ["java", "-jar", "-Dspring.profiles.active=rotate", "app.jar"]

worker/docker-compose-blackwhite.yml

@@ -0,0 +1,52 @@
+networks:
+  kafka-net:
+    name: kafka-net
+    driver: bridge
+  api-net:
+    name: api-net
+    driver: bridge
+
+services:
+
+  backend-worker-blackwhite:
+    container_name: backend-worker-blackwhite
+    networks:
+      - kafka-net
+      - api-net
+    build:
+      context: .
+      dockerfile: Dockerfile-blackwhite
+    ports:
+      - 8083:8081
+    env_file:
+      - .env
+
+  redis-blackwhite:
+    networks:
+      - api-net
+    image: redis:7.2-rc-alpine
+    restart: always
+    container_name: redis-blackwhite
+    ports:
+      - '6381:6379'
+    command: redis-server --save 20 1 --loglevel debug --requirepass ${REDIS_PASSWORD}
+    volumes:
+      - redis-rotate-data:/data
+
+#  minio-api:
+#    networks:
+#      - api-net
+#    image: minio/minio:RELEASE.2024-02-14T21-36-02Z
+#    container_name: minio-api
+#    env_file:
+#      - .env
+#    command: server ~/minio --console-address :9090
+#    ports:
+#      - '9090:9090'
+#      - '9000:9000'
+#    volumes:
+#      - minio-api-data:/minio
+
+volumes:
+#  minio-api-data:
+  redis-rotate-data: