From 4e9737d74c90551eaeb810cf440e3359b73f88b8 Mon Sep 17 00:00:00 2001 From: Noel Date: Fri, 16 Dec 2022 23:11:00 -0700 Subject: [PATCH] Update licenses to 2022-2023, add tests for uploading 1,000 and 100,000 objects in S3 using MinIO, add back Renovate --- .github/workflows/Linting.yaml | 8 ++ bom/build.gradle.kts | 2 +- build.gradle.kts | 2 +- buildSrc/build.gradle.kts | 4 - .../org/noelware/remi/gradle/Extensions.kt | 2 +- .../org/noelware/remi/gradle/Metadata.kt | 4 +- .../src/main/kotlin/remi-module.gradle.kts | 9 +- renovate.json | 7 ++ settings.gradle.kts | 85 +++++++++++++++++- support/azure/build.gradle.kts | 4 +- support/fs/build.gradle.kts | 7 +- .../filesystem/FilesystemStorageService.java | 7 +- support/gcs/build.gradle.kts | 3 +- support/s3/build.gradle.kts | 5 +- .../support/s3/AmazonS3StorageService.java | 37 +++----- .../s3/AmazonS3StorageServiceTests.java | 86 ++++++++++++++++++- 16 files changed, 212 insertions(+), 60 deletions(-) create mode 100644 renovate.json diff --git a/.github/workflows/Linting.yaml b/.github/workflows/Linting.yaml index 1384134f..60920282 100644 --- a/.github/workflows/Linting.yaml +++ b/.github/workflows/Linting.yaml @@ -66,6 +66,9 @@ jobs: - name: Validate Gradle Wrapper uses: gradle/wrapper-validation-action@v1 + - name: Upload Google Credentials in ~/.google-creds.json + run: echo "${{secrets.GOOGLE_CREDENTIALS_JSON}}" >> ~/.google-creds.json + - name: Lint code-base with Spotless uses: gradle/gradle-build-action@v2 with: @@ -75,3 +78,8 @@ jobs: uses: gradle/gradle-build-action@v2 with: arguments: compileJava --no-daemon --scan + + - name: Run all unit tests + uses: gradle/gradle-build-action@v2 + with: + arguments: test --no-daemon --scan diff --git a/bom/build.gradle.kts b/bom/build.gradle.kts index 4f841032..db747189 100644 --- a/bom/build.gradle.kts +++ b/bom/build.gradle.kts @@ -84,7 +84,7 @@ val snapshotRelease: Boolean = run { publishing { publications { - create("ktor") { + create("remi") { from(components["javaPlatform"]) artifactId = "remi-bom" diff --git a/build.gradle.kts b/build.gradle.kts index 6c293131..8eb65751 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/buildSrc/build.gradle.kts b/buildSrc/build.gradle.kts index 41c60ba9..594c3a4f 100644 --- a/buildSrc/build.gradle.kts +++ b/buildSrc/build.gradle.kts @@ -34,10 +34,6 @@ repositories { dependencies { implementation("com.diffplug.spotless:spotless-plugin-gradle:6.12.0") - implementation("org.jetbrains.dokka:dokka-gradle-plugin:1.7.20") - implementation(kotlin("gradle-plugin", version = "1.7.22")) - implementation(kotlin("serialization", version = "1.7.22")) - implementation("io.kotest:kotest-gradle-plugin:0.3.9") implementation("dev.floofy.commons:gradle:2.4.0") implementation(gradleApi()) } diff --git a/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Extensions.kt b/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Extensions.kt index 8103026c..7a5e0ab2 100644 --- a/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Extensions.kt +++ b/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Extensions.kt @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal diff --git a/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Metadata.kt b/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Metadata.kt index 0674f2e6..169f8bc9 100644 --- a/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Metadata.kt +++ b/buildSrc/src/main/kotlin/org/noelware/remi/gradle/Metadata.kt @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -26,5 +26,5 @@ package org.noelware.remi.gradle import org.gradle.api.JavaVersion import dev.floofy.utils.gradle.* -val VERSION = Version(0, 5, 0, 0, ReleaseType.Snapshot) +val VERSION = Version(0, 5, 1, 0, ReleaseType.Beta) val JAVA_VERSION = JavaVersion.VERSION_17 diff --git a/buildSrc/src/main/kotlin/remi-module.gradle.kts b/buildSrc/src/main/kotlin/remi-module.gradle.kts index bc4abc6a..5af7838f 100644 --- a/buildSrc/src/main/kotlin/remi-module.gradle.kts +++ b/buildSrc/src/main/kotlin/remi-module.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -46,11 +46,16 @@ repositories { dependencies { implementation("org.jetbrains:annotations:23.0.0") + implementation("org.slf4j:slf4j-api:2.0.6") // test deps testImplementation("org.junit.jupiter:junit-jupiter-api:5.9.1") testRuntimeOnly("org.junit.jupiter:junit-jupiter-engine:5.9.1") - testImplementation("org.slf4j:slf4j-simple:2.0.3") + testImplementation("org.slf4j:slf4j-simple:2.0.6") + + if (path.startsWith(":support")) { + implementation(project(":core")) + } } spotless { diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..171832b1 --- /dev/null +++ b/renovate.json @@ -0,0 +1,7 @@ +{ + "automerge": true, + "extends": ["config:base", "default:timezone(America/Phoenix)", "docker:disableMajor"], + "vulnerabilityAlerts": { + "labels": ["security"] + } +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 8f6215dd..89bcd4a4 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -23,13 +23,94 @@ rootProject.name = "remi" +pluginManagement { + repositories { + gradlePluginPortal() + mavenCentral() + mavenLocal() + } +} + +plugins { + id("com.gradle.enterprise") version "3.12" +} + include( ":bom", ":core", - ":serialization", ":support:azure", ":support:fs", ":support:gcs", ":support:oracle-cloud", ":support:s3" ) + +gradle.settingsEvaluated { + logger.info("Checking if we can overwrite cache...") + val overrideBuildCacheProp: String? = System.getProperty("org.noelware.gradle.overwriteCache") + val buildCacheDir = when (val prop = System.getProperty("org.noelware.gradle.cachedir")) { + null -> "${System.getProperty("user.dir")}/.caches/gradle" + else -> when { + prop.startsWith("~/") -> "${System.getProperty("user.home")}${prop.substring(1)}" + prop.startsWith("./") -> "${System.getProperty("user.dir")}${prop.substring(1)}" + else -> prop + } + } + + if (overrideBuildCacheProp == null) { + logger.info(""" + |If you wish to override the build cache for this Gradle process, you can use the + |-Dorg.noelware.gradle.overwriteCache= Java property in `~/.gradle/gradle.properties` + |to overwrite it in $buildCacheDir! + """.trimMargin("|")) + } else { + logger.info("Setting up build cache in directory [$buildCacheDir]") + val file = File(buildCacheDir) + if (!file.exists()) file.mkdirs() + + buildCache { + local { + directory = "$file" + removeUnusedEntriesAfterDays = 7 + } + } + } + + val disableJavaSanityCheck = when { + System.getProperty("org.noelware.gradle.ignoreJavaCheck", "false").matches("^(yes|true|1|si|si*)$".toRegex()) -> true + (System.getenv("DISABLE_JAVA_SANITY_CHECK") ?: "false").matches("^(yes|true|1|si|si*)$".toRegex()) -> true + else -> false + } + + if (disableJavaSanityCheck) + return@settingsEvaluated + + val version = JavaVersion.current() + if (version.majorVersion.toInt() < 17) + throw GradleException("Developing charted-server requires JDK 17 or higher, it is currently set in [${System.getProperty("java.home")}, ${System.getProperty("java.version")}] - You can ignore this check by providing the `-Dorg.noelware.charted.ignoreJavaCheck=true` system property.") +} + +val buildScanServer = System.getProperty("org.noelware.gradle.build-scan-server", "") ?: "" +gradleEnterprise { + buildScan { + if (buildScanServer.isNotEmpty()) { + server = buildScanServer + isCaptureTaskInputFiles = true + publishAlways() + } else { + termsOfServiceUrl = "https://gradle.com/terms-of-service" + termsOfServiceAgree = "yes" + + // Always publish if we're on CI. + if (System.getenv("CI") != null) { + publishAlways() + } + } + + obfuscation { + ipAddresses { listOf("0.0.0.0") } + hostname { "[redacted]" } + username { "[redacted]" } + } + } +} diff --git a/support/azure/build.gradle.kts b/support/azure/build.gradle.kts index 3f07ba95..9b68c7d5 100644 --- a/support/azure/build.gradle.kts +++ b/support/azure/build.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -31,6 +31,4 @@ dependencies { testImplementation("org.testcontainers:junit-jupiter:1.17.6") api("com.azure:azure-storage-blob:12.20.1") - implementation("org.slf4j:slf4j-api:2.0.5") - implementation(project(":core")) } diff --git a/support/fs/build.gradle.kts b/support/fs/build.gradle.kts index 43b6b2b1..c69aa4d9 100644 --- a/support/fs/build.gradle.kts +++ b/support/fs/build.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -24,8 +24,3 @@ plugins { `remi-module` } - -dependencies { - implementation("org.slf4j:slf4j-api:2.0.5") - implementation(project(":core")) -} diff --git a/support/fs/src/main/java/org/noelware/remi/support/filesystem/FilesystemStorageService.java b/support/fs/src/main/java/org/noelware/remi/support/filesystem/FilesystemStorageService.java index ab6978dd..85cf7567 100644 --- a/support/fs/src/main/java/org/noelware/remi/support/filesystem/FilesystemStorageService.java +++ b/support/fs/src/main/java/org/noelware/remi/support/filesystem/FilesystemStorageService.java @@ -232,16 +232,15 @@ public List blobs() throws IOException { * @param request The request options object * @throws IOException If any I/O exceptions had occurred while uploading the file. */ + @SuppressWarnings("ResultOfMethodCallIgnored") @Override public void upload(UploadRequest request) throws IOException { - final InputStream stream = request.inputStream(); final String path = request.path(); - final File file = Paths.get(config.directory(), path).toFile(); Files.createDirectories(Paths.get(file.getParent())); file.createNewFile(); - try (stream; + try (final InputStream stream = request.inputStream(); final FileOutputStream out = new FileOutputStream(file)) { byte[] buf = new byte[stream.available()]; int len; @@ -338,7 +337,7 @@ public boolean delete(String path) throws IOException { public void init() { final File directory = new File(config.directory()); if (!directory.exists()) { - LOG.debug("Directory [{}] does not exist on local disk, creating!", normalizePath(config.directory())); + LOG.debug("Directory [{}] does not exist on local disk, creating!", config.directory()); directory.mkdirs(); } diff --git a/support/gcs/build.gradle.kts b/support/gcs/build.gradle.kts index eb2d1eb1..2239d4a6 100644 --- a/support/gcs/build.gradle.kts +++ b/support/gcs/build.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -27,5 +27,4 @@ plugins { dependencies { api("com.google.cloud:google-cloud-storage:2.16.0") - implementation(project(":core")) } diff --git a/support/s3/build.gradle.kts b/support/s3/build.gradle.kts index bde34bd9..3cd1e7ab 100644 --- a/support/s3/build.gradle.kts +++ b/support/s3/build.gradle.kts @@ -1,6 +1,6 @@ /* * 🧶 Remi: Robust, and simple Java-based library to handle storage-related communications with different storage provider. - * Copyright (c) 2022 Noelware + * Copyright (c) 2022-2023 Noelware * * Permission is hereby granted, free of charge, to any person obtaining a copy * of this software and associated documentation files (the "Software"), to deal @@ -29,6 +29,5 @@ dependencies { testImplementation("org.testcontainers:testcontainers:1.17.6") testImplementation("org.testcontainers:junit-jupiter:1.17.6") - implementation(project(":core")) - api("software.amazon.awssdk:s3:2.18.24") + api("software.amazon.awssdk:s3:2.18.35") } diff --git a/support/s3/src/main/java/org/noelware/remi/support/s3/AmazonS3StorageService.java b/support/s3/src/main/java/org/noelware/remi/support/s3/AmazonS3StorageService.java index fa23c9b5..cb004fe0 100644 --- a/support/s3/src/main/java/org/noelware/remi/support/s3/AmazonS3StorageService.java +++ b/support/s3/src/main/java/org/noelware/remi/support/s3/AmazonS3StorageService.java @@ -163,34 +163,12 @@ public List blobs(@Nullable ListBlobsRequest request) throws IOException { final ListObjectsV2Response resp = client.listObjectsV2(listObjectsV2Request); try (final Stream stream = resp.contents().parallelStream()) { final List newBlobs = stream.map(obj -> { - if (obj.key().endsWith("/")) return null; - - byte[] data; - try (final InputStream s = open(obj.key())) { - assert s != null; - data = s.readAllBytes(); - } catch (IOException e) { - throw new RuntimeException(e); - } - - String contentType; try { - contentType = getContentTypeOf(data); + return fromS3Object(obj); } catch (IOException e) { - // fallback - contentType = "application/octet-stream"; + // Do we throw or return null? I have no idea to be honest... + throw new RuntimeException(e); } - - return new Blob( - obj.lastModified(), - null, - contentType, - new ByteArrayInputStream(data), - obj.eTag(), - toPrefixedString(obj.key()), - "s3", - format("s3://%s", toPrefixedString(obj.key())), - obj.size()); }) .filter(Objects::nonNull) .toList(); @@ -226,6 +204,7 @@ public List blobs() throws IOException { */ @Override public void upload(UploadRequest request) throws IOException { + LOG.debug("Uploading object in path [{}] with content type [{}]", request.path(), request.contentType()); try (final InputStream stream = request.inputStream()) { if (stream.available() == 0) throw new IllegalStateException( @@ -258,7 +237,11 @@ public boolean exists(String path) { builder.key(toPrefixedString(path)); }); - return !resp.deleteMarker(); + final Boolean deleteMarker = resp.deleteMarker(); + if (deleteMarker != null) return !deleteMarker; + + // assume it is true + return true; } catch (NoSuchKeyException ignored) { return false; } @@ -430,6 +413,8 @@ private String toPrefixedString(String key) { @Nullable private Blob fromS3Object(S3Object obj) throws IOException { if (obj.key().endsWith("/")) return null; + + LOG.debug(""); return blob(obj.key()); } } diff --git a/support/s3/src/test/java/org/noelware/remi/testing/support/s3/AmazonS3StorageServiceTests.java b/support/s3/src/test/java/org/noelware/remi/testing/support/s3/AmazonS3StorageServiceTests.java index f981e026..0b9d33c4 100644 --- a/support/s3/src/test/java/org/noelware/remi/testing/support/s3/AmazonS3StorageServiceTests.java +++ b/support/s3/src/test/java/org/noelware/remi/testing/support/s3/AmazonS3StorageServiceTests.java @@ -26,22 +26,30 @@ import static java.lang.String.format; import static org.junit.jupiter.api.Assertions.*; +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.List; import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.noelware.remi.core.Blob; +import org.noelware.remi.core.UploadRequest; import org.noelware.remi.support.s3.AmazonS3StorageConfig; import org.noelware.remi.support.s3.AmazonS3StorageService; +import org.slf4j.LoggerFactory; import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.output.Slf4jLogConsumer; import org.testcontainers.containers.wait.strategy.HttpWaitStrategy; import org.testcontainers.junit.jupiter.Testcontainers; import org.testcontainers.utility.DockerImageName; @Testcontainers(disabledWithoutDocker = true) public class AmazonS3StorageServiceTests { + private static final AtomicReference service = new AtomicReference<>(null); private static final GenericContainer minioContainer = new GenericContainer<>( DockerImageName.parse("quay.io/minio/minio:RELEASE.2022-12-07T00-56-37Z")) .withExposedPorts(9000, 9090) @@ -53,6 +61,12 @@ public class AmazonS3StorageServiceTests { "MINIO_SECRET_KEY", "remitest")); private void withAmazonS3StorageService(Consumer serviceConsumer) { + final AmazonS3StorageService s3Service = service.get(); + if (s3Service != null) { + serviceConsumer.accept(s3Service); + return; + } + final AmazonS3StorageConfig config = AmazonS3StorageConfig.builder() .withAccessKeyId("remitest") .withSecretAccessKey("remitest") @@ -61,9 +75,11 @@ private void withAmazonS3StorageService(Consumer service .withEndpoint(format("http://%s:%d", minioContainer.getHost(), minioContainer.getMappedPort(9000))) .build(); - final AmazonS3StorageService service = new AmazonS3StorageService(config); - service.init(); - serviceConsumer.accept(service); + final AmazonS3StorageService newS3Service = new AmazonS3StorageService(config); + newS3Service.init(); + + service.set(newS3Service); + serviceConsumer.accept(newS3Service); } @BeforeAll @@ -72,6 +88,8 @@ public static void beforeRun() { .forPath("/minio/health/ready") .forPort(9000) .withStartupTimeout(Duration.ofMinutes(2))); + + minioContainer.setLogConsumers(List.of(new Slf4jLogConsumer(LoggerFactory.getLogger("io.minio.docker")))); minioContainer.start(); } @@ -84,4 +102,66 @@ public void test_canWeConnectToS3() { assertEquals(0, blobs.size()); }); } + + @Test + public void test_canWeUploadObjects() { + withAmazonS3StorageService(service -> { + assertFalse(service.exists("/wuff.json")); + assertDoesNotThrow(() -> service.upload(UploadRequest.builder() + .withPath("/wuff.json") + .withInputStream(new ByteArrayInputStream("{\"wuffs\":true}".getBytes(StandardCharsets.UTF_8))) + .withContentType("application/json") + .build())); + + assertTrue(service.exists("/wuff.json")); + assertDoesNotThrow(() -> { + final List blobs = service.blobs(); + assertEquals(1, blobs.size()); + + final Blob wuffBlob = service.blob("/wuff.json"); + assertNotNull(wuffBlob); + assertEquals("application/json", wuffBlob.contentType()); + + try (final InputStream stream = wuffBlob.inputStream()) { + assert stream != null; + + final String content = new String(stream.readAllBytes()); + assertEquals("{\"wuffs\":true}", content); + } + }); + }); + } + + @Test + public void test_canWeVUploadAndFetch1000And100000Items() { + withAmazonS3StorageService(service -> { + assertDoesNotThrow(() -> { + assertEquals(1, service.blobs().size()); + + for (int i = 0; i < 1000; i++) { + final int key = i + 1; + assertDoesNotThrow(() -> service.upload(UploadRequest.builder() + .withPath(format("/test/%d.json", key)) + .withInputStream(new ByteArrayInputStream( + format("{\"key\":%d,\"wuffs\":true}", key).getBytes(StandardCharsets.UTF_8))) + .withContentType("application/json") + .build())); + } + + assertEquals(1001, service.blobs().size()); + + for (int i = 1000; i < 99999; i++) { + final int key = i + 1; + assertDoesNotThrow(() -> service.upload(UploadRequest.builder() + .withPath(format("/test/%d.json", key)) + .withInputStream(new ByteArrayInputStream( + format("{\"key\":%d,\"wuffs\":true}", key).getBytes(StandardCharsets.UTF_8))) + .withContentType("application/json") + .build())); + } + + assertEquals(100000, service.blobs().size()); + }); + }); + } }