Add "internal config update" endpoint to set externalUrl property aft…

…er the fake server start (#659)

* modify image name to sergseven

* return PublicURL() as location response header

* add .tool-versions

* revert 'return PublicURL() as location response header'

* add internal update server config endpoint

* revert wrong test case

* remove if body != nil since always non-nil

* fix examples to set a real external url

* fix test case name

* remove redundant return

* remove redundant test case

* add mapping for no public host path

* add java testcontainers example

* avoid dep download progress showed

* set encoding

* set project for storage client

* clean up

* try no updateExternalUrlWithContainerHost

* revert try no updateExternalUrlWithContainerHost

* rename fakeGcsExternalUrl

* update documentation

* modify "modify server config" endpoint to take json

* fix updateExternalUrlWithContainerUrl url

* use new image version

* rollback image name to fsouza/fake-gcs-server

* configure java example

* try docker-in-docker

* rename /internal/config endpoint to /_internal/config

* remove .tool-versions

* simplify java example to demonstrate a simple case

* remove docker-in-docker for tests at CI

* improve java example readme

* examples/java: cleanup some maven wrapper stuff

Co-authored-by: francisco souza <[email protected]>

.github/workflows/main.yml

@@ -136,6 +136,9 @@ jobs:
           - lang: scala
             docker-image: mozilla/sbt:latest
             entrypoint: /bin/sh
+          - lang: java
+            docker-image: openjdk:11
+            entrypoint: /bin/sh
 
     name: test-${{ matrix.lang }}-example
     runs-on: ubuntu-latest

README.md

@@ -76,7 +76,7 @@ curl http://0.0.0.0:4443/storage/v1/b/sample-bucket/o
 
 ## Client library examples
 
-For examples using the Python, Node.js and Go clients, check out the
+For examples using the Python, Node.js, Java and Go clients, check out the
 [``examples``](/examples/) directory.
 
 ### Building the image locally

ci/run-java-example.sh

@@ -0,0 +1,12 @@
+# Copyright 2022 Francisco Souza. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -e
+
+./fake-gcs-server -backend memory -scheme http -port 8080 -external-url "http://localhost:8080" &
+
+(
+	cd examples/java
+	./mvnw clean test -B
+)

examples/go/main.go

@@ -14,6 +14,8 @@ import (
 	"fmt"
 	"io/ioutil"
 	"log"
+	"net/http"
+	"strings"
 
 	"cloud.google.com/go/storage"
 	"google.golang.org/api/iterator"
@@ -48,6 +50,11 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
+	err = updateConfig()
+	if err != nil {
+		log.Fatal(err)
+	}
 }
 
 func list(client *storage.Client, bucketName string) ([]string, error) {
@@ -78,3 +85,17 @@ func downloadFile(client *storage.Client, bucketName, fileKey string) ([]byte, e
 func deleteFile(client *storage.Client, bucketName, fileKey string) error {
 	return client.Bucket(bucketName).Object(fileKey).Delete(context.TODO())
 }
+
+func updateConfig() error {
+	changeExternalUrl := "http://localhost:8080/_internal/config"
+
+	client := &http.Client{}
+	req, err := http.NewRequest(http.MethodPut, changeExternalUrl, strings.NewReader("{\"externalUrl\": \"http://localhost:8080\"}"))
+	req.Header.Add("Content-Type", "application/json")
+	if err != nil {
+		return err
+	}
+	_, err = client.Do(req)
+
+	return err
+}

examples/java/.gitignore

@@ -0,0 +1,2 @@
+target
+.mvn

examples/java/.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar

examples/java/README.md

@@ -0,0 +1,105 @@
+# Java example
+
+## Common case
+
+          Storage storageClient = StorageOptions.newBuilder()
+            .setHost(fakeGcsExternalUrl)
+            .setProjectId("test-project")
+            .setCredentials(NoCredentials.getInstance())
+            .build()
+            .getService();
+
+See [the example](/src/test/java/com/fsouza/fakegcsserver/java/examples/FakeGcsServerTest.java) for more details.
+
+## Resumable upload operations and containerised fake-gcs-server
+
+The main difficulty with the case when a fake-gcs-server is containerised (by [Testcontainers](https://www.testcontainers.org/) for example)
+is that the container IP and port are assigned after the container is eventually started, so we can not provide
+fake-gcs-server with `external-url` option beforehand(when a test container has not been started yet).
+
+It's necessary to update the server configuration before making resumable upload calls against the server, so the fake server can respond
+with correct `content` HTTP header.
+
+The example will look like as follows:
+
+* additional testcontainers dependencies at `pom.xml`:
+
+```xml
+<dependency>
+  <groupId>org.testcontainers</groupId>
+  <artifactId>testcontainers</artifactId>
+  <version>${testcontainers.version}</version>
+  <scope>test</scope>
+</dependency>
+<dependency>
+  <groupId>org.testcontainers</groupId>
+  <artifactId>junit-jupiter</artifactId>
+  <version>${testcontainers.version}</version>
+  <scope>test</scope>
+</dependency>
+```
+
+  * spin up fake-gcs-server testcontainer right in the test `FakeGcsServerTest.java`:
+```java
+@Testcontainers
+class FakeGcsServerTest {
+
+    @Container
+    static final GenericContainer<?> fakeGcs = new GenericContainer<>("fsouza/fake-gcs-server")
+      .withExposedPorts(8080)
+      .withCreateContainerCmdModifier(cmd -> cmd.withEntrypoint(
+          "/bin/fake-gcs-server",
+          "-scheme", "http"
+      ));
+
+    @BeforeAll
+    static void setUpFakeGcs() throws Exception {
+      String fakeGcsExternalUrl = "http://" + fakeGcs.getContainerIpAddress() + ":" + fakeGcs.getFirstMappedPort();
+
+      updateExternalUrlWithContainerUrl(fakeGcsExternalUrl);
+
+      storageClient = StorageOptions.newBuilder()
+          .setHost(fakeGcsExternalUrl)
+          .setProjectId("test-project")
+          .setCredentials(NoCredentials.getInstance())
+          .build()
+          .getService();
+    }
+
+    private static void updateExternalUrlWithContainerUrl(String fakeGcsExternalUrl) throws Exception {
+      String modifyExternalUrlRequestUri = fakeGcsExternalUrl + "/_internal/config";
+      String updateExternalUrlJson = "{"
+          + "\"externalUrl\": \"" + fakeGcsExternalUrl + "\""
+          + "}";
+
+      HttpRequest req = HttpRequest.newBuilder()
+          .uri(URI.create(modifyExternalUrlRequestUri))
+          .header("Content-Type", "application/json")
+          .PUT(BodyPublishers.ofString(updateExternalUrlJson))
+          .build();
+      HttpResponse<Void> response = HttpClient.newBuilder().build()
+          .send(req, BodyHandlers.discarding());
+
+      if (response.statusCode() != 200) {
+          throw new RuntimeException(
+              "error updating fake-gcs-server with external url, response status code " + response.statusCode() + " != 200");
+      }
+    }
+
+    @Test
+    void shouldUploadFileByWriterChannel() throws IOException {
+
+      storageClient.create(BucketInfo.newBuilder("sample-bucket2").build());
+
+      WriteChannel channel = storageClient.writer(BlobInfo.newBuilder("sample-bucket2", "some_file2.txt").build());
+      channel.write(ByteBuffer.wrap("line1\n".getBytes()));
+      channel.write(ByteBuffer.wrap("line2\n".getBytes()));
+      channel.close();
+
+      Blob someFile2 = storageClient.get("sample-bucket2", "some_file2.txt");
+      String fileContent = new String(someFile2.getContent());
+      assertEquals("line1\nline2\n", fileContent);
+    }
+}
+```
+