Add scheme option to bind to both HTTP and HTTPS (#1215)

* Add scheme option to bind to both HTTP and HTTPS * changes * Change to 2 listeners and a single server * fix * Apply suggestions from code review Co-authored-by: fsouza <[email protected]> * fix for default ports handling * check if flags default value using visit * add example test runs and unit tests --------- Co-authored-by: fsouza <[email protected]>
fsouza · Aug 1, 2023 · a33604b · a33604b
1 parent f71ca54
commit a33604b
Show file tree

Hide file tree

Showing 7 changed files with 375 additions and 51 deletions.
diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -147,6 +147,12 @@ jobs:
           - lang: curl
             docker-image: alpine
             entrypoint: /bin/sh
+          - lang: curl-both-scheme-default-ports
+            docker-image: alpine
+            entrypoint: /bin/sh
+          - lang: curl-both-scheme-non-default-ports
+            docker-image: alpine
+            entrypoint: /bin/sh
 
     name: test-${{ matrix.lang }}-example
     runs-on: ubuntu-latest

diff --git a/README.md b/README.md
@@ -7,33 +7,33 @@ fake-gcs-server provides an emulator for Google Cloud Storage API. It can be
 used as a library in Go projects and/or as a standalone binary/Docker image.
 
 The library is available inside the package
-[``github.com/fsouza/fake-gcs-server/fakestorage``](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc)
+[`github.com/fsouza/fake-gcs-server/fakestorage`](https://pkg.go.dev/github.com/fsouza/fake-gcs-server/fakestorage?tab=doc)
 and can be used from within test suites in Go package. The emulator is
 available as a binary that can be built manually, downloaded from the [releases
 page](https://github.com/fsouza/fake-gcs-server/releases) or pulled from Docker
-Hub ([``docker pull
-fsouza/fake-gcs-server``](https://hub.docker.com/r/fsouza/fake-gcs-server)).
+Hub ([`docker pull
+fsouza/fake-gcs-server`](https://hub.docker.com/r/fsouza/fake-gcs-server)).
 
 ## Using the emulator in Docker
 
 You can stub/mock Google Cloud Storage as a standalone server (like the datastore/pubsub emulators)
 which is ideal for integration tests and/or tests in other languages you may want to run the
-``fake-gcs-server`` inside a Docker container:
+`fake-gcs-server` inside a Docker container:
 
 ```shell
 docker run -d --name fake-gcs-server -p 4443:4443 fsouza/fake-gcs-server
 ```
 
 ### Preload data
 
-In case you want to preload some data in ``fake-gcs-server`` just mount a
-folder in the container at ``/data``:
+In case you want to preload some data in `fake-gcs-server` just mount a
+folder in the container at `/data`:
 
 ```shell
 docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server
 ```
 
-Where the content of ``${PWD}/examples/data`` is something like:
+Where the content of `${PWD}/examples/data` is something like:
 
 ```
 .
@@ -51,13 +51,14 @@ curl --insecure https://0.0.0.0:4443/storage/v1/b/sample-bucket/o
 {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]}
 ```
 
-This will result in one bucket called ``sample-bucket`` containing one object called ``some_file.txt``.
+This will result in one bucket called `sample-bucket` containing one object called `some_file.txt`.
 
 ### Running with HTTP
 
 fake-gcs-server defaults to HTTPS, but it can also be used with HTTP. The flag
-`-scheme` can be used to specify the protocol. For example, the previous
-example could be changed to pass `-scheme http`:
+`-scheme` can be used to specify the protocol.  
+The binding port will be `-port` (defaults to `4443`).  
+For example, the previous example could be changed to pass `-scheme http`:
 
 ```shell
 docker run -d --name fake-gcs-server -p 4443:4443 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme http
@@ -74,6 +75,16 @@ curl http://0.0.0.0:4443/storage/v1/b/sample-bucket/o
 {"kind":"storage#objects","items":[{"kind":"storage#object","name":"some_file.txt","id":"sample-bucket/some_file.txt","bucket":"sample-bucket","size":"33"}],"prefixes":[]}
 ```
 
+### Running with both HTTPS and HTTP
+
+To start both HTTPS and HTTP servers, pass `-scheme both`.  
+HTTPS will bind to `-port` (defaults to `4443`) and HTTP will bind to `-port-http` (defaults to `8000`).  
+For example, the previous example could be changed to pass `-scheme both`:
+
+```shell
+docker run -d --name fake-gcs-server -p 4443:4443 -p 8000:8000 -v ${PWD}/examples/data:/data fsouza/fake-gcs-server -scheme both
+```
+
 ### Using with signed URLs
 
 It is possible to use fake-gcs-server with signed URLs, although with a few caveats:
@@ -97,11 +108,11 @@ docker run --rm fsouza/fake-gcs-server -help
 ## Client library examples
 
 For examples using SDK from multiple languages, check out the
-[``examples``](/examples/) directory.
+[`examples`](/examples/) directory.
 
 ### Building the image locally
 
-You may use ``docker build`` to build the image locally instead of pulling it
+You may use `docker build` to build the image locally instead of pulling it
 from Docker Hub:
 
 ```shell

diff --git a/ci/run-curl-both-scheme-default-ports-example.sh b/ci/run-curl-both-scheme-default-ports-example.sh
@@ -0,0 +1,11 @@
+# Copyright 2023 Francisco Souza. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -e
+
+./fake-gcs-server -backend memory -scheme both -data ${PWD}/examples/data &
+
+apk add --update curl
+curl --silent --fail --insecure https://0.0.0.0:4443/storage/v1/b
+curl --silent --fail --insecure http://0.0.0.0:8000/storage/v1/b
diff --git a/ci/run-curl-both-scheme-non-default-ports-example.sh b/ci/run-curl-both-scheme-non-default-ports-example.sh
@@ -0,0 +1,11 @@
+# Copyright 2023 Francisco Souza. All rights reserved.
+# Use of this source code is governed by a BSD-style
+# license that can be found in the LICENSE file.
+
+set -e
+
+./fake-gcs-server -backend memory -port 5553 -port-http 9000 -scheme both -data ${PWD}/examples/data &
+
+apk add --update curl
+curl --silent --fail --insecure https://0.0.0.0:5553/storage/v1/b
+curl --silent --fail --insecure http://0.0.0.0:9000/storage/v1/b
diff --git a/internal/config/config.go b/internal/config/config.go
@@ -24,13 +24,21 @@ const (
 	eventDelete         = "delete"
 	eventMetadataUpdate = "metadataUpdate"
 	eventArchive        = "archive"
+	defaultHTTPSPort    = 4443
+	defaultHTTPPort     = 8000
+	schemeHTTPS         = "https"
+	schemeHTTP          = "http"
+	schemeBoth          = "both"
+	flagPort            = "port"
+	flagPortHTTP        = "port-http"
 )
 
 type Config struct {
 	Scheme              string
 	Seed                string
 	Host                string
 	Port                uint
+	PortHTTP            uint
 	CertificateLocation string
 	PrivateKeyLocation  string
 
@@ -65,11 +73,12 @@ func Load(args []string) (Config, error) {
 	fs.StringVar(&cfg.fsRoot, "filesystem-root", "/storage", "filesystem root (required for the filesystem backend). folder will be created if it doesn't exist")
 	fs.StringVar(&cfg.publicHost, "public-host", "storage.googleapis.com", "Optional URL for public host")
 	fs.StringVar(&cfg.externalURL, "external-url", "", "optional external URL, returned in the Location header for uploads. Defaults to the address where the server is running")
-	fs.StringVar(&cfg.Scheme, "scheme", "https", "using http or https")
+	fs.StringVar(&cfg.Scheme, "scheme", schemeHTTPS, "using 'http' or 'https' or 'both'")
 	fs.StringVar(&cfg.Host, "host", "0.0.0.0", "host to bind to")
 	fs.StringVar(&cfg.Seed, "data", "", "where to load data from (provided that the directory exists)")
 	fs.StringVar(&allowedCORSHeaders, "cors-headers", "", "comma separated list of headers to add to the CORS allowlist")
-	fs.UintVar(&cfg.Port, "port", 4443, "port to bind to")
+	fs.UintVar(&cfg.Port, flagPort, 0, "port to which https (default 4443) or http (default 8000) will be bound, based on the specified scheme. If the scheme is 'both', then bind to https")
+	fs.UintVar(&cfg.PortHTTP, flagPortHTTP, 0, "used only when scheme is 'both' as the port to bind http to (default 8000)")
 	fs.StringVar(&cfg.event.pubsubProjectID, "event.pubsub-project-id", "", "project ID containing the pubsub topic")
 	fs.StringVar(&cfg.event.pubsubTopic, "event.pubsub-topic", "", "pubsub topic name to publish events on")
 	fs.StringVar(&cfg.event.bucket, "event.bucket", "", "if not empty, only objects in this bucket will generate trigger events")
@@ -85,6 +94,27 @@ func Load(args []string) (Config, error) {
 		return cfg, err
 	}
 
+	// Create a map to store the flags and their values
+	setFlags := make(map[string]interface{})
+
+	// Check if a flag was used using Visit
+	fs.Visit(func(f *flag.Flag) {
+		setFlags[f.Name] = f.Value
+	})
+
+	// setting default values, if not provided, for port and http ports based on scheme value
+	if _, ok := setFlags[flagPort]; !ok {
+		if cfg.Scheme == schemeHTTPS || cfg.Scheme == schemeBoth {
+			cfg.Port = defaultHTTPSPort
+		} else if cfg.Scheme == schemeHTTP {
+			cfg.Port = defaultHTTPPort
+		}
+	}
+
+	if _, ok := setFlags[flagPortHTTP]; !ok && cfg.Scheme == schemeBoth {
+		cfg.PortHTTP = defaultHTTPPort
+	}
+
 	cfg.LogLevel, err = logrus.ParseLevel(logLevel)
 	if err != nil {
 		return cfg, err
@@ -98,7 +128,12 @@ func Load(args []string) (Config, error) {
 	}
 
 	if cfg.externalURL == "" {
-		cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port)
+		if cfg.Scheme != "both" {
+			cfg.externalURL = fmt.Sprintf("%s://%s:%d", cfg.Scheme, cfg.Host, cfg.Port)
+		} else {
+			// for scheme 'both' taking externalURL as HTTPs by default
+			cfg.externalURL = fmt.Sprintf("%s://%s:%d", schemeHTTPS, cfg.Host, cfg.Port)
+		}
 	}
 
 	return cfg, cfg.validate()
@@ -111,12 +146,15 @@ func (c *Config) validate() error {
 	if c.backend == filesystemBackend && c.fsRoot == "" {
 		return fmt.Errorf("backend %q requires the filesystem-root to be defined", c.backend)
 	}
-	if c.Scheme != "http" && c.Scheme != "https" {
-		return fmt.Errorf(`invalid scheme %s, must be either "http"" or "https"`, c.Scheme)
+	if c.Scheme != schemeHTTP && c.Scheme != schemeHTTPS && c.Scheme != schemeBoth {
+		return fmt.Errorf(`invalid scheme %s, must be either "%s", "%s" or "%s"`, c.Scheme, schemeHTTP, schemeHTTPS, schemeBoth)
 	}
 	if c.Port > math.MaxUint16 {
 		return fmt.Errorf("port %d is too high, maximum value is %d", c.Port, math.MaxUint16)
 	}
+	if c.PortHTTP > math.MaxUint16 {
+		return fmt.Errorf("port-http %d is too high, maximum value is %d", c.PortHTTP, math.MaxUint16)
+	}
 
 	return c.event.validate()
 }
@@ -148,7 +186,7 @@ func (c *EventConfig) validate() error {
 	return nil
 }
 
-func (c *Config) ToFakeGcsOptions() fakestorage.Options {
+func (c *Config) ToFakeGcsOptions(scheme string) fakestorage.Options {
 	storageRoot := c.fsRoot
 	if c.backend == memoryBackend {
 		storageRoot = ""
@@ -173,14 +211,18 @@ func (c *Config) ToFakeGcsOptions() fakestorage.Options {
 			}
 		}
 	}
+	port := c.Port
+	if c.Scheme == schemeBoth && scheme == schemeHTTP {
+		port = c.PortHTTP // this cli flag, for port http, is relevant only when scheme is both
+	}
 	logger := logrus.New()
 	logger.SetLevel(c.LogLevel)
 	opts := fakestorage.Options{
 		StorageRoot:         storageRoot,
 		Seed:                c.Seed,
-		Scheme:              c.Scheme,
+		Scheme:              scheme,
 		Host:                c.Host,
-		Port:                uint16(c.Port),
+		Port:                uint16(port),
 		PublicHost:          c.publicHost,
 		ExternalURL:         c.externalURL,
 		AllowedCORSHeaders:  c.allowedCORSHeaders,