Merge pull request #2 from mathis-marcotte/remove-multipart-and-fatal

Chore: Bump repo to latest forked commit

.travis.yml

@@ -9,7 +9,7 @@ install:
 - make s3proxy.jar
 script: travis_retry ./test/run-tests.sh GoofysTest
 go:
-  - 1.11.5
+  - 1.16
 go_import_path: github.com/kahing/goofys
 matrix:
   include:

Makefile

@@ -4,7 +4,7 @@ run-test: s3proxy.jar
 	./test/run-tests.sh
 
 s3proxy.jar:
-	wget https://github.com/gaul/s3proxy/releases/download/s3proxy-1.7.0/s3proxy -O s3proxy.jar
+	wget https://github.com/gaul/s3proxy/releases/download/s3proxy-1.8.0/s3proxy -O s3proxy.jar
 
 get-deps: s3proxy.jar
 	go get -t ./...

README-gcs.md

@@ -0,0 +1,29 @@
+# Google Cloud Storage (GCS)
+
+
+## Prerequisite
+
+Service Account credentials or user authentication. Ensure that either the service account or user has the proper permissions to the Bucket / Object under GCS.
+
+To have a successful mount, we require users to have object listing (`storage.objects.list`) permission to the bucket.
+
+### Service Account credentials
+
+Create a service account credentials (https://cloud.google.com/iam/docs/creating-managing-service-accounts) and generate the JSON credentials file.
+
+### User Authentication and `gcloud` Default Authentication
+User can authenticate to gcloud's default environment by first installing cloud sdk (https://cloud.google.com/sdk/) and running `gcloud auth application-default login` command.
+
+
+## Using Goofys for GCS
+
+### With service account credentials file
+```
+GOOGLE_APPLICATION_CREDENTIALS="/path/to/creds.json" goofys gs://[BUCKET] /path/to/mount
+```
+
+### With user authentication (`gcloud auth application-default login`)
+
+```
+goofys gs://[BUCKET] [MOUNT DIRECTORY]
+```

README.md

@@ -3,6 +3,7 @@
 [![Build Status](https://travis-ci.org/kahing/goofys.svg?branch=master)](https://travis-ci.org/kahing/goofys)
 [![Github All Releases](https://img.shields.io/github/downloads/kahing/goofys/total.svg)](https://github.com/kahing/goofys/releases/)
 [![Twitter Follow](https://img.shields.io/twitter/follow/s3goofys.svg?style=social&label=Follow)](https://twitter.com/s3goofys)
+[![Stack Overflow Questions](https://img.shields.io/stackexchange/stackoverflow/t/goofys?label=Stack%20Overflow%20questions)](https://stackoverflow.com/search?q=%5Bgoofys%5D+is%3Aquestion)
 
 # Overview
 
@@ -18,7 +19,8 @@ close-to-open.
 
 # Installation
 
-* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). You may also need to install fuse-utils first.
+* On Linux, install via [pre-built binaries](https://github.com/kahing/goofys/releases/latest/download/goofys). 
+You may also need to install fuse too if you want to mount it on startup.
 
 * On macOS, install via [Homebrew](https://brew.sh/):
 
@@ -100,9 +102,6 @@ List of non-POSIX behaviors/limitations:
   * `unlink` returns success even if file is not present
   * `fsync` is ignored, files are only flushed on `close`
 
-In addition to the items above, the following are supportable but not yet implemented:
-  * creating files larger than 1TB
-
 ## Compatibility with non-AWS S3
 
 goofys has been tested with the following non-AWS S3 providers:
@@ -115,6 +114,7 @@ goofys has been tested with the following non-AWS S3 providers:
 * Minio (limited)
 * OpenStack Swift
 * S3Proxy
+* Scaleway
 * Wasabi
 
 Additionally, goofys also works with the following non-S3 object stores:

api/api.go

@@ -25,9 +25,11 @@ func Mount(
 	if flags.DebugS3 {
 		SetCloudLogLevel(logrus.DebugLevel)
 	}
+
 	// Mount the file system.
 	mountCfg := &fuse.MountConfig{
 		FSName:                  bucketName,
+		Subtype:                 "goofys",
 		Options:                 flags.MountOptions,
 		ErrorLogger:             GetStdLogger(NewLogger("fuse"), logrus.ErrorLevel),
 		DisableWritebackCaching: true,
@@ -106,6 +108,10 @@ func Mount(
 				if spec.Prefix != "" {
 					bucketName += ":" + spec.Prefix
 				}
+			case "gs":
+				config := NewGCSConfig()
+				bucketName = spec.Bucket
+				flags.Backend = config
 			}
 		}
 	}

api/common/conf_azure.go

@@ -269,7 +269,7 @@ func azureFindAccount(client azblob.AccountsClient, account string) (*azblob.End
 		return nil, "", err
 	}
 
-	for _, acc := range *accountsRes.Value {
+	for _, acc := range accountsRes.Values() {
 		if *acc.Name == account {
 			// /subscriptions/{subscriptionId}/resourceGroups/{resourceGroupName}/...
 			parts := strings.SplitN(*acc.ID, "/", 6)
@@ -370,7 +370,7 @@ func AzureBlobConfig(endpoint string, location string, storageType string) (conf
 
 			if key == "" {
 				var keysRes azblob.AccountListKeysResult
-				keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account)
+				keysRes, err = client.ListKeys(context.TODO(), resourceGroup, account, azblob.Kerb)
 				if err != nil || len(*keysRes.Keys) == 0 {
 					err = fmt.Errorf("Missing key: configure via AZURE_STORAGE_KEY "+
 						"or %v/config", configDir)

api/common/conf_gcs.go

@@ -0,0 +1,35 @@
+package common
+
+import (
+	"context"
+	"golang.org/x/oauth2/google"
+)
+
+type GCSConfig struct {
+	Credentials *google.Credentials
+	ChunkSize   int // determine the size of chunks when uploading a file to GCS
+}
+
+const (
+	defaultGCSChunkSize int = 64 * 1024 * 1024
+)
+
+// NewGCSConfig returns a GCS Config.
+func NewGCSConfig() *GCSConfig {
+	// Credentials will be checked when initializing GCS bucket to create an authenticated / unauthenticated client
+	// We allow nil credentials for unauthenticated client.
+	credentials, err := google.FindDefaultCredentials(context.Background())
+	if err == nil {
+		// authenticated config
+		return &GCSConfig{
+			ChunkSize:   defaultGCSChunkSize,
+			Credentials: credentials,
+		}
+	} else {
+		log.Debugf("Initializing an unauthenticated config: %v", err)
+		// unauthenticated config
+		return &GCSConfig{
+			ChunkSize: defaultGCSChunkSize,
+		}
+	}
+}

api/common/logger.go

@@ -119,14 +119,14 @@ func GetLogger(name string) *LogHandle {
 
 	if logger, ok := loggers[name]; ok {
 		if name != "main" && name != "fuse" {
-			logger.Level = cloudLogLevel
+			logger.SetLevel(cloudLogLevel)
 		}
 		return logger
 	} else {
 		logger := NewLogger(name)
 		loggers[name] = logger
 		if name != "main" && name != "fuse" {
-			logger.Level = cloudLogLevel
+			logger.SetLevel(cloudLogLevel)
 		}
 		return logger
 	}

api/common/panic_logger.go

@@ -27,6 +27,8 @@ type FusePanicLogger struct {
 	Fs fuseutil.FileSystem
 }
 
+var _ fuseutil.FileSystem = FusePanicLogger{}
+
 func LogPanic(err *error) {
 	if e := recover(); e != nil {
 		log.Errorf("stacktrace from panic: %v \n"+string(debug.Stack()), e)
@@ -36,6 +38,10 @@ func LogPanic(err *error) {
 	}
 }
 
+func (fs FusePanicLogger) BatchForget(ctx context.Context, op *fuseops.BatchForgetOp) (err error) {
+	defer LogPanic(&err)
+	return fs.Fs.BatchForget(ctx, op)
+}
 func (fs FusePanicLogger) StatFS(ctx context.Context, op *fuseops.StatFSOp) (err error) {
 	defer LogPanic(&err)
 	return fs.Fs.StatFS(ctx, op)

bench/Dockerfile.gcs

@@ -0,0 +1,44 @@
+FROM golang:1.14 AS goofys-builder
+
+# install goofys
+WORKDIR $GOPATH/src/github.com/kahing/goofys
+
+COPY . .
+
+RUN git init
+RUN git submodule update --init --recursive
+RUN go build
+
+# install gcsfuse, the binary is added to /go/bin/gcsfuse
+RUN go get -u github.com/googlecloudplatform/gcsfuse
+
+FROM ubuntu:18.04
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && \
+    apt-get -y install --no-install-recommends \
+            # gcsfuse dependencies \
+            fuse \
+            # for running goofys benchmark \
+            curl python-setuptools python-pip gnuplot-nox imagemagick awscli \
+            # finally, clean up to make image smaller \
+            && apt-get clean
+
+# install catfs, required to run goofys with cache
+RUN curl -L -O https://github.com/kahing/catfs/releases/download/v0.8.0/catfs && \
+    mv catfs /usr/bin && chmod 0755 /usr/bin/catfs
+
+# goofys graph generation
+RUN pip install numpy
+
+ENV PATH=$PATH:/root/go/bin
+
+# copy go binaries
+COPY --from=goofys-builder /go/src/github.com/kahing/goofys/goofys /root/go/bin/goofys
+COPY --from=goofys-builder /go/bin/gcsfuse /root/go/bin/gcsfuse
+
+WORKDIR /root/go/src/github.com/kahing/goofys
+
+# copy bench scripts
+COPY bench bench
+
+ENTRYPOINT ["/root/go/src/github.com/kahing/goofys/bench/run_bench.sh"]