From 0db894a85cdd6358bfa33b0df8a6497bc30bd06b Mon Sep 17 00:00:00 2001 From: francisco souza <108725+fsouza@users.noreply.github.com> Date: Mon, 31 Oct 2022 21:36:37 -0400 Subject: [PATCH] Add support for Windows back It's limited: can't atomically rename things, but otherwise it should work! --- .github/workflows/main.yml | 1 + ci/.goreleaser.yml | 5 ++++ go.mod | 1 + go.sum | 2 ++ internal/backend/backend_test.go | 4 +++ internal/backend/fs.go | 35 ++++----------------------- internal/backend/writefile_unix.go | 17 +++++++++++++ internal/backend/writefile_windows.go | 13 ++++++++++ 8 files changed, 48 insertions(+), 30 deletions(-) create mode 100644 internal/backend/writefile_unix.go create mode 100644 internal/backend/writefile_windows.go diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 30d39a15c0..87ee027bab 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -18,6 +18,7 @@ jobs: os: - macos - ubuntu + - windows arch: - 386 - amd64 diff --git a/ci/.goreleaser.yml b/ci/.goreleaser.yml index 60c86ca6b7..17a8d26e73 100644 --- a/ci/.goreleaser.yml +++ b/ci/.goreleaser.yml @@ -9,10 +9,15 @@ builds: goos: - darwin - linux + - windows + ignore: + - goos: windows + goarch: arm64 archives: - replacements: darwin: Darwin linux: Linux + windows: Windows files: - LICENSE - README.md diff --git a/go.mod b/go.mod index 2a8dc6c115..4826d7d396 100644 --- a/go.mod +++ b/go.mod @@ -21,6 +21,7 @@ require ( github.com/felixge/httpsnoop v1.0.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect + github.com/google/renameio/v2 v2.0.0 github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.6.0 // indirect diff --git a/go.sum b/go.sum index 3f741347f6..d30ec26ff8 100644 --- a/go.sum +++ b/go.sum @@ -52,6 +52,8 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/renameio/v2 v2.0.0 h1:UifI23ZTGY8Tt29JbYFiuyIU3eX+RNFtUwefq9qAhxg= +github.com/google/renameio/v2 v2.0.0/go.mod h1:BtmJXm5YlszgC+TD4HOEEUFgkJP3nLxehU6hfe7jRt4= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= diff --git a/internal/backend/backend_test.go b/internal/backend/backend_test.go index 3700b401ad..a27f39ac35 100644 --- a/internal/backend/backend_test.go +++ b/internal/backend/backend_test.go @@ -104,6 +104,10 @@ func uploadAndCompare(t *testing.T, storage Storage, obj Object) int64 { } func TestObjectCRUD(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("time resolution on Windows makes this test flaky on that platform") + } + const bucketName = "prod-bucket" const objectName = "video/hi-res/best_video_1080p.mp4" content1 := []byte("content1") diff --git a/internal/backend/fs.go b/internal/backend/fs.go index d1a3d6e35a..ebe934af95 100644 --- a/internal/backend/fs.go +++ b/internal/backend/fs.go @@ -5,6 +5,7 @@ package backend import ( + "bytes" "encoding/json" "errors" "fmt" @@ -179,32 +180,11 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St path := filepath.Join(s.rootDir, url.PathEscape(obj.BucketName), url.PathEscape(obj.Name)) - tempFile, err := os.CreateTemp(filepath.Dir(path), "fake-gcs-object") - if err != nil { - return StreamingObject{}, err - } - tempFile.Close() - - tempFile, err = os.OpenFile(tempFile.Name(), os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o666) - if err != nil { - return StreamingObject{}, err - } - defer tempFile.Close() - - // The file is renamed below, which causes this to be a no-op. If the - // function returns before the rename, though, the temp file will be - // removed. - defer os.Remove(tempFile.Name()) - - err = os.Chmod(tempFile.Name(), 0o600) - if err != nil { - return StreamingObject{}, err - } - + var buf bytes.Buffer hasher := checksum.NewStreamingHasher() objectContent := io.TeeReader(obj.Content, hasher) - if _, err = io.Copy(tempFile, objectContent); err != nil { + if _, err = io.Copy(&buf, objectContent); err != nil { return StreamingObject{}, err } @@ -218,16 +198,11 @@ func (s *storageFS) CreateObject(obj StreamingObject, conditions Conditions) (St return StreamingObject{}, err } - if err = s.mh.write(tempFile.Name(), encoded); err != nil { - return StreamingObject{}, err - } - - err = os.Rename(tempFile.Name(), path) - if err != nil { + if err := writeFile(path, buf.Bytes(), 0o600); err != nil { return StreamingObject{}, err } - if err = s.mh.rename(tempFile.Name(), path); err != nil { + if err = s.mh.write(path, encoded); err != nil { return StreamingObject{}, err } diff --git a/internal/backend/writefile_unix.go b/internal/backend/writefile_unix.go new file mode 100644 index 0000000000..2e5e510fbc --- /dev/null +++ b/internal/backend/writefile_unix.go @@ -0,0 +1,17 @@ +// Copyright 2022 Francisco Souza. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build !windows + +package backend + +import ( + "os" + + "github.com/google/renameio/v2" +) + +func writeFile(filename string, data []byte, perm os.FileMode) error { + return renameio.WriteFile(filename, data, perm) +} diff --git a/internal/backend/writefile_windows.go b/internal/backend/writefile_windows.go new file mode 100644 index 0000000000..2d6600c803 --- /dev/null +++ b/internal/backend/writefile_windows.go @@ -0,0 +1,13 @@ +// Copyright 2022 Francisco Souza. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package backend + +import ( + "os" +) + +func writeFile(filename string, data []byte, perm os.FileMode) error { + return os.WriteFile(filename, data, perm) +}