Skip to content

Commit

Permalink
Move checksum functions to internal package
Browse files Browse the repository at this point in the history 
Don't want that as part of our public API.
  • Loading branch information
@fsouza
fsouza committed Jun 3, 2021
1 parent 0a8243d commit 2a2f94f
Show file tree
Hide file tree
Showing 6 changed files with 99 additions and 88 deletions.
65 changes: 33 additions & 32 deletions fakestorage/object_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"time"

"cloud.google.com/go/storage"
"github.com/fsouza/fake-gcs-server/internal/checksum"
"google.golang.org/api/iterator"
)

Expand All @@ -28,7 +29,7 @@ func uint32ToBytes(ui uint32) []byte {
}

func uint32Checksum(b []byte) uint32 {
checksummer := crc32.New(crc32cTable)
checksummer := crc32.New(crc32.MakeTable(crc32.Castagnoli))
checksummer.Write(b)
return checksummer.Sum32()
}
Expand All @@ -47,25 +48,25 @@ func getObjectTestCases() objectTestCases {
metaValue = "MetaValue"
)
testInitExecTime := time.Now().Truncate(time.Microsecond)
checksum := uint32Checksum([]byte(content))
hash := md5Hash([]byte(content))
u32Checksum := uint32Checksum([]byte(content))
hash := checksum.MD5Hash([]byte(content))

tests := objectTestCases{
{
"object but no creation nor modification date",
Object{BucketName: bucketName, Name: "img/low-res/party-01.jpg", Content: []byte(content), ContentType: contentType, ContentEncoding: contentEncoding, Crc32c: encodedChecksum(uint32ToBytes(checksum)), Md5Hash: encodedHash(hash)},
Object{BucketName: bucketName, Name: "img/low-res/party-01.jpg", Content: []byte(content), ContentType: contentType, ContentEncoding: contentEncoding, Crc32c: checksum.EncodedChecksum(uint32ToBytes(u32Checksum)), Md5Hash: checksum.EncodedHash(hash)},
},
{
"object with creation and modification dates",
Object{BucketName: bucketName, Name: "img/low-res/party-02.jpg", Content: []byte(content), ContentType: contentType, ContentEncoding: contentEncoding, Crc32c: encodedChecksum(uint32ToBytes(checksum)), Md5Hash: encodedHash(hash), Created: testInitExecTime, Updated: testInitExecTime},
Object{BucketName: bucketName, Name: "img/low-res/party-02.jpg", Content: []byte(content), ContentType: contentType, ContentEncoding: contentEncoding, Crc32c: checksum.EncodedChecksum(uint32ToBytes(u32Checksum)), Md5Hash: checksum.EncodedHash(hash), Created: testInitExecTime, Updated: testInitExecTime},
},
{
"object with creation, modification dates and generation",
Object{BucketName: bucketName, Name: "img/low-res/party-02.jpg", Content: []byte(content), ContentType: contentType, Crc32c: encodedChecksum(uint32ToBytes(checksum)), Md5Hash: encodedHash(hash), Created: testInitExecTime, Updated: testInitExecTime, Generation: testInitExecTime.UnixNano()},
Object{BucketName: bucketName, Name: "img/low-res/party-02.jpg", Content: []byte(content), ContentType: contentType, Crc32c: checksum.EncodedChecksum(uint32ToBytes(u32Checksum)), Md5Hash: checksum.EncodedHash(hash), Created: testInitExecTime, Updated: testInitExecTime, Generation: testInitExecTime.UnixNano()},
},
{
"object with everything",
Object{BucketName: bucketName, Name: "img/location/meta.jpg", Content: []byte(content), ContentType: contentType, ContentEncoding: contentEncoding, Crc32c: encodedChecksum(uint32ToBytes(checksum)), Md5Hash: encodedHash(hash), Metadata: map[string]string{"MetaHeader": metaValue}},
Object{BucketName: bucketName, Name: "img/location/meta.jpg", Content: []byte(content), ContentType: contentType, ContentEncoding: contentEncoding, Crc32c: checksum.EncodedChecksum(uint32ToBytes(u32Checksum)), Md5Hash: checksum.EncodedHash(hash), Metadata: map[string]string{"MetaHeader": metaValue}},
},
{
"object with no contents neither dates",
Expand Down Expand Up @@ -120,8 +121,8 @@ func checkObjectAttrs(testObj Object, attrs *storage.ObjectAttrs, t *testing.T)
if testObj.Content != nil && attrs.CRC32C != uint32Checksum(testObj.Content) {
t.Errorf("wrong checksum returned\nwant %d\ngot %d", uint32Checksum(testObj.Content), attrs.CRC32C)
}
if testObj.Content != nil && !bytes.Equal(attrs.MD5, md5Hash(testObj.Content)) {
t.Errorf("wrong hash returned\nwant %d\ngot %d", md5Hash(testObj.Content), attrs.MD5)
if testObj.Content != nil && !bytes.Equal(attrs.MD5, checksum.MD5Hash(testObj.Content)) {
t.Errorf("wrong hash returned\nwant %d\ngot %d", checksum.MD5Hash(testObj.Content), attrs.MD5)
}
if testObj.Metadata != nil {
if val, err := getMetadataHeaderFromAttrs(attrs, "MetaHeader"); err != nil || val != testObj.Metadata["MetaHeader"] {
Expand Down Expand Up @@ -175,7 +176,7 @@ func TestServerClientObjectAttrsAfterOverwriteWithVersioning(t *testing.T) {
metaValue = "MetaValue"
)
server.CreateBucketWithOpts(CreateBucketOpts{Name: bucketName, VersioningEnabled: true})
initialObj := Object{BucketName: bucketName, Name: "img/low-res/party-01.jpg", Content: []byte(content), ContentType: contentType, Crc32c: encodedChecksum(uint32ToBytes(uint32Checksum([]byte(content)))), Md5Hash: encodedHash(md5Hash([]byte(content))), Metadata: map[string]string{"MetaHeader": metaValue}}
initialObj := Object{BucketName: bucketName, Name: "img/low-res/party-01.jpg", Content: []byte(content), ContentType: contentType, Crc32c: checksum.EncodedChecksum(uint32ToBytes(uint32Checksum([]byte(content)))), Md5Hash: checksum.EncodedHash(checksum.MD5Hash([]byte(content))), Metadata: map[string]string{"MetaHeader": metaValue}}
server.CreateObject(initialObj)
client := server.Client()
objHandle := client.Bucket(bucketName).Object(initialObj.Name)
Expand All @@ -189,7 +190,7 @@ func TestServerClientObjectAttrsAfterOverwriteWithVersioning(t *testing.T) {
// sleep for at least 100ns or more, so the creation time will differ on all platforms.
time.Sleep(time.Microsecond)

latestObjVersion := Object{BucketName: bucketName, Name: "img/low-res/party-01.jpg", Content: []byte(content2), ContentType: contentType, Crc32c: encodedChecksum(uint32ToBytes(uint32Checksum([]byte(content2)))), Md5Hash: encodedHash(md5Hash([]byte(content2)))}
latestObjVersion := Object{BucketName: bucketName, Name: "img/low-res/party-01.jpg", Content: []byte(content2), ContentType: contentType, Crc32c: checksum.EncodedChecksum(uint32ToBytes(uint32Checksum([]byte(content2)))), Md5Hash: checksum.EncodedHash(checksum.MD5Hash([]byte(content2)))}
server.CreateObject(latestObjVersion)
objHandle = client.Bucket(bucketName).Object(latestObjVersion.Name)
latestAttrs, err := objHandle.Attrs(context.TODO())
Expand Down Expand Up @@ -906,16 +907,16 @@ func TestServiceClientRewriteObject(t *testing.T) {
content = "some content"
contentType = "text/plain; charset=utf-8"
)
checksum := uint32Checksum([]byte(content))
hash := md5Hash([]byte(content))
u32Checksum := uint32Checksum([]byte(content))
hash := checksum.MD5Hash([]byte(content))
objs := []Object{
{
BucketName: "first-bucket",
Name: "files/some-file.txt",
Content: []byte(content),
ContentType: contentType,
Crc32c: encodedChecksum(uint32ToBytes(checksum)),
Md5Hash: encodedHash(hash),
Crc32c: checksum.EncodedChecksum(uint32ToBytes(u32Checksum)),
Md5Hash: checksum.EncodedHash(hash),
Metadata: map[string]string{"foo": "bar"},
},
}
Expand All @@ -933,22 +934,22 @@ func TestServiceClientRewriteObject(t *testing.T) {
"same bucket same file",
"first-bucket",
"files/some-file.txt",
checksum,
encodedHash(hash),
u32Checksum,
checksum.EncodedHash(hash),
},
{
"same bucket",
"first-bucket",
"files/other-file.txt",
checksum,
encodedHash(hash),
u32Checksum,
checksum.EncodedHash(hash),
},
{
"different bucket",
"empty-bucket",
"some/interesting/file.txt",
checksum,
encodedHash(hash),
u32Checksum,
checksum.EncodedHash(hash),
},
}
for _, test := range tests {
Expand All @@ -973,8 +974,8 @@ func TestServiceClientRewriteObject(t *testing.T) {
if attrs.Size != int64(len(content)) {
t.Errorf("wrong size in copied object attrs\nwant %d\ngot %d", len(content), attrs.Size)
}
if attrs.CRC32C != checksum {
t.Errorf("wrong checksum in copied object attrs\nwant %d\ngot %d", checksum, attrs.CRC32C)
if attrs.CRC32C != u32Checksum {
t.Errorf("wrong checksum in copied object attrs\nwant %d\ngot %d", u32Checksum, attrs.CRC32C)
}
if attrs.ContentType != contentType {
t.Errorf("wrong content type\nwant %q\ngot %q", contentType, attrs.ContentType)
Expand All @@ -989,10 +990,10 @@ func TestServiceClientRewriteObject(t *testing.T) {
if string(obj.Content) != content {
t.Errorf("wrong content on object\nwant %q\ngot %q", content, string(obj.Content))
}
if expect := encodedChecksum(uint32ToBytes(checksum)); expect != obj.Crc32c {
if expect := checksum.EncodedChecksum(uint32ToBytes(u32Checksum)); expect != obj.Crc32c {
t.Errorf("wrong checksum on object\nwant %s\ngot %s", expect, obj.Crc32c)
}
if expect := encodedHash(hash); expect != obj.Md5Hash {
if expect := checksum.EncodedHash(hash); expect != obj.Md5Hash {
t.Errorf("wrong hash on object\nwant %s\ngot %s", expect, obj.Md5Hash)
}
if obj.ContentType != contentType {
Expand All @@ -1019,17 +1020,17 @@ func TestServiceClientRewriteObjectWithGenerations(t *testing.T) {
Name: "files/some-file.txt",
Content: []byte(overwrittenContent),
ContentType: contentType,
Crc32c: encodedChecksum(uint32ToBytes(uint32Checksum([]byte(overwrittenContent)))),
Md5Hash: encodedHash(md5Hash([]byte(overwrittenContent))),
Crc32c: checksum.EncodedChecksum(uint32ToBytes(uint32Checksum([]byte(overwrittenContent)))),
Md5Hash: checksum.EncodedHash(checksum.MD5Hash([]byte(overwrittenContent))),
Generation: overwrittenGeneration,
},
{
BucketName: "first-bucket",
Name: "files/some-file.txt",
Content: []byte(latestContent),
ContentType: contentType,
Crc32c: encodedChecksum(uint32ToBytes(uint32Checksum([]byte(latestContent)))),
Md5Hash: encodedHash(md5Hash([]byte(latestContent))),
Crc32c: checksum.EncodedChecksum(uint32ToBytes(uint32Checksum([]byte(latestContent)))),
Md5Hash: checksum.EncodedHash(checksum.MD5Hash([]byte(latestContent))),
},
}
tests := []struct {
Expand Down Expand Up @@ -1081,7 +1082,7 @@ func TestServiceClientRewriteObjectWithGenerations(t *testing.T) {
sourceObject := client.Bucket("first-bucket").Object("files/some-file.txt").Generation(overwrittenGeneration)
expectedContent := overwrittenContent
expectedChecksum := uint32Checksum([]byte(overwrittenContent))
expectedHash := md5Hash([]byte(overwrittenContent))
expectedHash := checksum.MD5Hash([]byte(overwrittenContent))
dstObject := client.Bucket(test.bucketName).Object(test.objectName)
copier := dstObject.CopierFrom(sourceObject)
copier.ContentType = contentType
Expand Down Expand Up @@ -1117,10 +1118,10 @@ func TestServiceClientRewriteObjectWithGenerations(t *testing.T) {
if string(obj.Content) != expectedContent {
t.Errorf("wrong content on object\nwant %q\ngot %q", expectedContent, string(obj.Content))
}
if expect := encodedChecksum(uint32ToBytes(expectedChecksum)); expect != obj.Crc32c {
if expect := checksum.EncodedChecksum(uint32ToBytes(expectedChecksum)); expect != obj.Crc32c {
t.Errorf("wrong checksum on object\nwant %s\ngot %s", expect, obj.Crc32c)
}
if expect := encodedHash(expectedHash); expect != obj.Md5Hash {
if expect := checksum.EncodedHash(expectedHash); expect != obj.Md5Hash {
t.Errorf("wrong hash on object\nwant %s\ngot %s", expect, obj.Md5Hash)
}
if obj.ContentType != contentType {
Expand Down
51 changes: 9 additions & 42 deletions fakestorage/upload.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,9 @@
package fakestorage

import (
"crypto/md5" // #nosec G501
"crypto/rand"
"encoding/base64"
"encoding/json"
"fmt"
"hash/crc32"
"io"
"io/ioutil"
"mime"
Expand All @@ -20,6 +17,7 @@ import (
"strings"

"cloud.google.com/go/storage"
"github.com/fsouza/fake-gcs-server/internal/checksum"
"github.com/gorilla/mux"
)

Expand Down Expand Up @@ -119,8 +117,8 @@ func (s *Server) simpleUpload(bucketName string, r *http.Request) jsonResponse {
Content: data,
ContentType: r.Header.Get(contentTypeHeader),
ContentEncoding: contentEncoding,
Crc32c: EncodedCrc32cChecksum(data),
Md5Hash: EncodedMd5Hash(data),
Crc32c: checksum.EncodedCrc32cChecksum(data),
Md5Hash: checksum.EncodedMd5Hash(data),
ACL: getObjectACL(predefinedACL),
}
obj, err = s.createObject(obj)
Expand Down Expand Up @@ -159,8 +157,8 @@ func (s *Server) signedUpload(bucketName string, r *http.Request) jsonResponse {
Content: data,
ContentType: r.Header.Get(contentTypeHeader),
ContentEncoding: contentEncoding,
Crc32c: EncodedCrc32cChecksum(data),
Md5Hash: EncodedMd5Hash(data),
Crc32c: checksum.EncodedCrc32cChecksum(data),
Md5Hash: checksum.EncodedMd5Hash(data),
ACL: getObjectACL(predefinedACL),
Metadata: metaData,
}
Expand Down Expand Up @@ -189,37 +187,6 @@ func getObjectACL(predefinedACL string) []storage.ACLRule {
}
}

var crc32cTable = crc32.MakeTable(crc32.Castagnoli)

func crc32cChecksum(content []byte) []byte {
checksummer := crc32.New(crc32cTable)
checksummer.Write(content)
return checksummer.Sum(make([]byte, 0, 4))
}

func encodedChecksum(checksum []byte) string {
return base64.StdEncoding.EncodeToString(checksum)
}

func EncodedCrc32cChecksum(content []byte) string {
return encodedChecksum(crc32cChecksum(content))
}

func md5Hash(b []byte) []byte {
/* #nosec G401 */
h := md5.New()
h.Write(b)
return h.Sum(nil)
}

func encodedHash(hash []byte) string {
return base64.StdEncoding.EncodeToString(hash)
}

func EncodedMd5Hash(content []byte) string {
return encodedHash(md5Hash(content))
}

func (s *Server) multipartUpload(bucketName string, r *http.Request) jsonResponse {
defer r.Body.Close()
_, params, err := mime.ParseMediaType(r.Header.Get(contentTypeHeader))
Expand Down Expand Up @@ -268,8 +235,8 @@ func (s *Server) multipartUpload(bucketName string, r *http.Request) jsonRespons
Content: content,
ContentType: contentType,
ContentEncoding: metadata.ContentEncoding,
Crc32c: EncodedCrc32cChecksum(content),
Md5Hash: EncodedMd5Hash(content),
Crc32c: checksum.EncodedCrc32cChecksum(content),
Md5Hash: checksum.EncodedMd5Hash(content),
ACL: getObjectACL(predefinedACL),
Metadata: metadata.Metadata,
}
Expand Down Expand Up @@ -364,8 +331,8 @@ func (s *Server) uploadFileContent(r *http.Request) jsonResponse {
commit := true
status := http.StatusOK
obj.Content = append(obj.Content, content...)
obj.Crc32c = EncodedCrc32cChecksum(obj.Content)
obj.Md5Hash = EncodedMd5Hash(obj.Content)
obj.Crc32c = checksum.EncodedCrc32cChecksum(obj.Content)
obj.Md5Hash = checksum.EncodedMd5Hash(obj.Content)
obj.ContentType = r.Header.Get(contentTypeHeader)
responseHeader := make(http.Header)
if contentRange := r.Header.Get("Content-Range"); contentRange != "" {
Expand Down
20 changes: 8 additions & 12 deletions fakestorage/upload_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,15 @@ import (
"testing"

"cloud.google.com/go/storage"
"github.com/fsouza/fake-gcs-server/internal/checksum"
"google.golang.org/api/googleapi"
)

func TestServerClientObjectWriter(t *testing.T) {
const baseContent = "some nice content"
content := strings.Repeat(baseContent+"\n", googleapi.MinUploadChunkSize)
checksum := uint32Checksum([]byte(content))
hash := md5Hash([]byte(content))
u32Checksum := uint32Checksum([]byte(content))
hash := checksum.MD5Hash([]byte(content))

runServersTest(t, nil, func(t *testing.T, server *Server) {
tests := []struct {
Expand Down Expand Up @@ -80,16 +81,16 @@ func TestServerClientObjectWriter(t *testing.T) {
n, baseContent)
}

if returnedChecksum := w.Attrs().CRC32C; returnedChecksum != checksum {
t.Errorf("wrong writer.Attrs() checksum returned\nwant %d\ngot %d", checksum, returnedChecksum)
if returnedChecksum := w.Attrs().CRC32C; returnedChecksum != u32Checksum {
t.Errorf("wrong writer.Attrs() checksum returned\nwant %d\ngot %d", u32Checksum, returnedChecksum)
}
if base64Checksum := encodedChecksum(uint32ToBytes(checksum)); obj.Crc32c != base64Checksum {
if base64Checksum := checksum.EncodedChecksum(uint32ToBytes(u32Checksum)); obj.Crc32c != base64Checksum {
t.Errorf("wrong obj.Crc32c returned\nwant %s\ngot %s", base64Checksum, obj.Crc32c)
}
if returnedHash := w.Attrs().MD5; !bytes.Equal(returnedHash, hash) {
t.Errorf("wrong writer.Attrs() hash returned\nwant %d\ngot %d", hash, returnedHash)
}
if stringHash := encodedHash(hash); obj.Md5Hash != stringHash {
if stringHash := checksum.EncodedHash(hash); obj.Md5Hash != stringHash {
t.Errorf("wrong obj.Md5Hash returned\nwant %s\ngot %s", stringHash, obj.Md5Hash)
}
if obj.ContentType != contentType {
Expand Down Expand Up @@ -120,7 +121,7 @@ func TestServerClientObjectWriter(t *testing.T) {

func checkChecksum(t *testing.T, content []byte, obj Object) {
t.Helper()
if expect := EncodedCrc32cChecksum(content); expect != obj.Crc32c {
if expect := checksum.EncodedCrc32cChecksum(content); expect != obj.Crc32c {
t.Errorf("wrong checksum in the object\nwant %s\ngot %s", expect, obj.Crc32c)
}
}
Expand Down Expand Up @@ -242,7 +243,6 @@ func TestServerClientSimpleUpload(t *testing.T) {
req.Header.Set("Content-Type", contentType)
client := http.Client{
Transport: &http.Transport{
// #nosec
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
Expand Down Expand Up @@ -287,7 +287,6 @@ func TestServerClientSignedUpload(t *testing.T) {
req.Header.Set("X-Goog-Meta-Key", "Value")
client := http.Client{
Transport: &http.Transport{
// #nosec
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
Expand Down Expand Up @@ -379,7 +378,6 @@ func TestServerClientUploadWithPredefinedAclPublicRead(t *testing.T) {
req.Header.Set("Content-Type", contentType)
client := http.Client{
Transport: &http.Transport{
// #nosec
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
Expand Down Expand Up @@ -436,7 +434,6 @@ func TestServerClientSimpleUploadNoName(t *testing.T) {
}
client := http.Client{
Transport: &http.Transport{
// #nosec
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
Expand All @@ -462,7 +459,6 @@ func TestServerInvalidUploadType(t *testing.T) {
}
client := http.Client{
Transport: &http.Transport{
// #nosec
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
},
}
Expand Down
Loading

0 comments on commit 2a2f94f

Please sign in to comment.