From 961c04cae992eadb42d286d2f85f8a675bdc68c8 Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 29 Jan 2018 14:17:40 -0800 Subject: Upgrading server dependancies (#8154) --- .../pkg/s3signer/request-signature-streaming.go | 7 +-- .../s3signer/request-signature-streaming_test.go | 11 ++-- .../minio-go/pkg/s3signer/request-signature-v2.go | 65 +++++++++++----------- .../pkg/s3signer/request-signature-v2_test.go | 3 +- .../minio-go/pkg/s3signer/request-signature-v4.go | 5 +- .../pkg/s3signer/request-signature-v4_test.go | 50 +++++++++++++++++ .../pkg/s3signer/request-signature_test.go | 3 +- .../minio/minio-go/pkg/s3signer/test-utils_test.go | 3 +- .../minio/minio-go/pkg/s3signer/utils.go | 12 +++- .../minio/minio-go/pkg/s3signer/utils_test.go | 6 +- 10 files changed, 116 insertions(+), 49 deletions(-) create mode 100644 vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4_test.go (limited to 'vendor/github.com/minio/minio-go/pkg/s3signer') diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go index d831436cd..156a6d63a 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,7 +33,6 @@ import ( // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming const ( streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" - streamingEncoding = "aws-chunked" streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD" emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" payloadChunkSize = 64 * 1024 @@ -99,9 +99,8 @@ func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int if sessionToken != "" { req.Header.Set("X-Amz-Security-Token", sessionToken) } - req.Header.Add("Content-Encoding", streamingEncoding) - req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) + req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) // Set content length with streaming signature for each chunk included. req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize)) req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10)) diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go index 1f49f2234..297ab97be 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -32,7 +33,7 @@ func TestGetSeedSignature(t *testing.T) { req := NewRequest("PUT", "/examplebucket/chunkObject.txt", body) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") - req.URL.Host = "s3.amazonaws.com" + req.Host = "s3.amazonaws.com" reqTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") if err != nil { @@ -42,7 +43,7 @@ func TestGetSeedSignature(t *testing.T) { req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime) actualSeedSignature := req.Body.(*StreamingReader).seedSignature - expectedSeedSignature := "007480502de61457e955731b0f5d191f7e6f54a8a0f6cc7974a5ebd887965686" + expectedSeedSignature := "38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" if actualSeedSignature != expectedSeedSignature { t.Errorf("Expected %s but received %s", expectedSeedSignature, actualSeedSignature) } @@ -68,13 +69,14 @@ func TestSetStreamingAuthorization(t *testing.T) { req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.Host = "" req.URL.Host = "s3.amazonaws.com" dataLen := int64(65 * 1024) reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) - expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=007480502de61457e955731b0f5d191f7e6f54a8a0f6cc7974a5ebd887965686" + expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" actualAuthorization := req.Header.Get("Authorization") if actualAuthorization != expectedAuthorization { @@ -92,6 +94,7 @@ func TestStreamingReader(t *testing.T) { req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") req.ContentLength = 65 * 1024 + req.Host = "" req.URL.Host = "s3.amazonaws.com" baseReader := ioutil.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024))) diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go index 39c4e0187..0b90c41f6 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -39,22 +40,23 @@ const ( ) // Encode input URL path to URL encoded path. -func encodeURL2Path(u *url.URL) (path string) { +func encodeURL2Path(req *http.Request) (path string) { + reqHost := getHostAddr(req) // Encode URL path. - if isS3, _ := filepath.Match("*.s3*.amazonaws.com", u.Host); isS3 { - bucketName := u.Host[:strings.LastIndex(u.Host, ".s3")] + if isS3, _ := filepath.Match("*.s3*.amazonaws.com", reqHost); isS3 { + bucketName := reqHost[:strings.LastIndex(reqHost, ".s3")] path = "/" + bucketName - path += u.Path + path += req.URL.Path path = s3utils.EncodePath(path) return } - if strings.HasSuffix(u.Host, ".storage.googleapis.com") { - path = "/" + strings.TrimSuffix(u.Host, ".storage.googleapis.com") - path += u.Path + if strings.HasSuffix(reqHost, ".storage.googleapis.com") { + path = "/" + strings.TrimSuffix(reqHost, ".storage.googleapis.com") + path += req.URL.Path path = s3utils.EncodePath(path) return } - path = s3utils.EncodePath(u.Path) + path = s3utils.EncodePath(req.URL.Path) return } @@ -76,7 +78,7 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in } // Get presigned string to sign. - stringToSign := preStringifyHTTPReq(req) + stringToSign := preStringToSignV2(req) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) @@ -85,7 +87,7 @@ func PreSignV2(req http.Request, accessKeyID, secretAccessKey string, expires in query := req.URL.Query() // Handle specially for Google Cloud Storage. - if strings.Contains(req.URL.Host, ".storage.googleapis.com") { + if strings.Contains(getHostAddr(&req), ".storage.googleapis.com") { query.Set("GoogleAccessId", accessKeyID) } else { query.Set("AWSAccessKeyId", accessKeyID) @@ -145,7 +147,7 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request } // Calculate HMAC for secretAccessKey. - stringToSign := stringifyHTTPReq(req) + stringToSign := stringToSignV2(req) hm := hmac.New(sha1.New, []byte(secretAccessKey)) hm.Write([]byte(stringToSign)) @@ -170,15 +172,14 @@ func SignV2(req http.Request, accessKeyID, secretAccessKey string) *http.Request // Expires + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; -func preStringifyHTTPReq(req http.Request) string { +func preStringToSignV2(req http.Request) string { buf := new(bytes.Buffer) // Write standard headers. writePreSignV2Headers(buf, req) // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) // Write canonicalized Query resources if any. - isPreSign := true - writeCanonicalizedResource(buf, req, isPreSign) + writeCanonicalizedResource(buf, req) return buf.String() } @@ -198,15 +199,14 @@ func writePreSignV2Headers(buf *bytes.Buffer, req http.Request) { // Date + "\n" + // CanonicalizedProtocolHeaders + // CanonicalizedResource; -func stringifyHTTPReq(req http.Request) string { +func stringToSignV2(req http.Request) string { buf := new(bytes.Buffer) // Write standard headers. writeSignV2Headers(buf, req) // Write canonicalized protocol headers if any. writeCanonicalizedHeaders(buf, req) // Write canonicalized Query resources if any. - isPreSign := false - writeCanonicalizedResource(buf, req, isPreSign) + writeCanonicalizedResource(buf, req) return buf.String() } @@ -253,17 +253,27 @@ func writeCanonicalizedHeaders(buf *bytes.Buffer, req http.Request) { } } -// The following list is already sorted and should always be, otherwise we could -// have signature-related issues +// AWS S3 Signature V2 calculation rule is give here: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign + +// Whitelist resource list that will be used in query string for signature-V2 calculation. +// The list should be alphabetically sorted var resourceList = []string{ "acl", "delete", + "lifecycle", "location", "logging", "notification", "partNumber", "policy", "requestPayment", + "response-cache-control", + "response-content-disposition", + "response-content-encoding", + "response-content-language", + "response-content-type", + "response-expires", "torrent", "uploadId", "uploads", @@ -278,22 +288,11 @@ var resourceList = []string{ // CanonicalizedResource = [ "/" + Bucket ] + // + // [ sub-resource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; -func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, isPreSign bool) { +func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request) { // Save request URL. requestURL := req.URL // Get encoded URL path. - path := encodeURL2Path(requestURL) - if isPreSign { - // Get encoded URL path. - if len(requestURL.Query()) > 0 { - // Keep the usual queries unescaped for string to sign. - query, _ := url.QueryUnescape(s3utils.QueryEncode(requestURL.Query())) - path = path + "?" + query - } - buf.WriteString(path) - return - } - buf.WriteString(path) + buf.WriteString(encodeURL2Path(&req)) if requestURL.RawQuery != "" { var n int vals, _ := url.ParseQuery(requestURL.RawQuery) diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2_test.go index 3c0e0ecea..042b6e65c 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2_test.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go index 0d75dc162..daf02fedf 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -143,7 +144,7 @@ func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) strin buf.WriteByte(':') switch { case k == "host": - buf.WriteString(req.URL.Host) + buf.WriteString(getHostAddr(&req)) fallthrough default: for idx, v := range vals[k] { diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4_test.go new file mode 100644 index 000000000..a109a4f2a --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4_test.go @@ -0,0 +1,50 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "io" + "net/http" + "strings" + "testing" +) + +func TestRequestHost(t *testing.T) { + req, _ := buildRequest("dynamodb", "us-east-1", "{}") + req.URL.RawQuery = "Foo=z&Foo=o&Foo=m&Foo=a" + req.Host = "myhost" + canonicalHeaders := getCanonicalHeaders(*req, v4IgnoredHeaders) + + if !strings.Contains(canonicalHeaders, "host:"+req.Host) { + t.Errorf("canonical host header invalid") + } +} + +func buildRequest(serviceName, region, body string) (*http.Request, io.ReadSeeker) { + endpoint := "https://" + serviceName + "." + region + ".amazonaws.com" + reader := strings.NewReader(body) + req, _ := http.NewRequest("POST", endpoint, reader) + req.URL.Opaque = "//example.org/bucket/key-._~,!@#$%^&*()" + req.Header.Add("X-Amz-Target", "prefix.Operation") + req.Header.Add("Content-Type", "application/x-amz-json-1.0") + req.Header.Add("Content-Length", string(len(body))) + req.Header.Add("X-Amz-Meta-Other-Header", "some-value=!@#$%^&* (+)") + req.Header.Add("X-Amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") + req.Header.Add("X-amz-Meta-Other-Header_With_Underscore", "some-value=!@#$%^&* (+)") + return req, reader +} diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go index 85ff063df..d53483e4e 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go index 049e5813d..cf96d66c8 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/utils.go b/vendor/github.com/minio/minio-go/pkg/s3signer/utils.go index 0619b3082..33b175208 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/utils.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/utils.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,6 +20,7 @@ package s3signer import ( "crypto/hmac" "crypto/sha256" + "net/http" ) // unsignedPayload - value to be set to X-Amz-Content-Sha256 header when @@ -37,3 +39,11 @@ func sumHMAC(key []byte, data []byte) []byte { hash.Write(data) return hash.Sum(nil) } + +// getHostAddr returns host header if available, otherwise returns host from URL +func getHostAddr(req *http.Request) string { + if req.Host != "" { + return req.Host + } + return req.URL.Host +} diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/utils_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/utils_test.go index 26f609013..407eddab3 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/utils_test.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/utils_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,6 +19,7 @@ package s3signer import ( "fmt" + "net/http" "net/url" "testing" ) @@ -65,7 +67,7 @@ func TestEncodeURL2Path(t *testing.T) { t.Fatal("Error:", err) } urlPath := "/" + bucketName + "/" + o.encodedObjName - if urlPath != encodeURL2Path(u) { + if urlPath != encodeURL2Path(&http.Request{URL: u}) { t.Fatal("Error") } } -- cgit v1.2.3-1-g7c22