diff options
Diffstat (limited to 'vendor/github.com/minio')
78 files changed, 5762 insertions, 617 deletions
diff --git a/vendor/github.com/minio/go-homedir/LICENSE b/vendor/github.com/minio/go-homedir/LICENSE new file mode 100644 index 000000000..f9c841a51 --- /dev/null +++ b/vendor/github.com/minio/go-homedir/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. diff --git a/vendor/github.com/minio/go-homedir/README.md b/vendor/github.com/minio/go-homedir/README.md new file mode 100644 index 000000000..085f57775 --- /dev/null +++ b/vendor/github.com/minio/go-homedir/README.md @@ -0,0 +1,16 @@ +# go-homedir + +This is a Go library for detecting the user's home directory without +the use of cgo, so the library can be used in cross-compilation environments. + +Usage is incredibly simple, just call `homedir.Dir()` to get the home directory +for a user, and `homedir.Expand()` to expand the `~` in a path to the home +directory. + +**Why not just use `os/user`?** The built-in `os/user` package is not +available on certain architectures such as i386 or PNaCl. Additionally +it has a cgo dependency on Darwin systems. This means that any Go code +that uses that package cannot cross compile. But 99% of the time the +use for `os/user` is just to retrieve the home directory, which we can +do for the current user without cgo. This library does that, enabling +cross-compilation. diff --git a/vendor/github.com/minio/go-homedir/dir_posix.go b/vendor/github.com/minio/go-homedir/dir_posix.go new file mode 100644 index 000000000..4615fe063 --- /dev/null +++ b/vendor/github.com/minio/go-homedir/dir_posix.go @@ -0,0 +1,64 @@ +// +build !windows + +// Copyright 2016 (C) Mitchell Hashimoto +// Distributed under the MIT License. + +package homedir + +import ( + "bytes" + "errors" + "os" + "os/exec" + "os/user" + "strconv" + "strings" +) + +// dir returns the homedir of current user for all POSIX compatible +// operating systems. +func dir() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + + // user.Current is not implemented for i386 and PNaCL like environments. + if currUser, err := user.Current(); err == nil { + return currUser.HomeDir, nil + } + + // If that fails, try getent + var stdout bytes.Buffer + cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid())) + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + // If "getent" is missing, ignore it + if err != exec.ErrNotFound { + return "", err + } + } else { + if passwd := strings.TrimSpace(stdout.String()); passwd != "" { + // username:password:uid:gid:gecos:home:shell + passwdParts := strings.SplitN(passwd, ":", 7) + if len(passwdParts) > 5 { + return passwdParts[5], nil + } + } + } + + // If all else fails, try the shell + stdout.Reset() + cmd = exec.Command("sh", "-c", "cd && pwd") + cmd.Stdout = &stdout + if err := cmd.Run(); err != nil { + return "", err + } + + result := strings.TrimSpace(stdout.String()) + if result == "" { + return "", errors.New("blank output when reading home directory") + } + + return result, nil +} diff --git a/vendor/github.com/minio/go-homedir/dir_windows.go b/vendor/github.com/minio/go-homedir/dir_windows.go new file mode 100644 index 000000000..85e5218c7 --- /dev/null +++ b/vendor/github.com/minio/go-homedir/dir_windows.go @@ -0,0 +1,28 @@ +// Copyright 2016 (C) Mitchell Hashimoto +// Distributed under the MIT License. + +package homedir + +import ( + "errors" + "os" +) + +// dir returns the homedir of current user for MS Windows OS. +func dir() (string, error) { + // First prefer the HOME environmental variable + if home := os.Getenv("HOME"); home != "" { + return home, nil + } + drive := os.Getenv("HOMEDRIVE") + path := os.Getenv("HOMEPATH") + home := drive + path + if drive == "" || path == "" { + home = os.Getenv("USERPROFILE") + } + if home == "" { + return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank") + } + + return home, nil +} diff --git a/vendor/github.com/minio/go-homedir/homedir.go b/vendor/github.com/minio/go-homedir/homedir.go new file mode 100644 index 000000000..092373801 --- /dev/null +++ b/vendor/github.com/minio/go-homedir/homedir.go @@ -0,0 +1,68 @@ +// Copyright 2016 (C) Mitchell Hashimoto +// Distributed under the MIT License. + +// Package homedir implements a portable function to determine current user's homedir. +package homedir + +import ( + "errors" + "path/filepath" + "sync" +) + +// DisableCache will disable caching of the home directory. Caching is enabled +// by default. +var DisableCache bool + +var homedirCache string +var cacheLock sync.Mutex + +// Dir returns the home directory for the executing user. +// +// This uses an OS-specific method for discovering the home directory. +// An error is returned if a home directory cannot be detected. +func Dir() (string, error) { + cacheLock.Lock() + defer cacheLock.Unlock() + + // Return cached homedir if available. + if !DisableCache { + if homedirCache != "" { + return homedirCache, nil + } + } + + // Determine OS speific current homedir. + result, err := dir() + if err != nil { + return "", err + } + + // Cache for future lookups. + homedirCache = result + return result, nil +} + +// Expand expands the path to include the home directory if the path +// is prefixed with `~`. If it isn't prefixed with `~`, the path is +// returned as-is. +func Expand(path string) (string, error) { + if len(path) == 0 { + return path, nil + } + + if path[0] != '~' { + return path, nil + } + + if len(path) > 1 && path[1] != '/' && path[1] != '\\' { + return "", errors.New("cannot expand user-specific home dir") + } + + dir, err := Dir() + if err != nil { + return "", err + } + + return filepath.Join(dir, path[1:]), nil +} diff --git a/vendor/github.com/minio/go-homedir/homedir_test.go b/vendor/github.com/minio/go-homedir/homedir_test.go new file mode 100644 index 000000000..a45121ff1 --- /dev/null +++ b/vendor/github.com/minio/go-homedir/homedir_test.go @@ -0,0 +1,114 @@ +package homedir + +import ( + "os" + "os/user" + "path/filepath" + "testing" +) + +func patchEnv(key, value string) func() { + bck := os.Getenv(key) + deferFunc := func() { + os.Setenv(key, bck) + } + + os.Setenv(key, value) + return deferFunc +} + +func BenchmarkDir(b *testing.B) { + // We do this for any "warmups" + for i := 0; i < 10; i++ { + Dir() + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + Dir() + } +} + +func TestDir(t *testing.T) { + // NOTE: This test is not portable. If user.Current() worked + // everywhere, we wouldn't need our package in the first place. + u, err := user.Current() + if err != nil { + t.Fatalf("err: %s", err) + } + + dir, err := Dir() + if err != nil { + t.Fatalf("err: %s", err) + } + + if u.HomeDir != dir { + t.Fatalf("%#v != %#v", u.HomeDir, dir) + } +} + +func TestExpand(t *testing.T) { + u, err := user.Current() + if err != nil { + t.Fatalf("err: %s", err) + } + + cases := []struct { + Input string + Output string + Err bool + }{ + { + "/foo", + "/foo", + false, + }, + + { + "~/foo", + filepath.Join(u.HomeDir, "foo"), + false, + }, + + { + "", + "", + false, + }, + + { + "~", + u.HomeDir, + false, + }, + + { + "~foo/foo", + "", + true, + }, + } + + for _, tc := range cases { + actual, err := Expand(tc.Input) + if (err != nil) != tc.Err { + t.Fatalf("Input: %#v\n\nErr: %s", tc.Input, err) + } + + if actual != tc.Output { + t.Fatalf("Input: %#v\n\nOutput: %#v", tc.Input, actual) + } + } + + DisableCache = true + defer func() { DisableCache = false }() + defer patchEnv("HOME", "/custom/path/")() + expected := filepath.Join("/", "custom", "path", "foo/bar") + actual, err := Expand("~/foo/bar") + + if err != nil { + t.Errorf("No error is expected, got: %v", err) + } else if actual != expected { + t.Errorf("Expected: %v; actual: %v", expected, actual) + } +} diff --git a/vendor/github.com/minio/minio-go/.travis.yml b/vendor/github.com/minio/minio-go/.travis.yml index 0c353ba76..066cbe883 100644 --- a/vendor/github.com/minio/minio-go/.travis.yml +++ b/vendor/github.com/minio/minio-go/.travis.yml @@ -12,6 +12,7 @@ go: - 1.5.3 - 1.6 - 1.7.4 +- 1.8 script: - diff -au <(gofmt -d .) <(printf "") diff --git a/vendor/github.com/minio/minio-go/README.md b/vendor/github.com/minio/minio-go/README.md index fb94f8010..4a91dc9c8 100644 --- a/vendor/github.com/minio/minio-go/README.md +++ b/vendor/github.com/minio/minio-go/README.md @@ -1,4 +1,4 @@ -# Minio Go Client SDK for Amazon S3 Compatible Cloud Storage [](https://slack.minio.io) +# Minio Go Client SDK for Amazon S3 Compatible Cloud Storage [](https://slack.minio.io) [](https://sourcegraph.com/github.com/minio/minio-go?badge) The Minio Go Client SDK provides simple APIs to access any Amazon S3 compatible object storage. @@ -8,7 +8,6 @@ The Minio Go Client SDK provides simple APIs to access any Amazon S3 compatible - Amazon S3 - Minio - - AWS Signature Version 2 - Google Cloud Storage (Compatibility Mode) - Openstack Swift + Swift3 middleware @@ -19,19 +18,14 @@ This quickstart guide will show you how to install the Minio client SDK, connect This document assumes that you have a working [Go development environment](https://docs.minio.io/docs/how-to-install-golang). - ## Download from Github - ```sh - go get -u github.com/minio/minio-go - ``` -## Initialize Minio Client +## Initialize Minio Client Minio client requires the following four parameters specified to connect to an Amazon S3 compatible object storage. - | Parameter | Description| | :--- | :--- | | endpoint | URL to object storage service. | @@ -41,7 +35,6 @@ Minio client requires the following four parameters specified to connect to an A ```go - package main import ( @@ -62,21 +55,14 @@ func main() { } log.Println("%v", minioClient) // minioClient is now setup - - ``` ## Quick Start Example - File Uploader - This example program connects to an object storage server, creates a bucket and uploads a file to the bucket. - - - We will use the Minio server running at [https://play.minio.io:9000](https://play.minio.io:9000) in this example. Feel free to use this service for testing and development. Access credentials shown in this example are open to the public. -#### FileUploader.go - +### FileUploader.go ```go package main @@ -97,7 +83,7 @@ func main() { log.Fatalln(err) } - // Make a new bucked called mymusic. + // Make a new bucket called mymusic. bucketName := "mymusic" location := "us-east-1" @@ -128,21 +114,17 @@ func main() { } ``` -#### Run FileUploader - +### Run FileUploader ```sh - go run file-uploader.go 2016/08/13 17:03:28 Successfully created mymusic 2016/08/13 17:03:40 Successfully uploaded golden-oldies.zip of size 16253413 mc ls play/mymusic/ [2016-05-27 16:02:16 PDT] 17MiB golden-oldies.zip - ``` ## API Reference - The full API Reference is available here. * [Complete API Reference](https://docs.minio.io/docs/golang-client-api-reference) @@ -179,12 +161,18 @@ The full API Reference is available here. * [`GetObject`](https://docs.minio.io/docs/golang-client-api-reference#GetObject) * [`PutObject`](https://docs.minio.io/docs/golang-client-api-reference#PutObject) +* [`PutObjectStreaming`](https://docs.minio.io/docs/golang-client-api-reference#PutObjectStreaming) * [`StatObject`](https://docs.minio.io/docs/golang-client-api-reference#StatObject) * [`CopyObject`](https://docs.minio.io/docs/golang-client-api-reference#CopyObject) * [`RemoveObject`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObject) * [`RemoveObjects`](https://docs.minio.io/docs/golang-client-api-reference#RemoveObjects) * [`RemoveIncompleteUpload`](https://docs.minio.io/docs/golang-client-api-reference#RemoveIncompleteUpload) +### API Reference: Encrypted Object Operations + +* [`GetEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#GetEncryptedObject) +* [`PutEncryptedObject`](https://docs.minio.io/docs/golang-client-api-reference#PutEncryptedObject) + ### API Reference : Presigned Operations * [`PresignedGetObject`](https://docs.minio.io/docs/golang-client-api-reference#PresignedGetObject) @@ -238,6 +226,11 @@ The full API Reference is available here. * [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go) * [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go) +#### Full Examples : Encrypted Object Operations + +* [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go) +* [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go) + #### Full Examples : Presigned Operations * [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go) * [presignedputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedputobject.go) @@ -246,7 +239,7 @@ The full API Reference is available here. ## Explore Further * [Complete Documentation](https://docs.minio.io) * [Minio Go Client SDK API Reference](https://docs.minio.io/docs/golang-client-api-reference) -* [Go Music Player App- Full Application Example ](https://docs.minio.io/docs/go-music-player-app) +* [Go Music Player App Full Application Example](https://docs.minio.io/docs/go-music-player-app) ## Contribute diff --git a/vendor/github.com/minio/minio-go/api-error-response.go b/vendor/github.com/minio/minio-go/api-error-response.go index fee3c7d53..ff8b8b109 100644 --- a/vendor/github.com/minio/minio-go/api-error-response.go +++ b/vendor/github.com/minio/minio-go/api-error-response.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -48,6 +48,9 @@ type ErrorResponse struct { // Region where the bucket is located. This header is returned // only in HEAD bucket and ListObjects response. Region string + + // Headers of the returned S3 XML error + Headers http.Header `xml:"-" json:"-"` } // ToErrorResponse - Returns parsed ErrorResponse struct from body and @@ -72,8 +75,15 @@ func ToErrorResponse(err error) ErrorResponse { } } -// Error - Returns HTTP error string +// Error - Returns S3 error string. func (e ErrorResponse) Error() string { + if e.Message == "" { + msg, ok := s3ErrorResponseMap[e.Code] + if !ok { + msg = fmt.Sprintf("Error response code %s.", e.Code) + } + return msg + } return e.Message } @@ -91,6 +101,7 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) return ErrInvalidArgument(msg) } var errResp ErrorResponse + err := xmlDecoder(resp.Body, &errResp) // Xml decoding failed with no body, fall back to HTTP headers. if err != nil { @@ -101,9 +112,6 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) Code: "NoSuchBucket", Message: "The specified bucket does not exist.", BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), } } else { errResp = ErrorResponse{ @@ -111,9 +119,6 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) Message: "The specified key does not exist.", BucketName: bucketName, Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), } } case http.StatusForbidden: @@ -122,30 +127,44 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string) Message: "Access Denied.", BucketName: bucketName, Key: objectName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), } case http.StatusConflict: errResp = ErrorResponse{ Code: "Conflict", Message: "Bucket not empty.", BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), + } + case http.StatusPreconditionFailed: + errResp = ErrorResponse{ + Code: "PreconditionFailed", + Message: s3ErrorResponseMap["PreconditionFailed"], + BucketName: bucketName, + Key: objectName, } default: errResp = ErrorResponse{ Code: resp.Status, Message: resp.Status, BucketName: bucketName, - RequestID: resp.Header.Get("x-amz-request-id"), - HostID: resp.Header.Get("x-amz-id-2"), - Region: resp.Header.Get("x-amz-bucket-region"), } } } + + // Save hodID, requestID and region information + // from headers if not available through error XML. + if errResp.RequestID == "" { + errResp.RequestID = resp.Header.Get("x-amz-request-id") + } + if errResp.HostID == "" { + errResp.HostID = resp.Header.Get("x-amz-id-2") + } + if errResp.Region == "" { + errResp.Region = resp.Header.Get("x-amz-bucket-region") + } + + // Save headers returned in the API XML error + errResp.Headers = resp.Header + return errResp } diff --git a/vendor/github.com/minio/minio-go/api-error-response_test.go b/vendor/github.com/minio/minio-go/api-error-response_test.go index 11f57165f..595cb50bd 100644 --- a/vendor/github.com/minio/minio-go/api-error-response_test.go +++ b/vendor/github.com/minio/minio-go/api-error-response_test.go @@ -72,6 +72,7 @@ func TestHttpRespToErrorResponse(t *testing.T) { RequestID: resp.Header.Get("x-amz-request-id"), HostID: resp.Header.Get("x-amz-id-2"), Region: resp.Header.Get("x-amz-bucket-region"), + Headers: resp.Header, } return errResp } @@ -172,7 +173,7 @@ func TestHttpRespToErrorResponse(t *testing.T) { for i, testCase := range testCases { actualResult := httpRespToErrorResponse(testCase.inputHTTPResp, testCase.bucketName, testCase.objectName) if !reflect.DeepEqual(testCase.expectedResult, actualResult) { - t.Errorf("Test %d: Expected result to be '%+v', but instead got '%+v'", i+1, testCase.expectedResult, actualResult) + t.Errorf("Test %d: Expected result to be '%#v', but instead got '%#v'", i+1, testCase.expectedResult, actualResult) } } } @@ -261,3 +262,21 @@ func TestErrInvalidArgument(t *testing.T) { t.Errorf("Expected result to be '%+v', but instead got '%+v'", expectedResult, actualResult) } } + +// Tests if the Message field is missing. +func TestErrWithoutMessage(t *testing.T) { + errResp := ErrorResponse{ + Code: "AccessDenied", + RequestID: "minio", + } + if errResp.Error() != "Access Denied." { + t.Errorf("Expected \"Access Denied.\", got %s", errResp) + } + errResp = ErrorResponse{ + Code: "InvalidArgument", + RequestID: "minio", + } + if errResp.Error() != "Error response code InvalidArgument." { + t.Errorf("Expected \"Error response code InvalidArgument.\", got %s", errResp) + } +} diff --git a/vendor/github.com/minio/minio-go/api-get-object-file.go b/vendor/github.com/minio/minio-go/api-get-object-file.go index a38fc852a..477a0969f 100644 --- a/vendor/github.com/minio/minio-go/api-get-object-file.go +++ b/vendor/github.com/minio/minio-go/api-get-object-file.go @@ -78,8 +78,15 @@ func (c Client) FGetObject(bucketName, objectName, filePath string) error { return err } + // Initialize get object request headers to set the + // appropriate range offsets to read from. + reqHeaders := NewGetReqHeaders() + if st.Size() > 0 { + reqHeaders.SetRange(st.Size(), 0) + } + // Seek to current position for incoming reader. - objectReader, objectStat, err := c.getObject(bucketName, objectName, st.Size(), 0) + objectReader, objectStat, err := c.getObject(bucketName, objectName, reqHeaders) if err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/api-get-object.go b/vendor/github.com/minio/minio-go/api-get-object.go index 48ee947ff..2abd4608e 100644 --- a/vendor/github.com/minio/minio-go/api-get-object.go +++ b/vendor/github.com/minio/minio-go/api-get-object.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,8 +24,36 @@ import ( "strings" "sync" "time" + + "github.com/minio/minio-go/pkg/encrypt" ) +// GetEncryptedObject deciphers and streams data stored in the server after applying a specifed encryption materiels +func (c Client) GetEncryptedObject(bucketName, objectName string, encryptMaterials encrypt.Materials) (io.Reader, error) { + + if encryptMaterials == nil { + return nil, ErrInvalidArgument("Unable to recognize empty encryption properties") + } + + // Fetch encrypted object + encReader, err := c.GetObject(bucketName, objectName) + if err != nil { + return nil, err + } + // Stat object to get its encryption metadata + st, err := encReader.Stat() + if err != nil { + return nil, err + } + + // Setup object for decrytion, object is transparently + // decrypted as the consumer starts reading. + encryptMaterials.SetupDecryptMode(encReader, st.Metadata.Get(amzHeaderIV), st.Metadata.Get(amzHeaderKey)) + + // Success. + return encryptMaterials, nil +} + // GetObject - returns an seekable, readable object. func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // Input validation. @@ -39,6 +67,7 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { var httpReader io.ReadCloser var objectInfo ObjectInfo var err error + // Create request channel. reqCh := make(chan getRequest) // Create response channel. @@ -51,6 +80,9 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { defer close(reqCh) defer close(resCh) + // Used to verify if etag of object has changed since last read. + var etag string + // Loop through the incoming control messages and read data. for { select { @@ -69,16 +101,22 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { if req.isFirstReq { // First request is a Read/ReadAt. if req.isReadOp { + reqHeaders := NewGetReqHeaders() // Differentiate between wanting the whole object and just a range. if req.isReadAt { // If this is a ReadAt request only get the specified range. // Range is set with respect to the offset and length of the buffer requested. // Do not set objectInfo from the first readAt request because it will not get // the whole object. - httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer))) + reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + httpReader, objectInfo, err = c.getObject(bucketName, objectName, reqHeaders) } else { + if req.Offset > 0 { + reqHeaders.SetRange(req.Offset, 0) + } + // First request is a Read request. - httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0) + httpReader, objectInfo, err = c.getObject(bucketName, objectName, reqHeaders) } if err != nil { resCh <- getResponse{ @@ -86,6 +124,7 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { } return } + etag = objectInfo.ETag // Read at least firstReq.Buffer bytes, if not we have // reached our EOF. size, err := io.ReadFull(httpReader, req.Buffer) @@ -112,13 +151,18 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // Exit the go-routine. return } + etag = objectInfo.ETag // Send back the first response. resCh <- getResponse{ objectInfo: objectInfo, } } } else if req.settingObjectInfo { // Request is just to get objectInfo. - objectInfo, err := c.StatObject(bucketName, objectName) + reqHeaders := NewGetReqHeaders() + if etag != "" { + reqHeaders.SetMatchETag(etag) + } + objectInfo, err := c.statObject(bucketName, objectName, reqHeaders) if err != nil { resCh <- getResponse{ Error: err, @@ -138,6 +182,10 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // new ones when they haven't been already. // All readAt requests are new requests. if req.DidOffsetChange || !req.beenRead { + reqHeaders := NewGetReqHeaders() + if etag != "" { + reqHeaders.SetMatchETag(etag) + } if httpReader != nil { // Close previously opened http reader. httpReader.Close() @@ -145,9 +193,15 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // If this request is a readAt only get the specified range. if req.isReadAt { // Range is set with respect to the offset and length of the buffer requested. - httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer))) + reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + httpReader, _, err = c.getObject(bucketName, objectName, reqHeaders) } else { - httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0) + // Range is set with respect to the offset. + if req.Offset > 0 { + reqHeaders.SetRange(req.Offset, 0) + } + + httpReader, objectInfo, err = c.getObject(bucketName, objectName, reqHeaders) } if err != nil { resCh <- getResponse{ @@ -202,8 +256,8 @@ type getResponse struct { objectInfo ObjectInfo // Used for the first request. } -// Object represents an open object. It implements Read, ReadAt, -// Seeker, Close for a HTTP stream. +// Object represents an open object. It implements +// Reader, ReaderAt, Seeker, Closer for a HTTP stream. type Object struct { // Mutex. mutex *sync.Mutex @@ -241,6 +295,12 @@ type Object struct { func (o *Object) doGetRequest(request getRequest) (getResponse, error) { o.reqCh <- request response := <-o.resCh + + // Return any error to the top level. + if response.Error != nil { + return response, response.Error + } + // This was the first request. if !o.isStarted { // The object has been operated on. @@ -256,11 +316,6 @@ func (o *Object) doGetRequest(request getRequest) (getResponse, error) { if !o.beenRead { o.beenRead = response.didRead } - // Return any error to the top level. - if response.Error != nil { - return response, response.Error - } - // Data are ready on the wire, no need to reinitiate connection in lower level o.seekData = false @@ -566,7 +621,7 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectInfo, error) { +func (c Client) getObject(bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) { // Validate input arguments. if err := isValidBucketName(bucketName); err != nil { return nil, ObjectInfo{}, err @@ -575,22 +630,18 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( return nil, ObjectInfo{}, err } + // Set all the necessary reqHeaders. customHeader := make(http.Header) - // Set ranges if length and offset are valid. - // See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. - if length > 0 && offset >= 0 { - customHeader.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } else if offset > 0 && length == 0 { - customHeader.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } else if length < 0 && offset == 0 { - customHeader.Set("Range", fmt.Sprintf("bytes=%d", length)) + for key, value := range reqHeaders.Header { + customHeader[key] = value } // Execute GET on objectName. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - objectName: objectName, - customHeader: customHeader, + bucketName: bucketName, + objectName: objectName, + customHeader: customHeader, + contentSHA256Bytes: emptySHA256, }) if err != nil { return nil, ObjectInfo{}, err @@ -617,6 +668,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( Region: resp.Header.Get("x-amz-bucket-region"), } } + // Get content-type. contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) if contentType == "" { diff --git a/vendor/github.com/minio/minio-go/api-get-policy.go b/vendor/github.com/minio/minio-go/api-get-policy.go index da0a409cd..7491df330 100644 --- a/vendor/github.com/minio/minio-go/api-get-policy.go +++ b/vendor/github.com/minio/minio-go/api-get-policy.go @@ -34,8 +34,12 @@ func (c Client) GetBucketPolicy(bucketName, objectPrefix string) (bucketPolicy p if err := isValidObjectPrefix(objectPrefix); err != nil { return policy.BucketPolicyNone, err } - policyInfo, err := c.getBucketPolicy(bucketName, objectPrefix) + policyInfo, err := c.getBucketPolicy(bucketName) if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchBucketPolicy" { + return policy.BucketPolicyNone, nil + } return policy.BucketPolicyNone, err } return policy.GetPolicy(policyInfo.Statements, bucketName, objectPrefix), nil @@ -50,15 +54,24 @@ func (c Client) ListBucketPolicies(bucketName, objectPrefix string) (bucketPolic if err := isValidObjectPrefix(objectPrefix); err != nil { return map[string]policy.BucketPolicy{}, err } - policyInfo, err := c.getBucketPolicy(bucketName, objectPrefix) + policyInfo, err := c.getBucketPolicy(bucketName) if err != nil { + errResponse := ToErrorResponse(err) + if errResponse.Code == "NoSuchBucketPolicy" { + return map[string]policy.BucketPolicy{}, nil + } return map[string]policy.BucketPolicy{}, err } return policy.GetPolicies(policyInfo.Statements, bucketName), nil } +// Default empty bucket access policy. +var emptyBucketAccessPolicy = policy.BucketAccessPolicy{ + Version: "2012-10-17", +} + // Request server for current bucket policy. -func (c Client) getBucketPolicy(bucketName string, objectPrefix string) (policy.BucketAccessPolicy, error) { +func (c Client) getBucketPolicy(bucketName string) (policy.BucketAccessPolicy, error) { // Get resources properly escaped and lined up before // using them in http request. urlValues := make(url.Values) @@ -66,27 +79,25 @@ func (c Client) getBucketPolicy(bucketName string, objectPrefix string) (policy. // Execute GET on bucket to list objects. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { - return policy.BucketAccessPolicy{}, err + return emptyBucketAccessPolicy, err } if resp != nil { if resp.StatusCode != http.StatusOK { - errResponse := httpRespToErrorResponse(resp, bucketName, "") - if ToErrorResponse(errResponse).Code == "NoSuchBucketPolicy" { - return policy.BucketAccessPolicy{Version: "2012-10-17"}, nil - } - return policy.BucketAccessPolicy{}, errResponse + return emptyBucketAccessPolicy, httpRespToErrorResponse(resp, bucketName, "") } } + bucketPolicyBuf, err := ioutil.ReadAll(resp.Body) if err != nil { - return policy.BucketAccessPolicy{}, err + return emptyBucketAccessPolicy, err } policy := policy.BucketAccessPolicy{} diff --git a/vendor/github.com/minio/minio-go/api-list.go b/vendor/github.com/minio/minio-go/api-list.go index adfaa0a7a..6a228179e 100644 --- a/vendor/github.com/minio/minio-go/api-list.go +++ b/vendor/github.com/minio/minio-go/api-list.go @@ -35,7 +35,7 @@ import ( // func (c Client) ListBuckets() ([]BucketInfo, error) { // Execute GET on service. - resp, err := c.executeMethod("GET", requestMetadata{}) + resp, err := c.executeMethod("GET", requestMetadata{contentSHA256Bytes: emptySHA256}) defer closeResponse(resp) if err != nil { return nil, err @@ -168,14 +168,14 @@ func (c Client) ListObjectsV2(bucketName, objectPrefix string, recursive bool, d // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (listBucketV2Result, error) { +func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { // Validate bucket name. if err := isValidBucketName(bucketName); err != nil { - return listBucketV2Result{}, err + return ListBucketV2Result{}, err } // Validate object prefix. if err := isValidObjectPrefix(objectPrefix); err != nil { - return listBucketV2Result{}, err + return ListBucketV2Result{}, err } // Get resources properly escaped and lined up before // using them in http request. @@ -211,21 +211,22 @@ func (c Client) listObjectsV2Query(bucketName, objectPrefix, continuationToken s // Execute GET on bucket to list objects. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { - return listBucketV2Result{}, err + return ListBucketV2Result{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return listBucketV2Result{}, httpRespToErrorResponse(resp, bucketName, "") + return ListBucketV2Result{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode listBuckets XML. - listBucketResult := listBucketV2Result{} + listBucketResult := ListBucketV2Result{} err = xmlDecoder(resp.Body, &listBucketResult) if err != nil { return listBucketResult, err @@ -347,14 +348,14 @@ func (c Client) ListObjects(bucketName, objectPrefix string, recursive bool, don // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-keys - Sets the maximum number of keys returned in the response body. -func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (listBucketResult, error) { +func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimiter string, maxkeys int) (ListBucketResult, error) { // Validate bucket name. if err := isValidBucketName(bucketName); err != nil { - return listBucketResult{}, err + return ListBucketResult{}, err } // Validate object prefix. if err := isValidObjectPrefix(objectPrefix); err != nil { - return listBucketResult{}, err + return ListBucketResult{}, err } // Get resources properly escaped and lined up before // using them in http request. @@ -381,20 +382,21 @@ func (c Client) listObjectsQuery(bucketName, objectPrefix, objectMarker, delimit // Execute GET on bucket to list objects. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { - return listBucketResult{}, err + return ListBucketResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return listBucketResult{}, httpRespToErrorResponse(resp, bucketName, "") + return ListBucketResult{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode listBuckets XML. - listBucketResult := listBucketResult{} + listBucketResult := ListBucketResult{} err = xmlDecoder(resp.Body, &listBucketResult) if err != nil { return listBucketResult, err @@ -528,7 +530,7 @@ func (c Client) listIncompleteUploads(bucketName, objectPrefix string, recursive // ?delimiter - A delimiter is a character you use to group keys. // ?prefix - Limits the response to keys that begin with the specified prefix. // ?max-uploads - Sets the maximum number of multipart uploads returned in the response body. -func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (listMultipartUploadsResult, error) { +func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, prefix, delimiter string, maxUploads int) (ListMultipartUploadsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set uploads. @@ -559,20 +561,21 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, // Execute GET on bucketName to list multipart uploads. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { - return listMultipartUploadsResult{}, err + return ListMultipartUploadsResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return listMultipartUploadsResult{}, httpRespToErrorResponse(resp, bucketName, "") + return ListMultipartUploadsResult{}, httpRespToErrorResponse(resp, bucketName, "") } } // Decode response body. - listMultipartUploadsResult := listMultipartUploadsResult{} + listMultipartUploadsResult := ListMultipartUploadsResult{} err = xmlDecoder(resp.Body, &listMultipartUploadsResult) if err != nil { return listMultipartUploadsResult, err @@ -581,10 +584,10 @@ func (c Client) listMultipartUploadsQuery(bucketName, keyMarker, uploadIDMarker, } // listObjectParts list all object parts recursively. -func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsInfo map[int]objectPart, err error) { +func (c Client) listObjectParts(bucketName, objectName, uploadID string) (partsInfo map[int]ObjectPart, err error) { // Part number marker for the next batch of request. var nextPartNumberMarker int - partsInfo = make(map[int]objectPart) + partsInfo = make(map[int]ObjectPart) for { // Get list of uploaded parts a maximum of 1000 per request. listObjPartsResult, err := c.listObjectPartsQuery(bucketName, objectName, uploadID, nextPartNumberMarker, 1000) @@ -659,7 +662,7 @@ func (c Client) getTotalMultipartSize(bucketName, objectName, uploadID string) ( // ?part-number-marker - Specifies the part after which listing should // begin. // ?max-parts - Maximum parts to be listed per request. -func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (listObjectPartsResult, error) { +func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, partNumberMarker, maxParts int) (ListObjectPartsResult, error) { // Get resources properly escaped and lined up before using them in http request. urlValues := make(url.Values) // Set part number marker. @@ -676,21 +679,22 @@ func (c Client) listObjectPartsQuery(bucketName, objectName, uploadID string, pa // Execute GET on objectName to get list of parts. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: urlValues, + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { - return listObjectPartsResult{}, err + return ListObjectPartsResult{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return listObjectPartsResult{}, httpRespToErrorResponse(resp, bucketName, objectName) + return ListObjectPartsResult{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // Decode list object parts XML. - listObjectPartsResult := listObjectPartsResult{} + listObjectPartsResult := ListObjectPartsResult{} err = xmlDecoder(resp.Body, &listObjectPartsResult) if err != nil { return listObjectPartsResult, err diff --git a/vendor/github.com/minio/minio-go/api-notification.go b/vendor/github.com/minio/minio-go/api-notification.go index 9c2a2ebd2..cbea1c6da 100644 --- a/vendor/github.com/minio/minio-go/api-notification.go +++ b/vendor/github.com/minio/minio-go/api-notification.go @@ -47,8 +47,9 @@ func (c Client) getBucketNotification(bucketName string) (BucketNotification, er // Execute GET on bucket to list objects. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) @@ -102,6 +103,14 @@ type eventMeta struct { Object objectMeta `json:"object"` } +// sourceInfo represents information on the client that +// triggered the event notification. +type sourceInfo struct { + Host string `json:"host"` + Port string `json:"port"` + UserAgent string `json:"userAgent"` +} + // NotificationEvent represents an Amazon an S3 bucket notification event. type NotificationEvent struct { EventVersion string `json:"eventVersion"` @@ -113,6 +122,7 @@ type NotificationEvent struct { RequestParameters map[string]string `json:"requestParameters"` ResponseElements map[string]string `json:"responseElements"` S3 eventMeta `json:"s3"` + Source sourceInfo `json:"source"` } // NotificationInfo - represents the collection of notification events, additionally @@ -161,8 +171,9 @@ func (c Client) ListenBucketNotification(bucketName, prefix, suffix string, even // Execute GET on bucket to list objects. resp, err := c.executeMethod("GET", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) if err != nil { continue diff --git a/vendor/github.com/minio/minio-go/api-presigned.go b/vendor/github.com/minio/minio-go/api-presigned.go index f9d05ab9b..ece005d47 100644 --- a/vendor/github.com/minio/minio-go/api-presigned.go +++ b/vendor/github.com/minio/minio-go/api-presigned.go @@ -122,21 +122,38 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str return nil, nil, err } + // Get credentials from the configured credentials provider. + credValues, err := c.credsProvider.Get() + if err != nil { + return nil, nil, err + } + + var ( + signerType = credValues.SignerType + sessionToken = credValues.SessionToken + accessKeyID = credValues.AccessKeyID + secretAccessKey = credValues.SecretAccessKey + ) + + if signerType.IsAnonymous() { + return nil, nil, ErrInvalidArgument("Presigned operations are not supported for anonymous credentials") + } + // Keep time. t := time.Now().UTC() // For signature version '2' handle here. - if c.signature.isV2() { + if signerType.IsV2() { policyBase64 := p.base64() p.formData["policy"] = policyBase64 // For Google endpoint set this value to be 'GoogleAccessId'. if s3utils.IsGoogleEndpoint(c.endpointURL) { - p.formData["GoogleAccessId"] = c.accessKeyID + p.formData["GoogleAccessId"] = accessKeyID } else { // For all other endpoints set this value to be 'AWSAccessKeyId'. - p.formData["AWSAccessKeyId"] = c.accessKeyID + p.formData["AWSAccessKeyId"] = accessKeyID } // Sign the policy. - p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, c.secretAccessKey) + p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, secretAccessKey) return u, p.formData, nil } @@ -159,7 +176,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str } // Add a credential policy. - credential := s3signer.GetCredential(c.accessKeyID, location, t) + credential := s3signer.GetCredential(accessKeyID, location, t) if err = p.addNewPolicy(policyCondition{ matchType: "eq", condition: "$x-amz-credential", @@ -168,13 +185,27 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str return nil, nil, err } + if sessionToken != "" { + if err = p.addNewPolicy(policyCondition{ + matchType: "eq", + condition: "$x-amz-security-token", + value: sessionToken, + }); err != nil { + return nil, nil, err + } + } + // Get base64 encoded policy. policyBase64 := p.base64() + // Fill in the form data. p.formData["policy"] = policyBase64 p.formData["x-amz-algorithm"] = signV4Algorithm p.formData["x-amz-credential"] = credential p.formData["x-amz-date"] = t.Format(iso8601DateFormat) - p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, c.secretAccessKey, location) + if sessionToken != "" { + p.formData["x-amz-security-token"] = sessionToken + } + p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location) return u, p.formData, nil } diff --git a/vendor/github.com/minio/minio-go/api-put-bucket.go b/vendor/github.com/minio/minio-go/api-put-bucket.go index 7c7e03f49..2d91e6d16 100644 --- a/vendor/github.com/minio/minio-go/api-put-bucket.go +++ b/vendor/github.com/minio/minio-go/api-put-bucket.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -28,6 +29,7 @@ import ( "net/url" "path" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/policy" "github.com/minio/minio-go/pkg/s3signer" ) @@ -41,7 +43,14 @@ import ( // // For Amazon S3 for more supported regions - http://docs.aws.amazon.com/general/latest/gr/rande.html // For Google Cloud Storage for more supported regions - https://cloud.google.com/storage/docs/bucket-locations -func (c Client) MakeBucket(bucketName string, location string) error { +func (c Client) MakeBucket(bucketName string, location string) (err error) { + defer func() { + // Save the location into cache on a successful makeBucket response. + if err == nil { + c.bucketLocCache.Set(bucketName, location) + } + }() + // Validate the input arguments. if err := isValidBucketName(bucketName); err != nil { return err @@ -52,45 +61,70 @@ func (c Client) MakeBucket(bucketName string, location string) error { location = "us-east-1" } - // Instantiate the request. - req, err := c.makeBucketRequest(bucketName, location) - if err != nil { - return err - } + // Try creating bucket with the provided region, in case of + // invalid region error let's guess the appropriate region + // from S3 API headers - // Execute the request. - resp, err := c.do(req) - defer closeResponse(resp) - if err != nil { - return err - } + // Create a done channel to control 'newRetryTimer' go routine. + doneCh := make(chan struct{}, 1) + + // Indicate to our routine to exit cleanly upon return. + defer close(doneCh) + + // Blank indentifier is kept here on purpose since 'range' without + // blank identifiers is only supported since go1.4 + // https://golang.org/doc/go1.4#forrange. + for _ = range c.newRetryTimer(MaxRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { + // Initialize the makeBucket request. + req, err := c.makeBucketRequest(bucketName, location) + if err != nil { + return err + } + + // Execute make bucket request. + resp, err := c.do(req) + defer closeResponse(resp) + if err != nil { + return err + } - if resp != nil { if resp.StatusCode != http.StatusOK { - return httpRespToErrorResponse(resp, bucketName, "") + err := httpRespToErrorResponse(resp, bucketName, "") + errResp := ToErrorResponse(err) + if resp.StatusCode == http.StatusBadRequest && errResp.Region != "" { + // Fetch bucket region found in headers + // of S3 error response, attempt bucket + // create again. + location = errResp.Region + continue + } + // Nothing to retry, fail. + return err } - } - // Save the location into cache on a successful makeBucket response. - c.bucketLocCache.Set(bucketName, location) + // Control reaches here when bucket create was successful, + // break out. + break + } - // Return. + // Success. return nil } -// makeBucketRequest constructs request for makeBucket. +// Low level wrapper API For makeBucketRequest. func (c Client) makeBucketRequest(bucketName string, location string) (*http.Request, error) { // Validate input arguments. if err := isValidBucketName(bucketName); err != nil { return nil, err } - // In case of Amazon S3. The make bucket issued on already - // existing bucket would fail with 'AuthorizationMalformed' error - // if virtual style is used. So we default to 'path style' as that - // is the preferred method here. The final location of the - // 'bucket' is provided through XML LocationConstraint data with - // the request. + // In case of Amazon S3. The make bucket issued on + // already existing bucket would fail with + // 'AuthorizationMalformed' error if virtual style is + // used. So we default to 'path style' as that is the + // preferred method here. The final location of the + // 'bucket' is provided through XML LocationConstraint + // data with the request. targetURL := c.endpointURL targetURL.Path = path.Join(bucketName, "") + "/" @@ -103,8 +137,32 @@ func (c Client) makeBucketRequest(bucketName string, location string) (*http.Req // set UserAgent for the request. c.setUserAgent(req) + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + // set sha256 sum for signature calculation only with signature version '4'. - if c.signature.isV4() { + if signerType.IsV4() { req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) } @@ -122,19 +180,19 @@ func (c Client) makeBucketRequest(bucketName string, location string) (*http.Req req.ContentLength = int64(len(createBucketConfigBytes)) // Set content-md5. req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes))) - if c.signature.isV4() { + if signerType.IsV4() { // Set sha256. req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes))) } } // Sign the request. - if c.signature.isV4() { + if signerType.IsV4() { // Signature calculated for MakeBucket request should be for 'us-east-1', // regardless of the bucket's location constraint. - req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") - } else if c.signature.isV2() { - req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + } else if signerType.IsV2() { + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) } // Return signed request. @@ -157,11 +215,14 @@ func (c Client) SetBucketPolicy(bucketName string, objectPrefix string, bucketPo if err := isValidObjectPrefix(objectPrefix); err != nil { return err } + if !bucketPolicy.IsValidBucketPolicy() { return ErrInvalidArgument(fmt.Sprintf("Invalid bucket policy provided. %s", bucketPolicy)) } - policyInfo, err := c.getBucketPolicy(bucketName, objectPrefix) - if err != nil { + + policyInfo, err := c.getBucketPolicy(bucketName) + errResponse := ToErrorResponse(err) + if err != nil && errResponse.Code != "NoSuchBucketPolicy" { return err } @@ -236,8 +297,9 @@ func (c Client) removeBucketPolicy(bucketName string) error { // Execute DELETE on objectName. resp, err := c.executeMethod("DELETE", requestMetadata{ - bucketName: bucketName, - queryValues: urlValues, + bucketName: bucketName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { diff --git a/vendor/github.com/minio/minio-go/api-put-bucket_test.go b/vendor/github.com/minio/minio-go/api-put-bucket_test.go index ec33c8492..4f7ddb30e 100644 --- a/vendor/github.com/minio/minio-go/api-put-bucket_test.go +++ b/vendor/github.com/minio/minio-go/api-put-bucket_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +28,7 @@ import ( "path" "testing" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/s3signer" ) @@ -48,8 +50,32 @@ func TestMakeBucketRequest(t *testing.T) { // set UserAgent for the request. c.setUserAgent(req) + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + // set sha256 sum for signature calculation only with signature version '4'. - if c.signature.isV4() { + if signerType.IsV4() { req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) } @@ -67,19 +93,19 @@ func TestMakeBucketRequest(t *testing.T) { req.ContentLength = int64(len(createBucketConfigBytes)) // Set content-md5. req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes))) - if c.signature.isV4() { + if signerType.IsV4() { // Set sha256. req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes))) } } // Sign the request. - if c.signature.isV4() { + if signerType.IsV4() { // Signature calculated for MakeBucket request should be for 'us-east-1', // regardless of the bucket's location constraint. - req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") - } else if c.signature.isV2() { - req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + } else if signerType.IsV2() { + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) } // Return signed request. @@ -246,7 +272,7 @@ func TestMakeBucketRequest(t *testing.T) { } if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") { - t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1) + t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request %s doesn't match with that of the actual request %s", i+1, expectedReq.Header.Get("X-Amz-Content-Sha256"), actualReq.Header.Get("X-Amz-Content-Sha256")) } if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") { t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent")) diff --git a/vendor/github.com/minio/minio-go/api-put-object-common.go b/vendor/github.com/minio/minio-go/api-put-object-common.go index 5f5f568e6..68a459f4a 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-common.go +++ b/vendor/github.com/minio/minio-go/api-put-object-common.go @@ -44,7 +44,7 @@ func isReadAt(reader io.Reader) (ok bool) { } // shouldUploadPart - verify if part should be uploaded. -func shouldUploadPart(objPart objectPart, uploadReq uploadPartReq) bool { +func shouldUploadPart(objPart ObjectPart, uploadReq uploadPartReq) bool { // If part not found should upload the part. if uploadReq.Part == nil { return true @@ -185,9 +185,9 @@ func (c Client) newUploadID(bucketName, objectName string, metaData map[string][ // getMpartUploadSession returns the upload id and the uploaded parts to continue a previous upload session // or initiate a new multipart session if no current one found -func (c Client) getMpartUploadSession(bucketName, objectName string, metaData map[string][]string) (string, map[int]objectPart, error) { +func (c Client) getMpartUploadSession(bucketName, objectName string, metaData map[string][]string) (string, map[int]ObjectPart, error) { // A map of all uploaded parts. - var partsInfo map[int]objectPart + var partsInfo map[int]ObjectPart var err error uploadID, err := c.findUploadID(bucketName, objectName) @@ -220,7 +220,7 @@ func (c Client) getMpartUploadSession(bucketName, objectName string, metaData ma // Allocate partsInfo if not done yet if partsInfo == nil { - partsInfo = make(map[int]objectPart) + partsInfo = make(map[int]ObjectPart) } return uploadID, partsInfo, nil diff --git a/vendor/github.com/minio/minio-go/api-put-object-file.go b/vendor/github.com/minio/minio-go/api-put-object-file.go index aa554b321..fc475c9a3 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-file.go +++ b/vendor/github.com/minio/minio-go/api-put-object-file.go @@ -91,25 +91,11 @@ func (c Client) FPutObject(bucketName, objectName, filePath, contentType string) return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, objMetadata, nil) } - // NOTE: S3 doesn't allow anonymous multipart requests. - if s3utils.IsAmazonEndpoint(c.endpointURL) && c.anonymous { - if fileSize > int64(maxSinglePutObjectSize) { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: fmt.Sprintf("For anonymous requests Content-Length cannot be %d.", fileSize), - Key: objectName, - BucketName: bucketName, - } - } - // Do not compute MD5 for anonymous requests to Amazon - // S3. Uploads up to 5GiB in size. - return c.putObjectNoChecksum(bucketName, objectName, fileReader, fileSize, objMetadata, nil) - } - // Small object upload is initiated for uploads for input data size smaller than 5MiB. if fileSize < minPartSize && fileSize >= 0 { return c.putObjectSingle(bucketName, objectName, fileReader, fileSize, objMetadata, nil) } + // Upload all large objects as multipart. n, err = c.putObjectMultipartFromFile(bucketName, objectName, fileReader, fileSize, objMetadata, nil) if err != nil { @@ -187,7 +173,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe close(uploadPartsCh) // Use three 'workers' to upload parts in parallel. - for w := 1; w <= 3; w++ { + for w := 1; w <= totalWorkers; w++ { go func() { // Deal with each part as it comes through the channel. for uploadReq := range uploadPartsCh { @@ -196,7 +182,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe hashAlgos := make(map[string]hash.Hash) hashSums := make(map[string][]byte) hashAlgos["md5"] = md5.New() - if c.signature.isV4() && !c.secure { + if c.overrideSignerType.IsV4() && !c.secure { hashAlgos["sha256"] = sha256.New() } @@ -228,7 +214,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe } // Create the part to be uploaded. - verifyObjPart := objectPart{ + verifyObjPart := ObjectPart{ ETag: hex.EncodeToString(hashSums["md5"]), PartNumber: uploadReq.PartNum, Size: partSize, @@ -242,7 +228,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe // Verify if part should be uploaded. if shouldUploadPart(verifyObjPart, uploadReq) { // Proceed to upload the part. - var objPart objectPart + var objPart ObjectPart objPart, err = c.uploadPart(bucketName, objectName, uploadID, sectionReader, uploadReq.PartNum, hashSums["md5"], hashSums["sha256"], prtSize) if err != nil { uploadedPartsCh <- uploadedPartRes{ @@ -285,7 +271,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe } } // Store the part to be completed. - complMultipartUpload.Parts = append(complMultipartUpload.Parts, completePart{ + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, }) diff --git a/vendor/github.com/minio/minio-go/api-put-object-multipart.go b/vendor/github.com/minio/minio-go/api-put-object-multipart.go index f74eae626..5134a7fd2 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-multipart.go +++ b/vendor/github.com/minio/minio-go/api-put-object-multipart.go @@ -66,6 +66,112 @@ func (c Client) putObjectMultipart(bucketName, objectName string, reader io.Read return c.putObjectMultipartStream(bucketName, objectName, reader, size, metaData, progress) } +// putObjectMultipartStreamNoChecksum - upload a large object using +// multipart upload and streaming signature for signing payload. +// N B We don't resume an incomplete multipart upload, we overwrite +// existing parts of an incomplete upload. +func (c Client) putObjectMultipartStreamNoChecksum(bucketName, objectName string, + reader io.Reader, size int64, metadata map[string][]string, progress io.Reader) (int64, error) { + + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return 0, err + } + if err := isValidObjectName(objectName); err != nil { + return 0, err + } + + // Get the upload id of a previously partially uploaded object or initiate a new multipart upload + uploadID, err := c.findUploadID(bucketName, objectName) + if err != nil { + return 0, err + } + if uploadID == "" { + // Initiates a new multipart request + uploadID, err = c.newUploadID(bucketName, objectName, metadata) + if err != nil { + return 0, err + } + } + + // Calculate the optimal parts info for a given size. + totalPartsCount, partSize, lastPartSize, err := optimalPartInfo(size) + if err != nil { + return 0, err + } + + // Total data read and written to server. should be equal to 'size' at the end of the call. + var totalUploadedSize int64 + + // Initialize parts uploaded map. + partsInfo := make(map[int]ObjectPart) + + // Part number always starts with '1'. + var partNumber int + for partNumber = 1; partNumber <= totalPartsCount; partNumber++ { + // Update progress reader appropriately to the latest offset + // as we read from the source. + hookReader := newHook(reader, progress) + + // Proceed to upload the part. + if partNumber == totalPartsCount { + partSize = lastPartSize + } + + var objPart ObjectPart + objPart, err = c.uploadPart(bucketName, objectName, uploadID, + io.LimitReader(hookReader, partSize), partNumber, nil, nil, partSize) + // For unknown size, Read EOF we break away. + // We do not have to upload till totalPartsCount. + if err == io.EOF && size < 0 { + break + } + + if err != nil { + return totalUploadedSize, err + } + + // Save successfully uploaded part metadata. + partsInfo[partNumber] = objPart + + // Save successfully uploaded size. + totalUploadedSize += partSize + } + + // Verify if we uploaded all the data. + if size > 0 { + if totalUploadedSize != size { + return totalUploadedSize, ErrUnexpectedEOF(totalUploadedSize, size, bucketName, objectName) + } + } + + // Complete multipart upload. + var complMultipartUpload completeMultipartUpload + + // Loop over total uploaded parts to save them in + // Parts array before completing the multipart request. + for i := 1; i < partNumber; i++ { + part, ok := partsInfo[i] + if !ok { + return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) + } + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ + ETag: part.ETag, + PartNumber: part.PartNumber, + }) + } + + // Sort all completed parts. + sort.Sort(completedParts(complMultipartUpload.Parts)) + _, err = c.completeMultipartUpload(bucketName, objectName, uploadID, complMultipartUpload) + if err != nil { + return totalUploadedSize, err + } + + // Return final size. + return totalUploadedSize, nil +} + // putObjectStream uploads files bigger than 64MiB, and also supports // special case where size is unknown i.e '-1'. func (c Client) putObjectMultipartStream(bucketName, objectName string, reader io.Reader, size int64, metaData map[string][]string, progress io.Reader) (n int64, err error) { @@ -107,16 +213,14 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i hashSums := make(map[string][]byte) hashAlgos := make(map[string]hash.Hash) hashAlgos["md5"] = md5.New() - if c.signature.isV4() && !c.secure { + if c.overrideSignerType.IsV4() && !c.secure { hashAlgos["sha256"] = sha256.New() } // Calculates hash sums while copying partSize bytes into tmpBuffer. prtSize, rErr := hashCopyN(hashAlgos, hashSums, tmpBuffer, reader, partSize) - if rErr != nil { - if rErr != io.EOF { - return 0, rErr - } + if rErr != nil && rErr != io.EOF { + return 0, rErr } var reader io.Reader @@ -127,13 +231,13 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i part, ok := partsInfo[partNumber] // Verify if part should be uploaded. - if !ok || shouldUploadPart(objectPart{ + if !ok || shouldUploadPart(ObjectPart{ ETag: hex.EncodeToString(hashSums["md5"]), PartNumber: partNumber, Size: prtSize, }, uploadPartReq{PartNum: partNumber, Part: &part}) { // Proceed to upload the part. - var objPart objectPart + var objPart ObjectPart objPart, err = c.uploadPart(bucketName, objectName, uploadID, reader, partNumber, hashSums["md5"], hashSums["sha256"], prtSize) if err != nil { // Reset the temporary buffer upon any error. @@ -181,7 +285,7 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i if !ok { return 0, ErrInvalidArgument(fmt.Sprintf("Missing part number %d", i)) } - complMultipartUpload.Parts = append(complMultipartUpload.Parts, completePart{ + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, }) @@ -253,25 +357,25 @@ func (c Client) initiateMultipartUpload(bucketName, objectName string, metaData } // uploadPart - Uploads a part in a multipart upload. -func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Reader, partNumber int, md5Sum, sha256Sum []byte, size int64) (objectPart, error) { +func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Reader, partNumber int, md5Sum, sha256Sum []byte, size int64) (ObjectPart, error) { // Input validation. if err := isValidBucketName(bucketName); err != nil { - return objectPart{}, err + return ObjectPart{}, err } if err := isValidObjectName(objectName); err != nil { - return objectPart{}, err + return ObjectPart{}, err } if size > maxPartSize { - return objectPart{}, ErrEntityTooLarge(size, maxPartSize, bucketName, objectName) + return ObjectPart{}, ErrEntityTooLarge(size, maxPartSize, bucketName, objectName) } if size <= -1 { - return objectPart{}, ErrEntityTooSmall(size, bucketName, objectName) + return ObjectPart{}, ErrEntityTooSmall(size, bucketName, objectName) } if partNumber <= 0 { - return objectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.") + return ObjectPart{}, ErrInvalidArgument("Part number cannot be negative or equal to zero.") } if uploadID == "" { - return objectPart{}, ErrInvalidArgument("UploadID cannot be empty.") + return ObjectPart{}, ErrInvalidArgument("UploadID cannot be empty.") } // Get resources properly escaped and lined up before using them in http request. @@ -295,15 +399,15 @@ func (c Client) uploadPart(bucketName, objectName, uploadID string, reader io.Re resp, err := c.executeMethod("PUT", reqMetadata) defer closeResponse(resp) if err != nil { - return objectPart{}, err + return ObjectPart{}, err } if resp != nil { if resp.StatusCode != http.StatusOK { - return objectPart{}, httpRespToErrorResponse(resp, bucketName, objectName) + return ObjectPart{}, httpRespToErrorResponse(resp, bucketName, objectName) } } // Once successfully uploaded, return completed part. - objPart := objectPart{} + objPart := ObjectPart{} objPart.Size = size objPart.PartNumber = partNumber // Trim off the odd double quotes from ETag in the beginning and end. diff --git a/vendor/github.com/minio/minio-go/api-put-object-progress.go b/vendor/github.com/minio/minio-go/api-put-object-progress.go index 42f8ce4d1..e5b24ad2a 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-progress.go +++ b/vendor/github.com/minio/minio-go/api-put-object-progress.go @@ -20,6 +20,8 @@ import ( "io" "strings" + "github.com/minio/minio-go/pkg/credentials" + "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/s3utils" ) @@ -30,6 +32,29 @@ func (c Client) PutObjectWithProgress(bucketName, objectName string, reader io.R return c.PutObjectWithMetadata(bucketName, objectName, reader, metaData, progress) } +// PutEncryptedObject - Encrypt and store object. +func (c Client) PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials encrypt.Materials, metaData map[string][]string, progress io.Reader) (n int64, err error) { + + if encryptMaterials == nil { + return 0, ErrInvalidArgument("Unable to recognize empty encryption properties") + } + + if err := encryptMaterials.SetupEncryptMode(reader); err != nil { + return 0, err + } + + if metaData == nil { + metaData = make(map[string][]string) + } + + // Set the necessary encryption headers, for future decryption. + metaData[amzHeaderIV] = []string{encryptMaterials.GetIV()} + metaData[amzHeaderKey] = []string{encryptMaterials.GetKey()} + metaData[amzHeaderMatDesc] = []string{encryptMaterials.GetDesc()} + + return c.PutObjectWithMetadata(bucketName, objectName, encryptMaterials, metaData, progress) +} + // PutObjectWithMetadata - with metadata. func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.Reader, metaData map[string][]string, progress io.Reader) (n int64, err error) { // Input validation. @@ -75,28 +100,11 @@ func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.R return c.putObjectNoChecksum(bucketName, objectName, reader, size, metaData, progress) } - // NOTE: S3 doesn't allow anonymous multipart requests. - if s3utils.IsAmazonEndpoint(c.endpointURL) && c.anonymous { - if size <= -1 { - return 0, ErrorResponse{ - Code: "NotImplemented", - Message: "Content-Length cannot be negative for anonymous requests.", - Key: objectName, - BucketName: bucketName, - } - } - if size > maxSinglePutObjectSize { - return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) - } - // Do not compute MD5 for anonymous requests to Amazon - // S3. Uploads up to 5GiB in size. - return c.putObjectNoChecksum(bucketName, objectName, reader, size, metaData, progress) - } - // putSmall object. if size < minPartSize && size >= 0 { return c.putObjectSingle(bucketName, objectName, reader, size, metaData, progress) } + // For all sizes greater than 5MiB do multipart. n, err = c.putObjectMultipart(bucketName, objectName, reader, size, metaData, progress) if err != nil { @@ -115,3 +123,81 @@ func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.R } return n, nil } + +// PutObjectStreaming using AWS streaming signature V4 +func (c Client) PutObjectStreaming(bucketName, objectName string, reader io.Reader) (n int64, err error) { + return c.PutObjectStreamingWithProgress(bucketName, objectName, reader, nil, nil) +} + +// PutObjectStreamingWithMetadata using AWS streaming signature V4 +func (c Client) PutObjectStreamingWithMetadata(bucketName, objectName string, reader io.Reader, metadata map[string][]string) (n int64, err error) { + return c.PutObjectStreamingWithProgress(bucketName, objectName, reader, metadata, nil) +} + +// PutObjectStreamingWithProgress using AWS streaming signature V4 +func (c Client) PutObjectStreamingWithProgress(bucketName, objectName string, reader io.Reader, metadata map[string][]string, progress io.Reader) (n int64, err error) { + // NOTE: Streaming signature is not supported by GCS. + if s3utils.IsGoogleEndpoint(c.endpointURL) { + return 0, ErrorResponse{ + Code: "NotImplemented", + Message: "AWS streaming signature v4 is not supported with Google Cloud Storage", + Key: objectName, + BucketName: bucketName, + } + } + + if c.overrideSignerType.IsV2() { + return 0, ErrorResponse{ + Code: "NotImplemented", + Message: "AWS streaming signature v4 is not supported with minio client initialized for AWS signature v2", + Key: objectName, + BucketName: bucketName, + } + } + + // Size of the object. + var size int64 + + // Get reader size. + size, err = getReaderSize(reader) + if err != nil { + return 0, err + } + + // Check for largest object size allowed. + if size > int64(maxMultipartPutObjectSize) { + return 0, ErrEntityTooLarge(size, maxMultipartPutObjectSize, bucketName, objectName) + } + + // If size cannot be found on a stream, it is not possible + // to upload using streaming signature, fall back to multipart. + if size < 0 { + return c.putObjectMultipartStream(bucketName, objectName, reader, size, metadata, progress) + } + + // Set streaming signature. + c.overrideSignerType = credentials.SignatureV4Streaming + + if size < minPartSize && size >= 0 { + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) + } + + // For all sizes greater than 64MiB do multipart. + n, err = c.putObjectMultipartStreamNoChecksum(bucketName, objectName, reader, size, metadata, progress) + if err != nil { + errResp := ToErrorResponse(err) + // Verify if multipart functionality is not available, if not + // fall back to single PutObject operation. + if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { + // Verify if size of reader is greater than '5GiB'. + if size > maxSinglePutObjectSize { + return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName) + } + // Fall back to uploading as single PutObject operation. + return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress) + } + return n, err + } + + return n, nil +} diff --git a/vendor/github.com/minio/minio-go/api-put-object-readat.go b/vendor/github.com/minio/minio-go/api-put-object-readat.go index 4ab1095f6..0083d41fa 100644 --- a/vendor/github.com/minio/minio-go/api-put-object-readat.go +++ b/vendor/github.com/minio/minio-go/api-put-object-readat.go @@ -32,16 +32,16 @@ type uploadedPartRes struct { Error error // Any error encountered while uploading the part. PartNum int // Number of the part uploaded. Size int64 // Size of the part uploaded. - Part *objectPart + Part *ObjectPart } type uploadPartReq struct { PartNum int // Number of the part uploaded. - Part *objectPart // Size of the part uploaded. + Part *ObjectPart // Size of the part uploaded. } // shouldUploadPartReadAt - verify if part should be uploaded. -func shouldUploadPartReadAt(objPart objectPart, uploadReq uploadPartReq) bool { +func shouldUploadPartReadAt(objPart ObjectPart, uploadReq uploadPartReq) bool { // If part not found part should be uploaded. if uploadReq.Part == nil { return true @@ -115,7 +115,7 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read close(uploadPartsCh) // Receive each part number from the channel allowing three parallel uploads. - for w := 1; w <= 3; w++ { + for w := 1; w <= totalWorkers; w++ { go func() { // Read defaults to reading at 5MiB buffer. readAtBuffer := make([]byte, optimalReadBufferSize) @@ -146,7 +146,7 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read hashSums := make(map[string][]byte) hashAlgos := make(map[string]hash.Hash) hashAlgos["md5"] = md5.New() - if c.signature.isV4() && !c.secure { + if c.overrideSignerType.IsV4() && !c.secure { hashAlgos["sha256"] = sha256.New() } @@ -164,7 +164,7 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read } // Verify object if its uploaded. - verifyObjPart := objectPart{ + verifyObjPart := ObjectPart{ PartNumber: uploadReq.PartNum, Size: partSize, } @@ -178,7 +178,7 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read // to update any progress bar. if shouldUploadPartReadAt(verifyObjPart, uploadReq) { // Proceed to upload the part. - var objPart objectPart + var objPart ObjectPart objPart, err = c.uploadPart(bucketName, objectName, uploadID, tmpBuffer, uploadReq.PartNum, hashSums["md5"], hashSums["sha256"], prtSize) if err != nil { uploadedPartsCh <- uploadedPartRes{ @@ -224,7 +224,7 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read } } // Store the parts to be completed in order. - complMultipartUpload.Parts = append(complMultipartUpload.Parts, completePart{ + complMultipartUpload.Parts = append(complMultipartUpload.Parts, CompletePart{ ETag: part.ETag, PartNumber: part.PartNumber, }) diff --git a/vendor/github.com/minio/minio-go/api-put-object.go b/vendor/github.com/minio/minio-go/api-put-object.go index a779fbebe..0b6848fff 100644 --- a/vendor/github.com/minio/minio-go/api-put-object.go +++ b/vendor/github.com/minio/minio-go/api-put-object.go @@ -109,14 +109,24 @@ func getReaderSize(reader io.Reader) (size int64, err error) { case "|0", "|1": return } - size = st.Size() + var pos int64 + pos, err = v.Seek(0, 1) // SeekCurrent. + if err != nil { + return -1, err + } + size = st.Size() - pos case *Object: var st ObjectInfo st, err = v.Stat() if err != nil { return } - size = st.Size + var pos int64 + pos, err = v.Seek(0, 1) // SeekCurrent. + if err != nil { + return -1, err + } + size = st.Size - pos } } // Returns the size here. @@ -125,7 +135,7 @@ func getReaderSize(reader io.Reader) (size int64, err error) { // completedParts is a collection of parts sortable by their part numbers. // used for sorting the uploaded parts before completing the multipart request. -type completedParts []completePart +type completedParts []CompletePart func (a completedParts) Len() int { return len(a) } func (a completedParts) Swap(i, j int) { a[i], a[j] = a[j], a[i] } @@ -143,7 +153,6 @@ func (a completedParts) Less(i, j int) bool { return a[i].PartNumber < a[j].Part // NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT. // So we fall back to single PUT operation with the maximum limit of 5GiB. // -// NOTE: For anonymous requests Amazon S3 doesn't allow multipart upload. So we fall back to single PUT operation. func (c Client) PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int64, err error) { return c.PutObjectWithProgress(bucketName, objectName, reader, contentType, nil) } @@ -201,7 +210,7 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, hashAlgos := make(map[string]hash.Hash) hashSums := make(map[string][]byte) hashAlgos["md5"] = md5.New() - if c.signature.isV4() && !c.secure { + if c.overrideSignerType.IsV4() && !c.secure { hashAlgos["sha256"] = sha256.New() } @@ -230,10 +239,8 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader, reader = tmpFile } // Return error if its not io.EOF. - if err != nil { - if err != io.EOF { - return 0, err - } + if err != nil && err != io.EOF { + return 0, err } // Execute put object. st, err := c.putObjectDo(bucketName, objectName, reader, hashSums["md5"], hashSums["sha256"], size, metaData) diff --git a/vendor/github.com/minio/minio-go/api-remove.go b/vendor/github.com/minio/minio-go/api-remove.go index 2ca84458e..73790f002 100644 --- a/vendor/github.com/minio/minio-go/api-remove.go +++ b/vendor/github.com/minio/minio-go/api-remove.go @@ -35,7 +35,8 @@ func (c Client) RemoveBucket(bucketName string) error { } // Execute DELETE on bucket. resp, err := c.executeMethod("DELETE", requestMetadata{ - bucketName: bucketName, + bucketName: bucketName, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { @@ -64,8 +65,9 @@ func (c Client) RemoveObject(bucketName, objectName string) error { } // Execute DELETE on objectName. resp, err := c.executeMethod("DELETE", requestMetadata{ - bucketName: bucketName, - objectName: objectName, + bucketName: bucketName, + objectName: objectName, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { @@ -208,7 +210,6 @@ func (c Client) RemoveObjects(bucketName string, objectsCh <-chan string) <-chan } // RemoveIncompleteUpload aborts an partially uploaded object. -// Requires explicit authentication, no anonymous requests are allowed for multipart API. func (c Client) RemoveIncompleteUpload(bucketName, objectName string) error { // Input validation. if err := isValidBucketName(bucketName); err != nil { @@ -249,9 +250,10 @@ func (c Client) abortMultipartUpload(bucketName, objectName, uploadID string) er // Execute DELETE on multipart upload. resp, err := c.executeMethod("DELETE", requestMetadata{ - bucketName: bucketName, - objectName: objectName, - queryValues: urlValues, + bucketName: bucketName, + objectName: objectName, + queryValues: urlValues, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { diff --git a/vendor/github.com/minio/minio-go/api-s3-datatypes.go b/vendor/github.com/minio/minio-go/api-s3-datatypes.go index a34f82e97..ec63d6b94 100644 --- a/vendor/github.com/minio/minio-go/api-s3-datatypes.go +++ b/vendor/github.com/minio/minio-go/api-s3-datatypes.go @@ -41,8 +41,8 @@ type commonPrefix struct { Prefix string } -// listBucketResult container for listObjects V2 response. -type listBucketV2Result struct { +// ListBucketV2Result container for listObjects response version 2. +type ListBucketV2Result struct { // A response can contain CommonPrefixes only if you have // specified a delimiter. CommonPrefixes []commonPrefix @@ -70,8 +70,8 @@ type listBucketV2Result struct { StartAfter string } -// listBucketResult container for listObjects response. -type listBucketResult struct { +// ListBucketResult container for listObjects response. +type ListBucketResult struct { // A response can contain CommonPrefixes only if you have // specified a delimiter. CommonPrefixes []commonPrefix @@ -102,8 +102,8 @@ type listBucketResult struct { Prefix string } -// listMultipartUploadsResult container for ListMultipartUploads response -type listMultipartUploadsResult struct { +// ListMultipartUploadsResult container for ListMultipartUploads response +type ListMultipartUploadsResult struct { Bucket string KeyMarker string UploadIDMarker string `xml:"UploadIdMarker"` @@ -131,8 +131,8 @@ type copyObjectResult struct { LastModified string // time string format "2006-01-02T15:04:05.000Z" } -// objectPart container for particular part of an object. -type objectPart struct { +// ObjectPart container for particular part of an object. +type ObjectPart struct { // Part number identifies the part. PartNumber int @@ -147,8 +147,8 @@ type objectPart struct { Size int64 } -// listObjectPartsResult container for ListObjectParts response. -type listObjectPartsResult struct { +// ListObjectPartsResult container for ListObjectParts response. +type ListObjectPartsResult struct { Bucket string Key string UploadID string `xml:"UploadId"` @@ -163,7 +163,7 @@ type listObjectPartsResult struct { // Indicates whether the returned list of parts is truncated. IsTruncated bool - ObjectParts []objectPart `xml:"Part"` + ObjectParts []ObjectPart `xml:"Part"` EncodingType string } @@ -185,9 +185,9 @@ type completeMultipartUploadResult struct { ETag string } -// completePart sub container lists individual part numbers and their +// CompletePart sub container lists individual part numbers and their // md5sum, part of completeMultipartUpload. -type completePart struct { +type CompletePart struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ Part" json:"-"` // Part number identifies the part. @@ -198,7 +198,7 @@ type completePart struct { // completeMultipartUpload container for completing multipart upload. type completeMultipartUpload struct { XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUpload" json:"-"` - Parts []completePart `xml:"Part"` + Parts []CompletePart `xml:"Part"` } // createBucketConfiguration container for bucket configuration. diff --git a/vendor/github.com/minio/minio-go/api-stat.go b/vendor/github.com/minio/minio-go/api-stat.go index e3bb115d4..5b3dfe1b4 100644 --- a/vendor/github.com/minio/minio-go/api-stat.go +++ b/vendor/github.com/minio/minio-go/api-stat.go @@ -34,7 +34,8 @@ func (c Client) BucketExists(bucketName string) (bool, error) { // Execute HEAD on bucketName. resp, err := c.executeMethod("HEAD", requestMetadata{ - bucketName: bucketName, + bucketName: bucketName, + contentSHA256Bytes: emptySHA256, }) defer closeResponse(resp) if err != nil { @@ -85,11 +86,31 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { if err := isValidObjectName(objectName); err != nil { return ObjectInfo{}, err } + reqHeaders := NewHeadReqHeaders() + return c.statObject(bucketName, objectName, reqHeaders) +} + +// Lower level API for statObject supporting pre-conditions and range headers. +func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHeaders) (ObjectInfo, error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return ObjectInfo{}, err + } + if err := isValidObjectName(objectName); err != nil { + return ObjectInfo{}, err + } + + customHeader := make(http.Header) + for k, v := range reqHeaders.Header { + customHeader[k] = v + } // Execute HEAD on objectName. resp, err := c.executeMethod("HEAD", requestMetadata{ - bucketName: bucketName, - objectName: objectName, + bucketName: bucketName, + objectName: objectName, + contentSHA256Bytes: emptySHA256, + customHeader: customHeader, }) defer closeResponse(resp) if err != nil { @@ -122,6 +143,7 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { } } } + // Parse Last-Modified has http time format. date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { @@ -135,6 +157,7 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { Region: resp.Header.Get("x-amz-bucket-region"), } } + // Fetch content type if any present. contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) if contentType == "" { diff --git a/vendor/github.com/minio/minio-go/api.go b/vendor/github.com/minio/minio-go/api.go index a21c40e80..c04034a86 100644 --- a/vendor/github.com/minio/minio-go/api.go +++ b/vendor/github.com/minio/minio-go/api.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,6 +21,7 @@ import ( "bytes" "encoding/base64" "encoding/hex" + "errors" "fmt" "io" "io/ioutil" @@ -34,6 +36,7 @@ import ( "sync" "time" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/s3signer" "github.com/minio/minio-go/pkg/s3utils" ) @@ -45,14 +48,11 @@ type Client struct { // Parsed endpoint url provided by the user. endpointURL url.URL - // AccessKeyID required for authorized requests. - accessKeyID string - // SecretAccessKey required for authorized requests. - secretAccessKey string - // Choose a signature type if necessary. - signature SignatureType - // Set to 'true' if Client has no access and secret keys. - anonymous bool + // Holds various credential providers. + credsProvider *credentials.Credentials + + // Custom signerType value overrides all credentials. + overrideSignerType credentials.SignatureType // User supplied. appInfo struct { @@ -74,6 +74,9 @@ type Client struct { // S3 specific accelerated endpoint. s3AccelerateEndpoint string + // Region endpoint + region string + // Random seed. random *rand.Rand } @@ -81,7 +84,7 @@ type Client struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "2.0.4" + libraryVersion = "2.1.0" ) // User Agent should always following the below style. @@ -96,53 +99,67 @@ const ( // NewV2 - instantiate minio client with Amazon S3 signature version // '2' compatibility. func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, secure) + creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "") if err != nil { return nil, err } - // Set to use signature version '2'. - clnt.signature = SignatureV2 + clnt.overrideSignerType = credentials.SignatureV2 return clnt, nil } // NewV4 - instantiate minio client with Amazon S3 signature version // '4' compatibility. func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, secure) + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "") if err != nil { return nil, err } - // Set to use signature version '4'. - clnt.signature = SignatureV4 + clnt.overrideSignerType = credentials.SignatureV4 return clnt, nil } -// New - instantiate minio client Client, adds automatic verification -// of signature. -func New(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, secure) +// New - instantiate minio client, adds automatic verification of signature. +func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "") if err != nil { return nil, err } // Google cloud storage should be set to signature V2, force it if not. if s3utils.IsGoogleEndpoint(clnt.endpointURL) { - clnt.signature = SignatureV2 + clnt.overrideSignerType = credentials.SignatureV2 } - // If Amazon S3 set to signature v2.n + // If Amazon S3 set to signature v4. if s3utils.IsAmazonEndpoint(clnt.endpointURL) { - clnt.signature = SignatureV4 + clnt.overrideSignerType = credentials.SignatureV4 } return clnt, nil } +// NewWithCredentials - instantiate minio client with credentials provider +// for retrieving credentials from various credentials provider such as +// IAM, File, Env etc. +func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { + return privateNew(endpoint, creds, secure, region) +} + +// NewWithRegion - instantiate minio client, with region configured. Unlike New(), +// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. +// Use this function when if your application deals with single region. +func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + return privateNew(endpoint, creds, secure, region) +} + // lockedRandSource provides protected rand source, implements rand.Source interface. type lockedRandSource struct { lk sync.Mutex src rand.Source } -// Int63 returns a non-negative pseudo-random 63-bit integer as an -// int64. +// Int63 returns a non-negative pseudo-random 63-bit integer as an int64. func (r *lockedRandSource) Int63() (n int64) { r.lk.Lock() n = r.src.Int63() @@ -170,7 +187,7 @@ func redirectHeaders(req *http.Request, via []*http.Request) error { return nil } -func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { +func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, secure) if err != nil { @@ -179,11 +196,9 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Cl // instantiate new Client. clnt := new(Client) - clnt.accessKeyID = accessKeyID - clnt.secretAccessKey = secretAccessKey - if clnt.accessKeyID == "" || clnt.secretAccessKey == "" { - clnt.anonymous = true - } + + // Save the credentials. + clnt.credsProvider = creds // Remember whether we are using https or not clnt.secure = secure @@ -197,7 +212,10 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Cl CheckRedirect: redirectHeaders, } - // Instantiae bucket location cache. + // Sets custom region, if region is empty bucket location cache is used automatically. + clnt.region = region + + // Instantiate bucket location cache. clnt.bucketLocCache = newBucketLocationCache() // Introduce a new locked random seed. @@ -299,12 +317,12 @@ var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)") // Filter out signature value from Authorization header. func (c Client) filterSignature(req *http.Request) { - // For anonymous requests, no need to filter. - if c.anonymous { + origAuth := req.Header.Get("Authorization") + if origAuth != "" { return } - // Handle if Signature V2. - if c.signature.isV2() { + + if !strings.HasPrefix(origAuth, signV4Algorithm) { // Set a temporary redacted auth req.Header.Set("Authorization", "AWS **REDACTED**:**REDACTED**") return @@ -312,8 +330,6 @@ func (c Client) filterSignature(req *http.Request) { /// Signature V4 authorization header. - // Save the original auth. - origAuth := req.Header.Get("Authorization") // Strip out accessKeyID from: // Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") @@ -323,6 +339,7 @@ func (c Client) filterSignature(req *http.Request) { // Set a temporary redacted auth req.Header.Set("Authorization", newAuth) + return } @@ -411,7 +428,7 @@ func (c Client) do(req *http.Request) (*http.Response, error) { return nil, &url.Error{ Op: urlErr.Op, URL: urlErr.URL, - Err: fmt.Errorf("Connection closed by foreign host %s. Retry again.", urlErr.URL), + Err: errors.New("Connection closed by foreign host " + urlErr.URL + ". Retry again."), } } return nil, err @@ -460,9 +477,13 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt if metadata.contentBody != nil { // Check if body is seekable then it is retryable. bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) + switch bodySeeker { + case os.Stdin, os.Stdout, os.Stderr: + isRetryable = false + } } - // Create a done channel to control 'ListObjects' go routine. + // Create a done channel to control 'newRetryTimer' go routine. doneCh := make(chan struct{}, 1) // Indicate to our routine to exit cleanly upon return. @@ -471,7 +492,7 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Blank indentifier is kept here on purpose since 'range' without // blank identifiers is only supported since go1.4 // https://golang.org/doc/go1.4#forrange. - for _ = range c.newRetryTimer(MaxRetry, time.Second, time.Second*30, MaxJitter, doneCh) { + for _ = range c.newRetryTimer(MaxRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { // Retry executes the following function body if request has an // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a @@ -520,15 +541,22 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt if err != nil { return nil, err } + // Save the body. errBodySeeker := bytes.NewReader(errBodyBytes) res.Body = ioutil.NopCloser(errBodySeeker) // For errors verify if its retryable otherwise fail quickly. errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName)) - // Bucket region if set in error response, we can retry the - // request with the new region. - if errResponse.Region != "" { + + // Save the body back again. + errBodySeeker.Seek(0, 0) // Seek back to starting point. + res.Body = ioutil.NopCloser(errBodySeeker) + + // Bucket region if set in error response and the error + // code dictates invalid region, we can retry the request + // with the new region. + if res.StatusCode == http.StatusBadRequest && errResponse.Region != "" { c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) continue // Retry. } @@ -543,10 +571,6 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt continue // Retry. } - // Save the body back again. - errBodySeeker.Seek(0, 0) // Seek back to starting point. - res.Body = ioutil.NopCloser(errBodySeeker) - // For all other cases break out of the retry loop. break } @@ -588,39 +612,50 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } // Initialize a new HTTP request for the method. - req, err = http.NewRequest(method, targetURL.String(), nil) + req, err = http.NewRequest(method, targetURL.String(), metadata.contentBody) if err != nil { return nil, err } + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + // Generate presign url if needed, return right here. if metadata.expires != 0 && metadata.presignURL { - if c.anonymous { - return nil, ErrInvalidArgument("Requests cannot be presigned with anonymous credentials.") + if signerType.IsAnonymous() { + return nil, ErrInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") } - if c.signature.isV2() { + if signerType.IsV2() { // Presign URL with signature v2. - req = s3signer.PreSignV2(*req, c.accessKeyID, c.secretAccessKey, metadata.expires) - } else { + req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires) + } else if signerType.IsV4() { // Presign URL with signature v4. - req = s3signer.PreSignV4(*req, c.accessKeyID, c.secretAccessKey, location, metadata.expires) + req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) } return req, nil } - // Set content body if available. - if metadata.contentBody != nil { - req.Body = ioutil.NopCloser(metadata.contentBody) - } - - // FIXME: Enable this when Google Cloud Storage properly supports 100-continue. - // Skip setting 'expect' header for Google Cloud Storage, there - // are some known issues - https://github.com/restic/restic/issues/520 - if !s3utils.IsGoogleEndpoint(c.endpointURL) && c.s3AccelerateEndpoint == "" { - // Set 'Expect' header for the request. - req.Header.Set("Expect", "100-continue") - } - // Set 'User-Agent' header for the request. c.setUserAgent(req) @@ -634,37 +669,33 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R req.ContentLength = metadata.contentLength } - // Set sha256 sum only for non anonymous credentials. - if !c.anonymous { - // set sha256 sum for signature calculation only with - // signature version '4'. - if c.signature.isV4() { - shaHeader := unsignedPayload - if !c.secure { - if metadata.contentSHA256Bytes == nil { - shaHeader = hex.EncodeToString(sum256([]byte{})) - } else { - shaHeader = hex.EncodeToString(metadata.contentSHA256Bytes) - } - } - req.Header.Set("X-Amz-Content-Sha256", shaHeader) - } - } - // set md5Sum for content protection. if metadata.contentMD5Bytes != nil { req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(metadata.contentMD5Bytes)) } - // Sign the request for all authenticated requests. - if !c.anonymous { - if c.signature.isV2() { - // Add signature version '2' authorization header. - req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey) - } else if c.signature.isV4() { - // Add signature version '4' authorization header. - req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, location) + // For anonymous requests just return. + if signerType.IsAnonymous() { + return req, nil + } + + switch { + case signerType.IsV2(): + // Add signature version '2' authorization header. + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + case signerType.IsStreamingV4() && method == "PUT": + req = s3signer.StreamingSignV4(req, accessKeyID, + secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) + default: + // Set sha256 sum for signature calculation only with signature version '4'. + shaHeader := unsignedPayload + if len(metadata.contentSHA256Bytes) > 0 { + shaHeader = hex.EncodeToString(metadata.contentSHA256Bytes) } + req.Header.Set("X-Amz-Content-Sha256", shaHeader) + + // Add signature version '4' authorization header. + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) } // Return request. @@ -726,13 +757,16 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que } } } + // If there are any query values, add them to the end. if len(queryValues) > 0 { urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) } + u, err := url.Parse(urlStr) if err != nil { return nil, err } + return u, nil } diff --git a/vendor/github.com/minio/minio-go/api_functional_v2_test.go b/vendor/github.com/minio/minio-go/api_functional_v2_test.go index f41cc0ff4..7e5933778 100644 --- a/vendor/github.com/minio/minio-go/api_functional_v2_test.go +++ b/vendor/github.com/minio/minio-go/api_functional_v2_test.go @@ -36,6 +36,9 @@ func TestMakeBucketErrorV2(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for short runs") } + if os.Getenv("S3_ADDRESS") != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -198,14 +201,14 @@ func TestRemovePartiallyUploadedV2(t *testing.T) { go func() { i := 0 for i < 25 { - _, err = io.CopyN(writer, r, 128*1024) - if err != nil { - t.Fatal("Error:", err, bucketName) + _, cerr := io.CopyN(writer, r, 128*1024) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) } i++ r.Seek(0, 0) } - writer.CloseWithError(errors.New("Proactively closed to be verified later.")) + writer.CloseWithError(errors.New("proactively closed to be verified later")) }() objectName := bucketName + "-resumable" @@ -213,7 +216,7 @@ func TestRemovePartiallyUploadedV2(t *testing.T) { if err == nil { t.Fatal("Error: PutObject should fail.") } - if err.Error() != "Proactively closed to be verified later." { + if err.Error() != "proactively closed to be verified later" { t.Fatal("Error:", err) } err = c.RemoveIncompleteUpload(bucketName, objectName) @@ -571,6 +574,9 @@ func TestMakeBucketRegionsV2(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for short runs") } + if os.Getenv("S3_ADDRESS") != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) diff --git a/vendor/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/github.com/minio/minio-go/api_functional_v4_test.go index 426d2ddcc..a553ea2cd 100644 --- a/vendor/github.com/minio/minio-go/api_functional_v4_test.go +++ b/vendor/github.com/minio/minio-go/api_functional_v4_test.go @@ -18,7 +18,7 @@ package minio import ( "bytes" - crand "crypto/rand" + "encoding/hex" "errors" "fmt" "io" @@ -28,9 +28,11 @@ import ( "net/url" "os" "strconv" + "strings" "testing" "time" + "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/policy" ) @@ -64,6 +66,9 @@ func TestMakeBucketError(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for short runs") } + if os.Getenv("S3_ADDRESS") != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -110,6 +115,9 @@ func TestMakeBucketRegions(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for short runs") } + if os.Getenv("S3_ADDRESS") != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -192,14 +200,10 @@ func TestPutObjectReadAt(t *testing.T) { } // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. - buf := make([]byte, minPartSize*4) - // Use crand.Reader for multipart tests to ensure part order at the end. - size, err := io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) + // Use different data for each part for multipart tests to ensure part order at the end. + var buf []byte + for i := 0; i < 4; i++ { + buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } // Save the data @@ -286,14 +290,10 @@ func TestPutObjectWithMetadata(t *testing.T) { } // Generate data using 2 parts - buf := make([]byte, minPartSize*2) - // Use crand.Reader for multipart tests to ensure part order at the end. - size, err := io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*2 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*2, size) + // Use different data in each part for multipart tests to ensure part order at the end. + var buf []byte + for i := 0; i < 2; i++ { + buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } // Save the data @@ -302,7 +302,7 @@ func TestPutObjectWithMetadata(t *testing.T) { // Object custom metadata customContentType := "custom/contenttype" - n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{"Content-Type": []string{customContentType}}, nil) + n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{"Content-Type": {customContentType}}, nil) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -346,6 +346,70 @@ func TestPutObjectWithMetadata(t *testing.T) { } } +// Test put object with streaming signature. +func TestPutObjectStreaming(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), + "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Upload an object. + sizes := []int64{0, 64*1024 - 1, 64 * 1024} + objectName := "test-object" + for i, size := range sizes { + data := bytes.Repeat([]byte("a"), int(size)) + n, err := c.PutObjectStreaming(bucketName, objectName, bytes.NewReader(data)) + if err != nil { + t.Fatalf("Test %d Error: %v %s %s", i+1, err, bucketName, objectName) + } + + if n != size { + t.Errorf("Test %d Expected upload object size %d but got %d", i+1, size, n) + } + } + + // Remove the object. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + // Remove the bucket. + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + // Test listing partially uploaded objects. func TestListPartiallyUploaded(t *testing.T) { if testing.Short() { @@ -387,17 +451,14 @@ func TestListPartiallyUploaded(t *testing.T) { go func() { i := 0 for i < 25 { - _, err = io.CopyN(writer, r, (minPartSize*2)/25) - if err != nil { - t.Fatal("Error:", err, bucketName) + _, cerr := io.CopyN(writer, r, (minPartSize*2)/25) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) } i++ r.Seek(0, 0) } - err := writer.CloseWithError(errors.New("Proactively closed to be verified later.")) - if err != nil { - t.Fatal("Error:", err) - } + writer.CloseWithError(errors.New("proactively closed to be verified later")) }() objectName := bucketName + "-resumable" @@ -405,7 +466,7 @@ func TestListPartiallyUploaded(t *testing.T) { if err == nil { t.Fatal("Error: PutObject should fail.") } - if err.Error() != "Proactively closed to be verified later." { + if err.Error() != "proactively closed to be verified later" { t.Fatal("Error:", err) } @@ -650,7 +711,8 @@ func TestRemoveMultipleObjects(t *testing.T) { objectName := "sample" + strconv.Itoa(i) + ".txt" _, err = c.PutObject(bucketName, objectName, r, "application/octet-stream") if err != nil { - t.Fatal("Error: PutObject shouldn't fail.") + t.Error("Error: PutObject shouldn't fail.", err) + continue } objectsCh <- objectName } @@ -715,17 +777,14 @@ func TestRemovePartiallyUploaded(t *testing.T) { go func() { i := 0 for i < 25 { - _, err = io.CopyN(writer, r, 128*1024) - if err != nil { - t.Fatal("Error:", err, bucketName) + _, cerr := io.CopyN(writer, r, 128*1024) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) } i++ r.Seek(0, 0) } - err := writer.CloseWithError(errors.New("Proactively closed to be verified later.")) - if err != nil { - t.Fatal("Error:", err) - } + writer.CloseWithError(errors.New("proactively closed to be verified later")) }() objectName := bucketName + "-resumable" @@ -733,7 +792,7 @@ func TestRemovePartiallyUploaded(t *testing.T) { if err == nil { t.Fatal("Error: PutObject should fail.") } - if err.Error() != "Proactively closed to be verified later." { + if err.Error() != "proactively closed to be verified later" { t.Fatal("Error:", err) } err = c.RemoveIncompleteUpload(bucketName, objectName) @@ -788,7 +847,6 @@ func TestResumablePutObject(t *testing.T) { t.Fatal("Error:", err) } r := bytes.NewReader(bytes.Repeat([]byte("b"), minPartSize*2)) - // Copy 11MiB worth of random data. n, err := io.CopyN(file, r, minPartSize*2) if err != nil { t.Fatal("Error:", err) @@ -904,16 +962,13 @@ func TestResumableFPutObject(t *testing.T) { } // Upload 4 parts to use all 3 multipart 'workers' and have an extra part. - buffer := make([]byte, minPartSize*4) - // Use crand.Reader for multipart tests to ensure parts are uploaded in correct order. - size, err := io.ReadFull(crand.Reader, buffer) - if err != nil { - t.Fatal("Error:", err) + // Use different data in each part for multipart tests to ensure parts are uploaded in correct order. + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) - } - size, err = file.Write(buffer) + + size, err := file.Write(buffer) if err != nil { t.Fatal("Error:", err) } @@ -995,16 +1050,12 @@ func TestFPutObjectMultipart(t *testing.T) { } // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - buffer := make([]byte, minPartSize*4) - - size, err := io.ReadFull(crand.Reader, buffer) - if err != nil { - t.Fatal("Error:", err) + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) - } - size, err = file.Write(buffer) + + size, err := file.Write(buffer) if err != nil { t.Fatal("Error:", err) } @@ -1100,18 +1151,14 @@ func TestFPutObject(t *testing.T) { } // Upload 4 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. - buffer := make([]byte, minPartSize*4) - // Use random data for multipart tests to check parts are uploaded in correct order. - size, err := io.ReadFull(crand.Reader, buffer) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } // Write the data to the file. - size, err = file.Write(buffer) + size, err := file.Write(buffer) if err != nil { t.Fatal("Error:", err) } @@ -1784,10 +1831,178 @@ func TestCopyObject(t *testing.T) { } } +// TestEncryptionPutGet tests client side encryption +func TestEncryptionPutGet(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a symmetric key + symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + + // Generate an assymmetric key from predefine public and private certificates + privateKey, err := hex.DecodeString( + "30820277020100300d06092a864886f70d0101010500048202613082025d" + + "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + + "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + + "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + + "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + + "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + + "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + + "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + + "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + + "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + + "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + + "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + + "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + + "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + + "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + + "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + + "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + + "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + + "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + + "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + + "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + + "9945cb5c7d") + + if err != nil { + t.Fatal(err) + } + + publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + + "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + + "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + + "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + + "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + + "80a89e43f29b570203010001") + if err != nil { + t.Fatal(err) + } + + // Generate an asymmetric key + asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) + if err != nil { + t.Fatal(err) + } + + // Object custom metadata + customContentType := "custom/contenttype" + + testCases := []struct { + buf []byte + encKey encrypt.Key + }{ + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + // Secured object + cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) + if err != nil { + t.Fatal(err) + } + + // Put encrypted data + _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials, map[string][]string{"Content-Type": {customContentType}}, nil) + if err != nil { + t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + + // Read the data back + r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) + if err != nil { + t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + t.Fatalf("Test %d, error: %v", i+1, err) + } + if recvBuffer.Len() != len(testCase.buf) { + t.Fatalf("Test %d, error: number of bytes of received object does not match, want %v, got %v\n", + i+1, len(testCase.buf), recvBuffer.Len()) + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + t.Fatalf("Test %d, error: Encrypted sent is not equal to decrypted, want `%x`, go `%x`", i+1, testCase.buf, recvBuffer.Bytes()) + } + + // Remove test object + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatalf("Test %d, error: %v", i+1, err) + } + + } + + // Remove test bucket + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + +} + func TestBucketNotification(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for the short runs") } + if os.Getenv("NOTIFY_BUCKET") == "" || + os.Getenv("NOTIFY_SERVICE") == "" || + os.Getenv("NOTIFY_REGION") == "" || + os.Getenv("NOTIFY_ACCOUNTID") == "" || + os.Getenv("NOTIFY_RESOURCE") == "" { + t.Skip("skipping notification test if not configured") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -2187,3 +2402,172 @@ func TestFunctional(t *testing.T) { t.Fatal("Error: ", err) } } + +// Test for validating GetObject Reader* methods functioning when the +// object is modified in the object store. +func TestGetObjectObjectModified(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + // Upload an object. + objectName := "myobject" + content := "helloworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), "application/text") + if err != nil { + t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + defer c.RemoveObject(bucketName, objectName) + + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatalf("Failed to get object %s/%s: %v", bucketName, objectName, err) + } + defer reader.Close() + + // Read a few bytes of the object. + b := make([]byte, 5) + n, err := reader.ReadAt(b, 0) + if err != nil { + t.Fatalf("Failed to read object %s/%s at an offset: %v", bucketName, objectName, err) + } + + // Upload different contents to the same object while object is being read. + newContent := "goodbyeworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), "application/text") + if err != nil { + t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + // Confirm that a Stat() call in between doesn't change the Object's cached etag. + _, err = reader.Stat() + if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { + t.Errorf("Expected Stat to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) + } + + // Read again only to find object contents have been modified since last read. + _, err = reader.ReadAt(b, int64(n)) + if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { + t.Errorf("Expected ReadAt to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) + } +} + +// Test validates putObject to upload a file seeked at a given offset. +func TestPutObjectUploadSeekedObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") + if err != nil { + t.Fatal("Error:", err) + } + + var length = 120000 + data := bytes.Repeat([]byte("1"), length) + + if _, err = tempfile.Write(data); err != nil { + t.Fatal("Error:", err) + } + + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + + offset := length / 2 + if _, err := tempfile.Seek(int64(offset), 0); err != nil { + t.Fatal("Error:", err) + } + + n, err := c.PutObject(bucketName, objectName, tempfile, "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(length-offset) { + t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + tempfile.Close() + if err = os.Remove(tempfile.Name()); err != nil { + t.Fatal("Error:", err) + } + + length = int(n) + + obj, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + n, err = obj.Seek(int64(offset), 0) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(offset) { + t.Fatalf("Invalid offset returned, want %v, got %v", int64(offset), n) + } + + n, err = c.PutObject(bucketName, objectName+"getobject", obj, "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(length-offset) { + t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + + if err = c.RemoveObject(bucketName, objectName); err != nil { + t.Fatal("Error:", err) + } + + if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { + t.Fatal("Error:", err) + } +} diff --git a/vendor/github.com/minio/minio-go/api_unit_test.go b/vendor/github.com/minio/minio-go/api_unit_test.go index c1db0df5d..7bb5802bf 100644 --- a/vendor/github.com/minio/minio-go/api_unit_test.go +++ b/vendor/github.com/minio/minio-go/api_unit_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -25,6 +26,7 @@ import ( "strings" "testing" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/policy" ) @@ -228,18 +230,18 @@ func TestErrorResponse(t *testing.T) { // Tests signature type. func TestSignatureType(t *testing.T) { clnt := Client{} - if !clnt.signature.isV4() { + if !clnt.overrideSignerType.IsV4() { t.Fatal("Error") } - clnt.signature = SignatureV2 - if !clnt.signature.isV2() { + clnt.overrideSignerType = credentials.SignatureV2 + if !clnt.overrideSignerType.IsV2() { t.Fatal("Error") } - if clnt.signature.isV4() { + if clnt.overrideSignerType.IsV4() { t.Fatal("Error") } - clnt.signature = SignatureV4 - if !clnt.signature.isV4() { + clnt.overrideSignerType = credentials.SignatureV4 + if !clnt.overrideSignerType.IsV4() { t.Fatal("Error") } } diff --git a/vendor/github.com/minio/minio-go/appveyor.yml b/vendor/github.com/minio/minio-go/appveyor.yml index a5dc2b226..4f5c1b390 100644 --- a/vendor/github.com/minio/minio-go/appveyor.yml +++ b/vendor/github.com/minio/minio-go/appveyor.yml @@ -17,6 +17,8 @@ install: - go version - go env - go get -u github.com/golang/lint/golint + - go get -u github.com/go-ini/ini + - go get -u github.com/minio/go-homedir - go get -u github.com/remyoudompheng/go-misc/deadcode - go get -u github.com/gordonklaus/ineffassign @@ -24,7 +26,7 @@ install: build_script: - go vet ./... - gofmt -s -l . - - golint github.com/minio/minio-go... + - golint -set_exit_status github.com/minio/minio-go... - deadcode - ineffassign . - go test -short -v diff --git a/vendor/github.com/minio/minio-go/bucket-cache.go b/vendor/github.com/minio/minio-go/bucket-cache.go index c35e26b7c..7e7cc7717 100644 --- a/vendor/github.com/minio/minio-go/bucket-cache.go +++ b/vendor/github.com/minio/minio-go/bucket-cache.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -21,9 +22,9 @@ import ( "net/http" "net/url" "path" - "strings" "sync" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/s3signer" "github.com/minio/minio-go/pkg/s3utils" ) @@ -84,9 +85,6 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { if err := isValidBucketName(bucketName); err != nil { return "", err } - if location, ok := c.bucketLocCache.Get(bucketName); ok { - return location, nil - } if s3utils.IsAmazonChinaEndpoint(c.endpointURL) { // For china specifically we need to set everything to @@ -96,6 +94,15 @@ func (c Client) getBucketLocation(bucketName string) (string, error) { return "cn-north-1", nil } + // Region set then no need to fetch bucket location. + if c.region != "" { + return c.region, nil + } + + if location, ok := c.bucketLocCache.Get(bucketName); ok { + return location, nil + } + // Initialize a new request. req, err := c.getBucketLocationRequest(bucketName) if err != nil { @@ -125,7 +132,7 @@ func processBucketLocationResponse(resp *http.Response, bucketName string) (buck // For access denied error, it could be an anonymous // request. Move forward and let the top level callers // succeed if possible based on their policy. - if errResp.Code == "AccessDenied" && strings.Contains(errResp.Message, "Access Denied") { + if errResp.Code == "AccessDenied" { return "us-east-1", nil } return "", err @@ -176,8 +183,33 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro // Set UserAgent for the request. c.setUserAgent(req) + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } + // Set sha256 sum for signature calculation only with signature version '4'. - if c.signature.isV4() { + switch { + case signerType.IsV4(): var contentSha256 string if c.secure { contentSha256 = unsignedPayload @@ -185,13 +217,10 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro contentSha256 = hex.EncodeToString(sum256([]byte{})) } req.Header.Set("X-Amz-Content-Sha256", contentSha256) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + case signerType.IsV2(): + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) } - // Sign the request. - if c.signature.isV4() { - req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") - } else if c.signature.isV2() { - req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey) - } return req, nil } diff --git a/vendor/github.com/minio/minio-go/bucket-cache_test.go b/vendor/github.com/minio/minio-go/bucket-cache_test.go index 0c068c966..6ae4e7be4 100644 --- a/vendor/github.com/minio/minio-go/bucket-cache_test.go +++ b/vendor/github.com/minio/minio-go/bucket-cache_test.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +28,7 @@ import ( "reflect" "testing" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/s3signer" ) @@ -86,17 +88,46 @@ func TestGetBucketLocationRequest(t *testing.T) { // Set UserAgent for the request. c.setUserAgent(req) - // Set sha256 sum for signature calculation only with signature version '4'. - if c.signature.isV4() { - req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{}))) + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType } - // Sign the request. - if c.signature.isV4() { - req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1") - } else if c.signature.isV2() { - req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey) + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous } + + // Set sha256 sum for signature calculation only + // with signature version '4'. + switch { + case signerType.IsV4(): + var contentSha256 string + if c.secure { + contentSha256 = unsignedPayload + } else { + contentSha256 = hex.EncodeToString(sum256([]byte{})) + } + req.Header.Set("X-Amz-Content-Sha256", contentSha256) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1") + case signerType.IsV2(): + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + } + return req, nil } diff --git a/vendor/github.com/minio/minio-go/bucket-notification.go b/vendor/github.com/minio/minio-go/bucket-notification.go index 4f60f1c8b..5ac52e5f7 100644 --- a/vendor/github.com/minio/minio-go/bucket-notification.go +++ b/vendor/github.com/minio/minio-go/bucket-notification.go @@ -28,10 +28,13 @@ type NotificationEventType string // http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations const ( ObjectCreatedAll NotificationEventType = "s3:ObjectCreated:*" - ObjectCreatePut = "s3:ObjectCreated:Put" + ObjectCreatedPut = "s3:ObjectCreated:Put" ObjectCreatedPost = "s3:ObjectCreated:Post" ObjectCreatedCopy = "s3:ObjectCreated:Copy" - ObjectCreatedCompleteMultipartUpload = "sh:ObjectCreated:CompleteMultipartUpload" + ObjectCreatedCompleteMultipartUpload = "s3:ObjectCreated:CompleteMultipartUpload" + ObjectAccessedGet = "s3:ObjectAccessed:Get" + ObjectAccessedHead = "s3:ObjectAccessed:Head" + ObjectAccessedAll = "s3:ObjectAccessed:*" ObjectRemovedAll = "s3:ObjectRemoved:*" ObjectRemovedDelete = "s3:ObjectRemoved:Delete" ObjectRemovedDeleteMarkerCreated = "s3:ObjectRemoved:DeleteMarkerCreated" diff --git a/vendor/github.com/minio/minio-go/constants.go b/vendor/github.com/minio/minio-go/constants.go index 057c3eef4..6055bfdad 100644 --- a/vendor/github.com/minio/minio-go/constants.go +++ b/vendor/github.com/minio/minio-go/constants.go @@ -45,8 +45,18 @@ const optimalReadBufferSize = 1024 * 1024 * 5 // we don't want to sign the request payload const unsignedPayload = "UNSIGNED-PAYLOAD" +// Total number of parallel workers used for multipart operation. +var totalWorkers = 3 + // Signature related constants. const ( signV4Algorithm = "AWS4-HMAC-SHA256" iso8601DateFormat = "20060102T150405Z" ) + +// Encryption headers stored along with the object. +const ( + amzHeaderIV = "X-Amz-Meta-X-Amz-Iv" + amzHeaderKey = "X-Amz-Meta-X-Amz-Key" + amzHeaderMatDesc = "X-Amz-Meta-X-Amz-Matdesc" +) diff --git a/vendor/github.com/minio/minio-go/core.go b/vendor/github.com/minio/minio-go/core.go new file mode 100644 index 000000000..be9388cec --- /dev/null +++ b/vendor/github.com/minio/minio-go/core.go @@ -0,0 +1,113 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "io" + + "github.com/minio/minio-go/pkg/policy" +) + +// Core - Inherits Client and adds new methods to expose the low level S3 APIs. +type Core struct { + *Client +} + +// NewCore - Returns new initialized a Core client, this CoreClient should be +// only used under special conditions such as need to access lower primitives +// and being able to use them to write your own wrappers. +func NewCore(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Core, error) { + var s3Client Core + client, err := NewV4(endpoint, accessKeyID, secretAccessKey, secure) + if err != nil { + return nil, err + } + s3Client.Client = client + return &s3Client, nil +} + +// ListObjects - List all the objects at a prefix, optionally with marker and delimiter +// you can further filter the results. +func (c Core) ListObjects(bucket, prefix, marker, delimiter string, maxKeys int) (result ListBucketResult, err error) { + return c.listObjectsQuery(bucket, prefix, marker, delimiter, maxKeys) +} + +// ListObjectsV2 - Lists all the objects at a prefix, similar to ListObjects() but uses +// continuationToken instead of marker to further filter the results. +func (c Core) ListObjectsV2(bucketName, objectPrefix, continuationToken string, fetchOwner bool, delimiter string, maxkeys int) (ListBucketV2Result, error) { + return c.listObjectsV2Query(bucketName, objectPrefix, continuationToken, fetchOwner, delimiter, maxkeys) +} + +// PutObject - Upload object. Uploads using single PUT call. +func (c Core) PutObject(bucket, object string, size int64, data io.Reader, md5Sum, sha256Sum []byte, metadata map[string][]string) (ObjectInfo, error) { + return c.putObjectDo(bucket, object, data, md5Sum, sha256Sum, size, metadata) +} + +// NewMultipartUpload - Initiates new multipart upload and returns the new uploaID. +func (c Core) NewMultipartUpload(bucket, object string, metadata map[string][]string) (uploadID string, err error) { + result, err := c.initiateMultipartUpload(bucket, object, metadata) + return result.UploadID, err +} + +// ListMultipartUploads - List incomplete uploads. +func (c Core) ListMultipartUploads(bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartUploadsResult, err error) { + return c.listMultipartUploadsQuery(bucket, keyMarker, uploadIDMarker, prefix, delimiter, maxUploads) +} + +// PutObjectPart - Upload an object part. +func (c Core) PutObjectPart(bucket, object, uploadID string, partID int, size int64, data io.Reader, md5Sum, sha256Sum []byte) (ObjectPart, error) { + return c.uploadPart(bucket, object, uploadID, data, partID, md5Sum, sha256Sum, size) +} + +// ListObjectParts - List uploaded parts of an incomplete upload.x +func (c Core) ListObjectParts(bucket, object, uploadID string, partNumberMarker int, maxParts int) (result ListObjectPartsResult, err error) { + return c.listObjectPartsQuery(bucket, object, uploadID, partNumberMarker, maxParts) +} + +// CompleteMultipartUpload - Concatenate uploaded parts and commit to an object. +func (c Core) CompleteMultipartUpload(bucket, object, uploadID string, parts []CompletePart) error { + _, err := c.completeMultipartUpload(bucket, object, uploadID, completeMultipartUpload{Parts: parts}) + return err +} + +// AbortMultipartUpload - Abort an incomplete upload. +func (c Core) AbortMultipartUpload(bucket, object, uploadID string) error { + return c.abortMultipartUpload(bucket, object, uploadID) +} + +// GetBucketPolicy - fetches bucket access policy for a given bucket. +func (c Core) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) { + return c.getBucketPolicy(bucket) +} + +// PutBucketPolicy - applies a new bucket access policy for a given bucket. +func (c Core) PutBucketPolicy(bucket string, bucketPolicy policy.BucketAccessPolicy) error { + return c.putBucketPolicy(bucket, bucketPolicy) +} + +// GetObject is a lower level API implemented to support reading +// partial objects and also downloading objects with special conditions +// matching etag, modtime etc. +func (c Core) GetObject(bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) { + return c.getObject(bucketName, objectName, reqHeaders) +} + +// StatObject is a lower level API implemented to support special +// conditions matching etag, modtime on a request. +func (c Core) StatObject(bucketName, objectName string, reqHeaders RequestHeaders) (ObjectInfo, error) { + return c.statObject(bucketName, objectName, reqHeaders) +} diff --git a/vendor/github.com/minio/minio-go/core_test.go b/vendor/github.com/minio/minio-go/core_test.go new file mode 100644 index 000000000..657ad5260 --- /dev/null +++ b/vendor/github.com/minio/minio-go/core_test.go @@ -0,0 +1,368 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "bytes" + "crypto/md5" + + "io" + "math/rand" + "os" + "reflect" + "testing" + "time" +) + +// Tests for Core GetObject() function. +func TestGetObjectCore(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio core client object. + c, err := NewCore( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.Client.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + reqHeaders := NewGetReqHeaders() + + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, n) + buf4 := make([]byte, 1) + + reqHeaders.SetRange(offset, offset+int64(len(buf1))-1) + reader, objectInfo, err := c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + m, err := io.ReadFull(reader, buf1) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two GetObject from same offset.") + } + offset += 512 + + reqHeaders.SetRange(offset, offset+int64(len(buf2))-1) + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf2) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two GetObject from same offset.") + } + + reqHeaders.SetRange(0, int64(len(buf3))) + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf3) + if err != nil { + reader.Close() + t.Fatal(err) + } + reader.Close() + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf3, buf) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + reqHeaders = NewGetReqHeaders() + reqHeaders.SetMatchETag("etag") + _, _, err = c.GetObject(bucketName, objectName, reqHeaders) + if err == nil { + t.Fatal("Unexpected GetObject should fail with mismatching etags") + } + if errResp := ToErrorResponse(err); errResp.Code != "PreconditionFailed" { + t.Fatalf("Expected \"PreconditionFailed\" as code, got %s instead", errResp.Code) + } + + reqHeaders = NewGetReqHeaders() + reqHeaders.SetMatchETagExcept("etag") + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf3) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + if !bytes.Equal(buf3, buf) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + reqHeaders = NewGetReqHeaders() + reqHeaders.SetRange(0, 0) + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + + m, err = io.ReadFull(reader, buf4) + reader.Close() + if err != nil { + t.Fatal(err) + } + + if objectInfo.Size != int64(m) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, m) + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Tests get bucket policy core API. +func TestGetBucketPolicy(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewCore( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable to debug + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Verify if bucket exits and you have access. + var exists bool + exists, err = c.BucketExists(bucketName) + if err != nil { + t.Fatal("Error:", err, bucketName) + } + if !exists { + t.Fatal("Error: could not find ", bucketName) + } + + // Asserting the default bucket policy. + bucketPolicy, err := c.GetBucketPolicy(bucketName) + if err != nil { + errResp := ToErrorResponse(err) + if errResp.Code != "NoSuchBucketPolicy" { + t.Error("Error:", err, bucketName) + } + } + if !reflect.DeepEqual(bucketPolicy, emptyBucketAccessPolicy) { + t.Errorf("Bucket policy expected %#v, got %#v", emptyBucketAccessPolicy, bucketPolicy) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + +// Test Core PutObject. +func TestCorePutObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewCore( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + buf := bytes.Repeat([]byte("a"), minPartSize) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + // Object content type + objectContentType := "binary/octet-stream" + metadata := make(map[string][]string) + metadata["Content-Type"] = []string{objectContentType} + + objInfo, err := c.PutObject(bucketName, objectName, int64(len(buf)), bytes.NewReader(buf), md5.New().Sum(nil), nil, metadata) + if err == nil { + t.Fatal("Error expected: nil, got: ", err) + } + + objInfo, err = c.PutObject(bucketName, objectName, int64(len(buf)), bytes.NewReader(buf), nil, sum256(nil), metadata) + if err == nil { + t.Fatal("Error expected: nil, got: ", err) + } + + objInfo, err = c.PutObject(bucketName, objectName, int64(len(buf)), bytes.NewReader(buf), nil, nil, metadata) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if objInfo.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), objInfo.Size) + } + + // Read the data back + r, err := c.Client.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + st, err := r.Stat() + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if st.Size != int64(len(buf)) { + t.Fatalf("Error: number of bytes in stat does not match, want %v, got %v\n", + len(buf), st.Size) + } + + if st.ContentType != objectContentType { + t.Fatalf("Error: Content types don't match, expected: %+v, found: %+v\n", objectContentType, st.ContentType) + } + + if err := r.Close(); err != nil { + t.Fatal("Error:", err) + } + + if err := r.Close(); err == nil { + t.Fatal("Error: object is already closed, should return error") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} diff --git a/vendor/github.com/minio/minio-go/docs/API.md b/vendor/github.com/minio/minio-go/docs/API.md index dfb90b5f2..06735427e 100644 --- a/vendor/github.com/minio/minio-go/docs/API.md +++ b/vendor/github.com/minio/minio-go/docs/API.md @@ -5,7 +5,6 @@ ## Minio ```go - package main import ( @@ -25,13 +24,11 @@ func main() { return } } - ``` ## AWS S3 ```go - package main import ( @@ -51,20 +48,19 @@ func main() { return } } - ``` -| Bucket operations |Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | -|:---|:---|:---|:---|:---| -|[`MakeBucket`](#MakeBucket) |[`GetObject`](#GetObject) | [`PresignedGetObject`](#PresignedGetObject) |[`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | -|[`ListBuckets`](#ListBuckets) |[`PutObject`](#PutObject) |[`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | -|[`BucketExists`](#BucketExists) |[`CopyObject`](#CopyObject) |[`PresignedPostPolicy`](#PresignedPostPolicy) | [`ListBucketPolicies`](#ListBucketPolicies) | [`TraceOn`](#TraceOn) | -| [`RemoveBucket`](#RemoveBucket) |[`StatObject`](#StatObject) | | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | -|[`ListObjects`](#ListObjects) |[`RemoveObject`](#RemoveObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | -|[`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | -|[`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | [`ListenBucketNotification`](#ListenBucketNotification) | -| | [`FPutObject`](#FPutObject) | | | -| | [`FGetObject`](#FGetObject) | | | +| Bucket operations |Object operations | Encrypted Object operations | Presigned operations | Bucket Policy/Notification Operations | Client custom settings | +|:---|:---|:---|:---|:---|:---| +|[`MakeBucket`](#MakeBucket) |[`GetObject`](#GetObject) | [`NewSymmetricKey`](#NewSymmetricKey) | [`PresignedGetObject`](#PresignedGetObject) |[`SetBucketPolicy`](#SetBucketPolicy) | [`SetAppInfo`](#SetAppInfo) | +|[`ListBuckets`](#ListBuckets) |[`PutObject`](#PutObject) | [`NewAsymmetricKey`](#NewAsymmetricKey) |[`PresignedPutObject`](#PresignedPutObject) | [`GetBucketPolicy`](#GetBucketPolicy) | [`SetCustomTransport`](#SetCustomTransport) | +|[`BucketExists`](#BucketExists) |[`CopyObject`](#CopyObject) | [`GetEncryptedObject`](#GetEncryptedObject) |[`PresignedPostPolicy`](#PresignedPostPolicy) | [`ListBucketPolicies`](#ListBucketPolicies) | [`TraceOn`](#TraceOn) | +| [`RemoveBucket`](#RemoveBucket) |[`StatObject`](#StatObject) | [`PutObjectStreaming`](#PutObjectStreaming) | | [`SetBucketNotification`](#SetBucketNotification) | [`TraceOff`](#TraceOff) | +|[`ListObjects`](#ListObjects) |[`RemoveObject`](#RemoveObject) | [`PutEncryptedObject`](#PutEncryptedObject) | | [`GetBucketNotification`](#GetBucketNotification) | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) | +|[`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) | | | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification) | +|[`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) | | | [`ListenBucketNotification`](#ListenBucketNotification) | +| | [`FPutObject`](#FPutObject) | | | | +| | [`FGetObject`](#FGetObject) | | | | ## 1. Constructor <a name="Minio"></a> @@ -74,7 +70,6 @@ Initializes a new client object. __Parameters__ - |Param |Type |Description | |:---|:---| :---| |`endpoint` | _string_ |S3 compatible object storage endpoint | @@ -82,6 +77,18 @@ __Parameters__ |`secretAccessKey` | _string_ |Secret key for the object storage | |`ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | +### NewWithRegion(endpoint, accessKeyID, secretAccessKey string, ssl bool, region string) (*Client, error) +Initializes minio client, with region configured. Unlike New(), NewWithRegion avoids bucket-location lookup operations and it is slightly faster. Use this function when if your application deals with single region. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`endpoint` | _string_ |S3 compatible object storage endpoint | +|`accessKeyID` |_string_ |Access key for the object storage | +|`secretAccessKey` | _string_ |Secret key for the object storage | +|`ssl` | _bool_ | If 'true' API requests will be secure (HTTPS), and insecure (HTTP) otherwise | +|`region`| _string_ | Region for the object storage | ## 2. Bucket operations @@ -89,7 +96,6 @@ __Parameters__ ### MakeBucket(bucketName, location string) error Creates a new bucket. - __Parameters__ | Param | Type | Description | @@ -111,14 +117,12 @@ __Example__ ```go - err := minioClient.MakeBucket("mybucket", "us-east-1") if err != nil { fmt.Println(err) return } fmt.Println("Successfully created mybucket.") - ``` <a name="ListBuckets"></a> @@ -141,7 +145,6 @@ __Example__ ```go - buckets, err := minioClient.ListBuckets() if err != nil { fmt.Println(err) @@ -150,7 +153,6 @@ buckets, err := minioClient.ListBuckets() for _, bucket := range buckets { fmt.Println(bucket) } - ``` <a name="BucketExists"></a> @@ -178,7 +180,6 @@ __Example__ ```go - found, err := minioClient.BucketExists("mybucket") if err != nil { fmt.Println(err) @@ -187,7 +188,6 @@ if err != nil { if found { fmt.Println("Bucket found") } - ``` <a name="RemoveBucket"></a> @@ -206,13 +206,11 @@ __Example__ ```go - err := minioClient.RemoveBucket("mybucket") if err != nil { fmt.Println(err) return } - ``` <a name="ListObjects"></a> @@ -246,7 +244,6 @@ __Return Value__ ```go - // Create a done channel to control 'ListObjects' go routine. doneCh := make(chan struct{}) @@ -262,7 +259,6 @@ for object := range objectCh { } fmt.Println(object) } - ``` @@ -297,7 +293,6 @@ __Return Value__ ```go - // Create a done channel to control 'ListObjectsV2' go routine. doneCh := make(chan struct{}) @@ -313,7 +308,6 @@ for object := range objectCh { } fmt.Println(object) } - ``` <a name="ListIncompleteUploads"></a> @@ -351,7 +345,6 @@ __Example__ ```go - // Create a done channel to control 'ListObjects' go routine. doneCh := make(chan struct{}) @@ -367,7 +360,6 @@ for multiPartObject := range multiPartObjectCh { } fmt.Println(multiPartObject) } - ``` ## 3. Object operations @@ -375,7 +367,7 @@ for multiPartObject := range multiPartObjectCh { <a name="GetObject"></a> ### GetObject(bucketName, objectName string) (*Object, error) -Downloads an object. +Returns a stream of the object data. Most of the common errors occur when reading the stream. __Parameters__ @@ -399,7 +391,6 @@ __Example__ ```go - object, err := minioClient.GetObject("mybucket", "photo.jpg") if err != nil { fmt.Println(err) @@ -414,7 +405,6 @@ if _, err = io.Copy(localFile, object); err != nil { fmt.Println(err) return } - ``` <a name="FGetObject"></a> @@ -436,19 +426,19 @@ __Example__ ```go - err := minioClient.FGetObject("mybucket", "photo.jpg", "/tmp/photo.jpg") if err != nil { fmt.Println(err) return } - ``` <a name="PutObject"></a> ### PutObject(bucketName, objectName string, reader io.Reader, contentType string) (n int, err error) -Uploads an object. +Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object as parts of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. + +In the event that PutObject fails to upload an object, the user may attempt to re-upload the same object. If the same object is being uploaded, PutObject API examines the previous partial attempt to upload this object and resumes automatically from where it left off. __Parameters__ @@ -465,13 +455,40 @@ __Parameters__ __Example__ -Uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than 64MiB in size, PutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. +```go +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() -In the event that PutObject fails to upload an object, the user may attempt to re-upload the same object. If the same object is being uploaded, PutObject API examines the previous partial attempt to upload this object and resumes automatically from where it left off. +n, err := minioClient.PutObject("mybucket", "myobject", file, "application/octet-stream") +if err != nil { + fmt.Println(err) + return +} +``` +<a name="PutObjectStreaming"></a> +### PutObjectStreaming(bucketName, objectName string, reader io.Reader) (n int, err error) -```go +Uploads an object as multiple chunks keeping memory consumption constant. It is similar to PutObject in how objects are broken into multiple parts. Each part in turn is transferred as multiple chunks with constant memory usage. However resuming previously failed uploads from where it was left is not supported. + + +__Parameters__ + + +|Param |Type |Description | +|:---|:---|:---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`reader` | _io.Reader_ |Any Go type that implements io.Reader | +__Example__ + + +```go file, err := os.Open("my-testfile") if err != nil { fmt.Println(err) @@ -479,12 +496,11 @@ if err != nil { } defer file.Close() -n, err := minioClient.PutObject("mybucket", "myobject", file, "application/octet-stream") +n, err := minioClient.PutObjectStreaming("mybucket", "myobject", file) if err != nil { fmt.Println(err) return } - ``` @@ -511,7 +527,7 @@ __Example__ ```go // Use-case-1 // To copy an existing object to a new object with _no_ copy conditions. -copyConditions := minio.CopyConditions{} +copyConds := minio.CopyConditions{} err := minioClient.CopyObject("mybucket", "myobject", "my-sourcebucketname/my-sourceobjectname", copyConds) if err != nil { fmt.Println(err) @@ -541,14 +557,17 @@ if err != nil { fmt.Println(err) return } - ``` <a name="FPutObject"></a> -### FPutObject(bucketName, objectName, filePath, contentType string) error +### FPutObject(bucketName, objectName, filePath, contentType string) (length int64, err error) Uploads contents from a file to objectName. +FPutObject uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than the 64MiB in size, FPutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. + +In the event that FPutObject fails to upload an object, the user may attempt to re-upload the same object. If the same object is being uploaded, FPutObject API examines the previous partial attempt to upload this object and resumes automatically from where it left off. + __Parameters__ @@ -564,18 +583,12 @@ __Parameters__ __Example__ -FPutObject uploads objects that are less than 64MiB in a single PUT operation. For objects that are greater than the 64MiB in size, FPutObject seamlessly uploads the object in chunks of 64MiB or more depending on the actual file size. The max upload size for an object is 5TB. - -In the event that FPutObject fails to upload an object, the user may attempt to re-upload the same object. If the same object is being uploaded, FPutObject API examines the previous partial attempt to upload this object and resumes automatically from where it left off. - ```go - n, err := minioClient.FPutObject("mybucket", "myobject.csv", "/tmp/otherobject.csv", "application/csv") if err != nil { fmt.Println(err) return } - ``` <a name="StatObject"></a> @@ -612,14 +625,12 @@ __Return Value__ ```go - objInfo, err := minioClient.StatObject("mybucket", "photo.jpg") if err != nil { fmt.Println(err) return } fmt.Println(objInfo) - ``` <a name="RemoveObject"></a> @@ -638,13 +649,11 @@ __Parameters__ ```go - err := minioClient.RemoveObject("mybucket", "photo.jpg") if err != nil { fmt.Println(err) return } - ``` <a name="RemoveObjects"></a> ### RemoveObjects(bucketName string, objectsCh chan string) errorCh chan minio.RemoveObjectError @@ -669,12 +678,10 @@ __Return Values__ ```go - errorCh := minioClient.RemoveObjects("mybucket", objectsCh) for e := range errorCh { fmt.Println("Error detected during deletion: " + e.Err.Error()) } - ``` @@ -696,18 +703,188 @@ __Example__ ```go - err := minioClient.RemoveIncompleteUpload("mybucket", "photo.jpg") if err != nil { fmt.Println(err) return } +``` + +## 4. Encrypted object operations + +<a name="NewSymmetricKey"></a> +### NewSymmetricKey(key []byte) *minio.SymmetricKey + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`key` | _string_ |Name of the bucket | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`symmetricKey` | _*minio.SymmetricKey_ |_minio.SymmetricKey_ represents a symmetric key structure which can be used to encrypt and decrypt data. | +```go +symKey := minio.NewSymmetricKey([]byte("my-secret-key-00")) ``` -## 4. Presigned operations + +<a name="NewAsymmetricKey"></a> +### NewAsymmetricKey(privateKey []byte, publicKey[]byte) (*minio.AsymmetricKey, error) + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`privateKey` | _[]byte_ | Private key data | +|`publicKey` | _[]byte_ | Public key data | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`asymmetricKey` | _*minio.AsymmetricKey_ | represents an asymmetric key structure which can be used to encrypt and decrypt data. | +|`err` | _error_ | encountered errors. | +```go +privateKey, err := ioutil.ReadFile("private.key") +if err != nil { + log.Fatal(err) +} + +publicKey, err := ioutil.ReadFile("public.key") +if err != nil { + log.Fatal(err) +} + +// Initialize the asymmetric key +asymmetricKey, err := minio.NewAsymmetricKey(privateKey, publicKey) +if err != nil { + log.Fatal(err) +} +``` + +<a name="GetEncryptedObject"></a> +### GetEncryptedObject(bucketName, objectName string, encryptMaterials minio.EncryptionMaterials) (io.Reader, error) + +Returns the decrypted stream of the object data based of the given encryption materiels. Most of the common errors occur when reading the stream. + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ | Name of the bucket | +|`objectName` | _string_ | Name of the object | +|`encryptMaterials` | _minio.EncryptionMaterials_ | The module to decrypt the object data | + + +__Return Value__ + +|Param |Type |Description | +|:---|:---| :---| +|`stream` | _io.Reader_ | Returns the deciphered object reader. | +|`err` | _error | Returns errors. | + + +__Example__ + + +```go +// Generate a master symmetric key +key := minio.NewSymmetricKey("my-secret-key-00") + +// Build the CBC encryption material +cbcMaterials, err := NewCBCSecureMaterials(key) +if err != nil { + t.Fatal(err) +} + +object, err := minioClient.GetEncryptedObject("mybucket", "photo.jpg", cbcMaterials) +if err != nil { + fmt.Println(err) + return +} +localFile, err := os.Create("/tmp/local-file.jpg") +if err != nil { + fmt.Println(err) + return +} +if _, err = io.Copy(localFile, object); err != nil { + fmt.Println(err) + return +} +``` + +<a name="PutEncryptedObject"></a> + +### PutEncryptedObject(bucketName, objectName string, reader io.Reader, encryptMaterials minio.EncryptionMaterials, metadata map[string][]string, progress io.Reader) (n int, err error) + +Encrypt and upload an object. + + +__Parameters__ + +|Param |Type |Description | +|:---|:---| :---| +|`bucketName` | _string_ |Name of the bucket | +|`objectName` | _string_ |Name of the object | +|`reader` | _io.Reader_ |Any Go type that implements io.Reader | +|`encryptMaterials` | _minio.EncryptionMaterials_ | The module that encrypts data | +|`metadata` | _map[string][]string_ | Object metadata to be stored | +|`progress` | io.Reader | A reader to update the upload progress | + + +__Example__ + +```go +// Load a private key +privateKey, err := ioutil.ReadFile("private.key") +if err != nil { + log.Fatal(err) +} + +// Load a public key +publicKey, err := ioutil.ReadFile("public.key") +if err != nil { + log.Fatal(err) +} + +// Build an asymmetric key +key, err := NewAssymetricKey(privateKey, publicKey) +if err != nil { + log.Fatal(err) +} + +// Build the CBC encryption module +cbcMaterials, err := NewCBCSecureMaterials(key) +if err != nil { + t.Fatal(err) +} + +// Open a file to upload +file, err := os.Open("my-testfile") +if err != nil { + fmt.Println(err) + return +} +defer file.Close() + +// Upload the encrypted form of the file +n, err := minioClient.PutEncryptedObject("mybucket", "myobject", file, encryptMaterials, nil, nil) +if err != nil { + fmt.Println(err) + return +} +``` + +## 5. Presigned operations + <a name="PresignedGetObject"></a> ### PresignedGetObject(bucketName, objectName string, expiry time.Duration, reqParams url.Values) (*url.URL, error) @@ -728,7 +905,6 @@ __Example__ ```go - // Set request parameters for content-disposition. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"your-filename.txt\"") @@ -739,7 +915,6 @@ if err != nil { fmt.Println(err) return } - ``` <a name="PresignedPutObject"></a> @@ -765,7 +940,6 @@ __Example__ ```go - // Generates a url which expires in a day. expiry := time.Second * 24 * 60 * 60 // 1 day. presignedURL, err := minioClient.PresignedPutObject("mybucket", "myobject", expiry) @@ -774,7 +948,6 @@ if err != nil { return } fmt.Println(presignedURL) - ``` <a name="PresignedPostPolicy"></a> @@ -786,16 +959,13 @@ Create policy : ```go - policy := minio.NewPostPolicy() - ``` Apply upload policy restrictions: ```go - policy.SetBucket("mybucket") policy.SetKey("myobject") policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days @@ -813,7 +983,6 @@ if err != nil { fmt.Println(err) return } - ``` @@ -829,7 +998,7 @@ fmt.Printf("-F file=@/etc/bash.bashrc ") fmt.Printf("%s\n", url) ``` -## 5. Bucket policy/notification operations +## 6. Bucket policy/notification operations <a name="SetBucketPolicy"></a> ### SetBucketPolicy(bucketname, objectPrefix string, policy policy.BucketPolicy) error @@ -845,11 +1014,11 @@ __Parameters__ |:---|:---| :---| |`bucketName` | _string_ |Name of the bucket| |`objectPrefix` | _string_ |Name of the object prefix| -|`policy` | _policy.BucketPolicy_ |Policy can be one of the following: | -|| |policy.BucketPolicyNone| -| | |policy.BucketPolicyReadOnly| -|| |policy.BucketPolicyReadWrite| -| | |policy.BucketPolicyWriteOnly| +|`policy` | _policy.BucketPolicy_ |Policy can be one of the following, | +| | | _policy.BucketPolicyNone_ | +| | | _policy.BucketPolicyReadOnly_ | +| | | _policy.BucketPolicyReadWrite_ | +| | | _policy.BucketPolicyWriteOnly_ | __Return Values__ @@ -864,13 +1033,11 @@ __Example__ ```go - err := minioClient.SetBucketPolicy("mybucket", "myprefix", policy.BucketPolicyReadWrite) if err != nil { fmt.Println(err) return } - ``` <a name="GetBucketPolicy"></a> @@ -900,14 +1067,12 @@ __Example__ ```go - bucketPolicy, err := minioClient.GetBucketPolicy("mybucket", "") if err != nil { fmt.Println(err) return } fmt.Println("Access permissions for mybucket is", bucketPolicy) - ``` <a name="ListBucketPolicies"></a> @@ -935,7 +1100,6 @@ __Example__ ```go - bucketPolicies, err := minioClient.ListBucketPolicies("mybucket", "") if err != nil { fmt.Println(err) @@ -944,7 +1108,6 @@ if err != nil { for resource, permission := range bucketPolicies { fmt.Println(resource, " => ", permission) } - ``` <a name="GetBucketNotification"></a> @@ -1087,7 +1250,6 @@ __Example__ ```go - // Create a done channel to control 'ListenBucketNotification' go routine. doneCh := make(chan struct{}) @@ -1097,6 +1259,7 @@ defer close(doneCh) // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }, doneCh) { if notificationInfo.Err != nil { @@ -1106,7 +1269,7 @@ for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET" } ``` -## 6. Client custom settings +## 7. Client custom settings <a name="SetAppInfo"></a> ### SetAppInfo(appName, appVersion string) @@ -1124,10 +1287,8 @@ __Example__ ```go - // Set Application name and version to be used in subsequent API requests. minioClient.SetAppInfo("myCloudApp", "1.0.0") - ``` <a name="SetCustomTransport"></a> @@ -1170,6 +1331,6 @@ __Parameters__ |`acceleratedEndpoint` | _string_ | Set to new S3 transfer acceleration endpoint.| -## 7. Explore Further +## 8. Explore Further - [Build your own Go Music Player App example](https://docs.minio.io/docs/go-music-player-app) diff --git a/vendor/github.com/minio/minio-go/examples/minio/listenbucketnotification.go b/vendor/github.com/minio/minio-go/examples/minio/listenbucketnotification.go index b682dcb42..037e2251c 100644 --- a/vendor/github.com/minio/minio-go/examples/minio/listenbucketnotification.go +++ b/vendor/github.com/minio/minio-go/examples/minio/listenbucketnotification.go @@ -49,6 +49,7 @@ func main() { // Listen for bucket notifications on "mybucket" filtered by prefix, suffix and events. for notificationInfo := range minioClient.ListenBucketNotification("YOUR-BUCKET", "PREFIX", "SUFFIX", []string{ "s3:ObjectCreated:*", + "s3:ObjectAccessed:*", "s3:ObjectRemoved:*", }, doneCh) { if notificationInfo.Err != nil { diff --git a/vendor/github.com/minio/minio-go/examples/s3/get-encrypted-object.go b/vendor/github.com/minio/minio-go/examples/s3/get-encrypted-object.go new file mode 100644 index 000000000..e997140be --- /dev/null +++ b/vendor/github.com/minio/minio-go/examples/s3/get-encrypted-object.go @@ -0,0 +1,86 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "io" + "log" + "os" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-bucketname, my-objectname and + // my-testfile are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESS-KEY-HERE", "YOUR-SECRET-KEY-HERE", true) + if err != nil { + log.Fatalln(err) + } + + //// Build an asymmetric key from private and public files + // + // privateKey, err := ioutil.ReadFile("private.key") + // if err != nil { + // t.Fatal(err) + // } + // + // publicKey, err := ioutil.ReadFile("public.key") + // if err != nil { + // t.Fatal(err) + // } + // + // asymmetricKey, err := NewAsymmetricKey(privateKey, publicKey) + // if err != nil { + // t.Fatal(err) + // } + //// + + // Build a symmetric key + symmetricKey := minio.NewSymmetricKey([]byte("my-secret-key-00")) + + // Build encryption materials which will encrypt uploaded data + cbcMaterials, err := minio.NewCBCSecureMaterials(symmetricKey) + if err != nil { + log.Fatalln(err) + } + + // Get a deciphered data from the server, deciphering is assured by cbcMaterials + reader, err := s3Client.GetEncryptedObject("my-bucketname", "my-objectname", cbcMaterials) + if err != nil { + log.Fatalln(err) + } + + // Local file which holds plain data + localFile, err := os.Create("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer localFile.Close() + + if _, err := io.Copy(localFile, reader); err != nil { + log.Fatalln(err) + } +} diff --git a/vendor/github.com/minio/minio-go/examples/s3/put-encrypted-object.go b/vendor/github.com/minio/minio-go/examples/s3/put-encrypted-object.go new file mode 100644 index 000000000..f03f82147 --- /dev/null +++ b/vendor/github.com/minio/minio-go/examples/s3/put-encrypted-object.go @@ -0,0 +1,83 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + // Open a local file that we will upload + file, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer file.Close() + + //// Build an asymmetric key from private and public files + // + // privateKey, err := ioutil.ReadFile("private.key") + // if err != nil { + // t.Fatal(err) + // } + // + // publicKey, err := ioutil.ReadFile("public.key") + // if err != nil { + // t.Fatal(err) + // } + // + // asymmetricKey, err := NewAsymmetricKey(privateKey, publicKey) + // if err != nil { + // t.Fatal(err) + // } + //// + + // Build a symmetric key + symmetricKey := minio.NewSymmetricKey([]byte("my-secret-key-00")) + + // Build encryption materials which will encrypt uploaded data + cbcMaterials, err := minio.NewCBCSecureMaterials(symmetricKey) + if err != nil { + log.Fatalln(err) + } + + // Encrypt file content and upload to the server + n, err := s3Client.PutEncryptedObject("my-bucketname", "my-objectname", file, cbcMaterials, nil, nil) + if err != nil { + log.Fatalln(err) + } + + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/vendor/github.com/minio/minio-go/examples/s3/putobject-streaming.go b/vendor/github.com/minio/minio-go/examples/s3/putobject-streaming.go new file mode 100644 index 000000000..d10407dbd --- /dev/null +++ b/vendor/github.com/minio/minio-go/examples/s3/putobject-streaming.go @@ -0,0 +1,54 @@ +// +build ignore + +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package main + +import ( + "log" + "os" + + minio "github.com/minio/minio-go" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and + // my-objectname are dummy values, please replace them with original values. + + // Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access. + // This boolean value is the last argument for New(). + + // New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically + // determined based on the Endpoint value. + s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) + if err != nil { + log.Fatalln(err) + } + + object, err := os.Open("my-testfile") + if err != nil { + log.Fatalln(err) + } + defer object.Close() + + n, err := s3Client.PutObjectStreaming("my-bucketname", "my-objectname", object) + if err != nil { + log.Fatalln(err) + } + + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/chain.go b/vendor/github.com/minio/minio-go/pkg/credentials/chain.go new file mode 100644 index 000000000..6b0e57440 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/chain.go @@ -0,0 +1,89 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "fmt" + +// A Chain will search for a provider which returns credentials +// and cache that provider until Retrieve is called again. +// +// The Chain provides a way of chaining multiple providers together +// which will pick the first available using priority order of the +// Providers in the list. +// +// If none of the Providers retrieve valid credentials Value, ChainProvider's +// Retrieve() will return the error, collecting all errors from all providers. +// +// If a Provider is found which returns valid credentials Value ChainProvider +// will cache that Provider for all calls to IsExpired(), until Retrieve is +// called again. +// +// creds := credentials.NewChainCredentials( +// []credentials.Provider{ +// &credentials.EnvAWSS3{}, +// &credentials.EnvMinio{}, +// }) +// +// // Usage of ChainCredentials. +// mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1") +// if err != nil { +// log.Fatalln(err) +// } +// +type Chain struct { + Providers []Provider + curr Provider +} + +// NewChainCredentials returns a pointer to a new Credentials object +// wrapping a chain of providers. +func NewChainCredentials(providers []Provider) *Credentials { + return New(&Chain{ + Providers: append([]Provider{}, providers...), + }) +} + +// Retrieve returns the credentials value or error if no provider returned +// without error. +// +// If a provider is found it will be cached and any calls to IsExpired() +// will return the expired state of the cached provider. +func (c *Chain) Retrieve() (Value, error) { + var errs []error + for _, p := range c.Providers { + creds, err := p.Retrieve() + if err != nil { + errs = append(errs, err) + continue + } // Success. + c.curr = p + return creds, nil + } + c.curr = nil + return Value{}, fmt.Errorf("No valid providers found %v", errs) +} + +// IsExpired will returned the expired state of the currently cached provider +// if there is one. If there is no current provider, true will be returned. +func (c *Chain) IsExpired() bool { + if c.curr != nil { + return c.curr.IsExpired() + } + + return true +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/chain_test.go b/vendor/github.com/minio/minio-go/pkg/credentials/chain_test.go new file mode 100644 index 000000000..cb5a6dda5 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/chain_test.go @@ -0,0 +1,137 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "errors" + "testing" +) + +type testCredProvider struct { + creds Value + expired bool + err error +} + +func (s *testCredProvider) Retrieve() (Value, error) { + s.expired = false + return s.creds, s.err +} +func (s *testCredProvider) IsExpired() bool { + return s.expired +} + +func TestChainGet(t *testing.T) { + p := &Chain{ + Providers: []Provider{ + &credProvider{err: errors.New("FirstError")}, + &credProvider{err: errors.New("SecondError")}, + &testCredProvider{ + creds: Value{ + AccessKeyID: "AKIF", + SecretAccessKey: "NOSECRET", + SessionToken: "", + }, + }, + &credProvider{ + creds: Value{ + AccessKeyID: "AKID", + SecretAccessKey: "SECRET", + SessionToken: "", + }, + }, + }, + } + + creds, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + // Also check credentials + if creds.AccessKeyID != "AKIF" { + t.Fatalf("Expected 'AKIF', got %s", creds.AccessKeyID) + } + if creds.SecretAccessKey != "NOSECRET" { + t.Fatalf("Expected 'NOSECRET', got %s", creds.SecretAccessKey) + } + if creds.SessionToken != "" { + t.Fatalf("Expected empty token, got %s", creds.SessionToken) + } +} + +func TestChainIsExpired(t *testing.T) { + credProvider := &credProvider{expired: true} + p := &Chain{ + Providers: []Provider{ + credProvider, + }, + } + + if !p.IsExpired() { + t.Fatal("Expected expired to be true before any Retrieve") + } + + _, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + if p.IsExpired() { + t.Fatal("Expected to be not expired after Retrieve") + } +} + +func TestChainWithNoProvider(t *testing.T) { + p := &Chain{ + Providers: []Provider{}, + } + if !p.IsExpired() { + t.Fatal("Expected to be expired with no providers") + } + _, err := p.Retrieve() + if err != nil { + if err.Error() != "No valid providers found []" { + t.Error(err) + } + } +} + +func TestChainProviderWithNoValidProvider(t *testing.T) { + errs := []error{ + errors.New("FirstError"), + errors.New("SecondError"), + } + p := &Chain{ + Providers: []Provider{ + &credProvider{err: errs[0]}, + &credProvider{err: errs[1]}, + }, + } + + if !p.IsExpired() { + t.Fatal("Expected to be expired with no providers") + } + + _, err := p.Retrieve() + if err != nil { + if err.Error() != "No valid providers found [FirstError SecondError]" { + t.Error(err) + } + } +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/config.json.sample b/vendor/github.com/minio/minio-go/pkg/credentials/config.json.sample new file mode 100644 index 000000000..130746f4b --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/config.json.sample @@ -0,0 +1,17 @@ +{ + "version": "8", + "hosts": { + "play": { + "url": "https://play.minio.io:9000", + "accessKey": "Q3AM3UQ867SPQQA43P2F", + "secretKey": "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG", + "api": "S3v2" + }, + "s3": { + "url": "https://s3.amazonaws.com", + "accessKey": "accessKey", + "secretKey": "secret", + "api": "S3v4" + } + } +}
\ No newline at end of file diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/credentials.go b/vendor/github.com/minio/minio-go/pkg/credentials/credentials.go new file mode 100644 index 000000000..cc3000532 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/credentials.go @@ -0,0 +1,175 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "sync" + "time" +) + +// A Value is the AWS credentials value for individual credential fields. +type Value struct { + // AWS Access key ID + AccessKeyID string + + // AWS Secret Access Key + SecretAccessKey string + + // AWS Session Token + SessionToken string + + // Signature Type. + SignerType SignatureType +} + +// A Provider is the interface for any component which will provide credentials +// Value. A provider is required to manage its own Expired state, and what to +// be expired means. +type Provider interface { + // Retrieve returns nil if it successfully retrieved the value. + // Error is returned if the value were not obtainable, or empty. + Retrieve() (Value, error) + + // IsExpired returns if the credentials are no longer valid, and need + // to be retrieved. + IsExpired() bool +} + +// A Expiry provides shared expiration logic to be used by credentials +// providers to implement expiry functionality. +// +// The best method to use this struct is as an anonymous field within the +// provider's struct. +// +// Example: +// type IAMCredentialProvider struct { +// Expiry +// ... +// } +type Expiry struct { + // The date/time when to expire on + expiration time.Time + + // If set will be used by IsExpired to determine the current time. + // Defaults to time.Now if CurrentTime is not set. + CurrentTime func() time.Time +} + +// SetExpiration sets the expiration IsExpired will check when called. +// +// If window is greater than 0 the expiration time will be reduced by the +// window value. +// +// Using a window is helpful to trigger credentials to expire sooner than +// the expiration time given to ensure no requests are made with expired +// tokens. +func (e *Expiry) SetExpiration(expiration time.Time, window time.Duration) { + e.expiration = expiration + if window > 0 { + e.expiration = e.expiration.Add(-window) + } +} + +// IsExpired returns if the credentials are expired. +func (e *Expiry) IsExpired() bool { + if e.CurrentTime == nil { + e.CurrentTime = time.Now + } + return e.expiration.Before(e.CurrentTime()) +} + +// Credentials - A container for synchronous safe retrieval of credentials Value. +// Credentials will cache the credentials value until they expire. Once the value +// expires the next Get will attempt to retrieve valid credentials. +// +// Credentials is safe to use across multiple goroutines and will manage the +// synchronous state so the Providers do not need to implement their own +// synchronization. +// +// The first Credentials.Get() will always call Provider.Retrieve() to get the +// first instance of the credentials Value. All calls to Get() after that +// will return the cached credentials Value until IsExpired() returns true. +type Credentials struct { + sync.Mutex + + creds Value + forceRefresh bool + provider Provider +} + +// New returns a pointer to a new Credentials with the provider set. +func New(provider Provider) *Credentials { + return &Credentials{ + provider: provider, + forceRefresh: true, + } +} + +// Get returns the credentials value, or error if the credentials Value failed +// to be retrieved. +// +// Will return the cached credentials Value if it has not expired. If the +// credentials Value has expired the Provider's Retrieve() will be called +// to refresh the credentials. +// +// If Credentials.Expire() was called the credentials Value will be force +// expired, and the next call to Get() will cause them to be refreshed. +func (c *Credentials) Get() (Value, error) { + c.Lock() + defer c.Unlock() + + if c.isExpired() { + creds, err := c.provider.Retrieve() + if err != nil { + return Value{}, err + } + c.creds = creds + c.forceRefresh = false + } + + return c.creds, nil +} + +// Expire expires the credentials and forces them to be retrieved on the +// next call to Get(). +// +// This will override the Provider's expired state, and force Credentials +// to call the Provider's Retrieve(). +func (c *Credentials) Expire() { + c.Lock() + defer c.Unlock() + + c.forceRefresh = true +} + +// IsExpired returns if the credentials are no longer valid, and need +// to be refreshed. +// +// If the Credentials were forced to be expired with Expire() this will +// reflect that override. +func (c *Credentials) IsExpired() bool { + c.Lock() + defer c.Unlock() + + return c.isExpired() +} + +// isExpired helper method wrapping the definition of expired credentials. +func (c *Credentials) isExpired() bool { + return c.forceRefresh || c.provider.IsExpired() +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/credentials.sample b/vendor/github.com/minio/minio-go/pkg/credentials/credentials.sample new file mode 100644 index 000000000..7fc91d9d2 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/credentials.sample @@ -0,0 +1,12 @@ +[default] +aws_access_key_id = accessKey +aws_secret_access_key = secret +aws_session_token = token + +[no_token] +aws_access_key_id = accessKey +aws_secret_access_key = secret + +[with_colon] +aws_access_key_id: accessKey +aws_secret_access_key: secret diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/credentials_test.go b/vendor/github.com/minio/minio-go/pkg/credentials/credentials_test.go new file mode 100644 index 000000000..cbfb673b7 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/credentials_test.go @@ -0,0 +1,73 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "errors" + "testing" +) + +type credProvider struct { + creds Value + expired bool + err error +} + +func (s *credProvider) Retrieve() (Value, error) { + s.expired = false + return s.creds, s.err +} +func (s *credProvider) IsExpired() bool { + return s.expired +} + +func TestCredentialsGet(t *testing.T) { + c := New(&credProvider{ + creds: Value{ + AccessKeyID: "UXHW", + SecretAccessKey: "MYSECRET", + SessionToken: "", + }, + expired: true, + }) + + creds, err := c.Get() + if err != nil { + t.Fatal(err) + } + if "UXHW" != creds.AccessKeyID { + t.Errorf("Expected \"UXHW\", got %s", creds.AccessKeyID) + } + if "MYSECRET" != creds.SecretAccessKey { + t.Errorf("Expected \"MYSECRET\", got %s", creds.SecretAccessKey) + } + if creds.SessionToken != "" { + t.Errorf("Expected session token to be empty, got %s", creds.SessionToken) + } +} + +func TestCredentialsGetWithError(t *testing.T) { + c := New(&credProvider{err: errors.New("Custom error")}) + + _, err := c.Get() + if err != nil { + if err.Error() != "Custom error" { + t.Errorf("Expected \"Custom error\", got %s", err.Error()) + } + } +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/doc.go b/vendor/github.com/minio/minio-go/pkg/credentials/doc.go new file mode 100644 index 000000000..fa1908aeb --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/doc.go @@ -0,0 +1,45 @@ +// Package credentials provides credential retrieval and management +// for S3 compatible object storage. +// +// By default the Credentials.Get() will cache the successful result of a +// Provider's Retrieve() until Provider.IsExpired() returns true. At which +// point Credentials will call Provider's Retrieve() to get new credential Value. +// +// The Provider is responsible for determining when credentials have expired. +// It is also important to note that Credentials will always call Retrieve the +// first time Credentials.Get() is called. +// +// Example of using the environment variable credentials. +// +// creds := NewFromEnv() +// // Retrieve the credentials value +// credValue, err := creds.Get() +// if err != nil { +// // handle error +// } +// +// Example of forcing credentials to expire and be refreshed on the next Get(). +// This may be helpful to proactively expire credentials and refresh them sooner +// than they would naturally expire on their own. +// +// creds := NewFromIAM("") +// creds.Expire() +// credsValue, err := creds.Get() +// // New credentials will be retrieved instead of from cache. +// +// +// Custom Provider +// +// Each Provider built into this package also provides a helper method to generate +// a Credentials pointer setup with the provider. To use a custom Provider just +// create a type which satisfies the Provider interface and pass it to the +// NewCredentials method. +// +// type MyProvider struct{} +// func (m *MyProvider) Retrieve() (Value, error) {...} +// func (m *MyProvider) IsExpired() bool {...} +// +// creds := NewCredentials(&MyProvider{}) +// credValue, err := creds.Get() +// +package credentials diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/env_aws.go b/vendor/github.com/minio/minio-go/pkg/credentials/env_aws.go new file mode 100644 index 000000000..11934433c --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/env_aws.go @@ -0,0 +1,71 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "os" + +// A EnvAWS retrieves credentials from the environment variables of the +// running process. EnvAWSironment credentials never expire. +// +// EnvAWSironment variables used: +// +// * Access Key ID: AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY. +// * Secret Access Key: AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY. +// * Secret Token: AWS_SESSION_TOKEN. +type EnvAWS struct { + retrieved bool +} + +// NewEnvAWS returns a pointer to a new Credentials object +// wrapping the environment variable provider. +func NewEnvAWS() *Credentials { + return New(&EnvAWS{}) +} + +// Retrieve retrieves the keys from the environment. +func (e *EnvAWS) Retrieve() (Value, error) { + e.retrieved = false + + id := os.Getenv("AWS_ACCESS_KEY_ID") + if id == "" { + id = os.Getenv("AWS_ACCESS_KEY") + } + + secret := os.Getenv("AWS_SECRET_ACCESS_KEY") + if secret == "" { + secret = os.Getenv("AWS_SECRET_KEY") + } + + signerType := SignatureV4 + if id == "" || secret == "" { + signerType = SignatureAnonymous + } + + e.retrieved = true + return Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: os.Getenv("AWS_SESSION_TOKEN"), + SignerType: signerType, + }, nil +} + +// IsExpired returns if the credentials have been retrieved. +func (e *EnvAWS) IsExpired() bool { + return !e.retrieved +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/env_minio.go b/vendor/github.com/minio/minio-go/pkg/credentials/env_minio.go new file mode 100644 index 000000000..791087ef5 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/env_minio.go @@ -0,0 +1,62 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "os" + +// A EnvMinio retrieves credentials from the environment variables of the +// running process. EnvMinioironment credentials never expire. +// +// EnvMinioironment variables used: +// +// * Access Key ID: MINIO_ACCESS_KEY. +// * Secret Access Key: MINIO_SECRET_KEY. +type EnvMinio struct { + retrieved bool +} + +// NewEnvMinio returns a pointer to a new Credentials object +// wrapping the environment variable provider. +func NewEnvMinio() *Credentials { + return New(&EnvMinio{}) +} + +// Retrieve retrieves the keys from the environment. +func (e *EnvMinio) Retrieve() (Value, error) { + e.retrieved = false + + id := os.Getenv("MINIO_ACCESS_KEY") + secret := os.Getenv("MINIO_SECRET_KEY") + + signerType := SignatureV4 + if id == "" || secret == "" { + signerType = SignatureAnonymous + } + + e.retrieved = true + return Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SignerType: signerType, + }, nil +} + +// IsExpired returns if the credentials have been retrieved. +func (e *EnvMinio) IsExpired() bool { + return !e.retrieved +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/env_test.go b/vendor/github.com/minio/minio-go/pkg/credentials/env_test.go new file mode 100644 index 000000000..2f72bea40 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/env_test.go @@ -0,0 +1,105 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "os" + "reflect" + "testing" +) + +func TestEnvAWSRetrieve(t *testing.T) { + os.Clearenv() + os.Setenv("AWS_ACCESS_KEY_ID", "access") + os.Setenv("AWS_SECRET_ACCESS_KEY", "secret") + os.Setenv("AWS_SESSION_TOKEN", "token") + + e := EnvAWS{} + if !e.IsExpired() { + t.Error("Expect creds to be expired before retrieve.") + } + + creds, err := e.Retrieve() + if err != nil { + t.Fatal(err) + } + + expectedCreds := Value{ + AccessKeyID: "access", + SecretAccessKey: "secret", + SessionToken: "token", + SignerType: SignatureV4, + } + if !reflect.DeepEqual(creds, expectedCreds) { + t.Errorf("Expected %v, got %v", expectedCreds, creds) + } + + if e.IsExpired() { + t.Error("Expect creds to not be expired after retrieve.") + } + + os.Clearenv() + os.Setenv("AWS_ACCESS_KEY", "access") + os.Setenv("AWS_SECRET_KEY", "secret") + + expectedCreds = Value{ + AccessKeyID: "access", + SecretAccessKey: "secret", + SignerType: SignatureV4, + } + + creds, err = e.Retrieve() + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(creds, expectedCreds) { + t.Errorf("Expected %v, got %v", expectedCreds, creds) + } + +} + +func TestEnvMinioRetrieve(t *testing.T) { + os.Clearenv() + + os.Setenv("MINIO_ACCESS_KEY", "access") + os.Setenv("MINIO_SECRET_KEY", "secret") + + e := EnvMinio{} + if !e.IsExpired() { + t.Error("Expect creds to be expired before retrieve.") + } + + creds, err := e.Retrieve() + if err != nil { + t.Fatal(err) + } + + expectedCreds := Value{ + AccessKeyID: "access", + SecretAccessKey: "secret", + SignerType: SignatureV4, + } + if !reflect.DeepEqual(creds, expectedCreds) { + t.Errorf("Expected %v, got %v", expectedCreds, creds) + } + + if e.IsExpired() { + t.Error("Expect creds to not be expired after retrieve.") + } +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/file_aws_credentials.go b/vendor/github.com/minio/minio-go/pkg/credentials/file_aws_credentials.go new file mode 100644 index 000000000..1be621385 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/file_aws_credentials.go @@ -0,0 +1,120 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "os" + "path/filepath" + + "github.com/go-ini/ini" + homedir "github.com/minio/go-homedir" +) + +// A FileAWSCredentials retrieves credentials from the current user's home +// directory, and keeps track if those credentials are expired. +// +// Profile ini file example: $HOME/.aws/credentials +type FileAWSCredentials struct { + // Path to the shared credentials file. + // + // If empty will look for "AWS_SHARED_CREDENTIALS_FILE" env variable. If the + // env value is empty will default to current user's home directory. + // Linux/OSX: "$HOME/.aws/credentials" + // Windows: "%USERPROFILE%\.aws\credentials" + filename string + + // AWS Profile to extract credentials from the shared credentials file. If empty + // will default to environment variable "AWS_PROFILE" or "default" if + // environment variable is also not set. + profile string + + // retrieved states if the credentials have been successfully retrieved. + retrieved bool +} + +// NewFileAWSCredentials returns a pointer to a new Credentials object +// wrapping the Profile file provider. +func NewFileAWSCredentials(filename string, profile string) *Credentials { + return New(&FileAWSCredentials{ + filename: filename, + profile: profile, + }) +} + +// Retrieve reads and extracts the shared credentials from the current +// users home directory. +func (p *FileAWSCredentials) Retrieve() (Value, error) { + if p.filename == "" { + p.filename = os.Getenv("AWS_SHARED_CREDENTIALS_FILE") + if p.filename == "" { + homeDir, err := homedir.Dir() + if err != nil { + return Value{}, err + } + p.filename = filepath.Join(homeDir, ".aws", "credentials") + } + } + if p.profile == "" { + p.profile = os.Getenv("AWS_PROFILE") + if p.profile == "" { + p.profile = "default" + } + } + + p.retrieved = false + + iniProfile, err := loadProfile(p.filename, p.profile) + if err != nil { + return Value{}, err + } + + // Default to empty string if not found. + id := iniProfile.Key("aws_access_key_id") + // Default to empty string if not found. + secret := iniProfile.Key("aws_secret_access_key") + // Default to empty string if not found. + token := iniProfile.Key("aws_session_token") + + p.retrieved = true + return Value{ + AccessKeyID: id.String(), + SecretAccessKey: secret.String(), + SessionToken: token.String(), + SignerType: SignatureV4, + }, nil +} + +// IsExpired returns if the shared credentials have expired. +func (p *FileAWSCredentials) IsExpired() bool { + return !p.retrieved +} + +// loadProfiles loads from the file pointed to by shared credentials filename for profile. +// The credentials retrieved from the profile will be returned or error. Error will be +// returned if it fails to read from the file, or the data is invalid. +func loadProfile(filename, profile string) (*ini.Section, error) { + config, err := ini.Load(filename) + if err != nil { + return nil, err + } + iniProfile, err := config.GetSection(profile) + if err != nil { + return nil, err + } + return iniProfile, nil +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go b/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go new file mode 100644 index 000000000..9e26dd302 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/file_minio_client.go @@ -0,0 +1,129 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "runtime" + + homedir "github.com/minio/go-homedir" +) + +// A FileMinioClient retrieves credentials from the current user's home +// directory, and keeps track if those credentials are expired. +// +// Configuration file example: $HOME/.mc/config.json +type FileMinioClient struct { + // Path to the shared credentials file. + // + // If empty will look for "MINIO_SHARED_CREDENTIALS_FILE" env variable. If the + // env value is empty will default to current user's home directory. + // Linux/OSX: "$HOME/.mc/config.json" + // Windows: "%USERALIAS%\mc\config.json" + filename string + + // Minio Alias to extract credentials from the shared credentials file. If empty + // will default to environment variable "MINIO_ALIAS" or "default" if + // environment variable is also not set. + alias string + + // retrieved states if the credentials have been successfully retrieved. + retrieved bool +} + +// NewFileMinioClient returns a pointer to a new Credentials object +// wrapping the Alias file provider. +func NewFileMinioClient(filename string, alias string) *Credentials { + return New(&FileMinioClient{ + filename: filename, + alias: alias, + }) +} + +// Retrieve reads and extracts the shared credentials from the current +// users home directory. +func (p *FileMinioClient) Retrieve() (Value, error) { + if p.filename == "" { + homeDir, err := homedir.Dir() + if err != nil { + return Value{}, err + } + p.filename = filepath.Join(homeDir, ".mc", "config.json") + if runtime.GOOS == "windows" { + p.filename = filepath.Join(homeDir, "mc", "config.json") + } + } + + if p.alias == "" { + p.alias = os.Getenv("MINIO_ALIAS") + if p.alias == "" { + p.alias = "s3" + } + } + + p.retrieved = false + + hostCfg, err := loadAlias(p.filename, p.alias) + if err != nil { + return Value{}, err + } + + p.retrieved = true + return Value{ + AccessKeyID: hostCfg.AccessKey, + SecretAccessKey: hostCfg.SecretKey, + SignerType: parseSignatureType(hostCfg.API), + }, nil +} + +// IsExpired returns if the shared credentials have expired. +func (p *FileMinioClient) IsExpired() bool { + return !p.retrieved +} + +// hostConfig configuration of a host. +type hostConfig struct { + URL string `json:"url"` + AccessKey string `json:"accessKey"` + SecretKey string `json:"secretKey"` + API string `json:"api"` +} + +// config config version. +type config struct { + Version string `json:"version"` + Hosts map[string]hostConfig `json:"hosts"` +} + +// loadAliass loads from the file pointed to by shared credentials filename for alias. +// The credentials retrieved from the alias will be returned or error. Error will be +// returned if it fails to read from the file. +func loadAlias(filename, alias string) (hostConfig, error) { + cfg := &config{} + configBytes, err := ioutil.ReadFile(filename) + if err != nil { + return hostConfig{}, err + } + if err = json.Unmarshal(configBytes, cfg); err != nil { + return hostConfig{}, err + } + return cfg.Hosts[alias], nil +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/file_test.go b/vendor/github.com/minio/minio-go/pkg/credentials/file_test.go new file mode 100644 index 000000000..c62c53365 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/file_test.go @@ -0,0 +1,189 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFileAWS(t *testing.T) { + os.Clearenv() + + creds := NewFileAWSCredentials("credentials.sample", "") + credValues, err := creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SessionToken != "token" { + t.Errorf("Expected 'token', got %s'", credValues.SessionToken) + } + + os.Setenv("AWS_SHARED_CREDENTIALS_FILE", "credentials.sample") + creds = NewFileAWSCredentials("", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SessionToken != "token" { + t.Errorf("Expected 'token', got %s'", credValues.SessionToken) + } + + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + + os.Setenv("AWS_SHARED_CREDENTIALS_FILE", filepath.Join(wd, "credentials.sample")) + creds = NewFileAWSCredentials("", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SessionToken != "token" { + t.Errorf("Expected 'token', got %s'", credValues.SessionToken) + } + + os.Clearenv() + os.Setenv("AWS_PROFILE", "no_token") + + creds = NewFileAWSCredentials("credentials.sample", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + + os.Clearenv() + + creds = NewFileAWSCredentials("credentials.sample", "no_token") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + + creds = NewFileAWSCredentials("credentials-non-existent.sample", "no_token") + _, err = creds.Get() + if !os.IsNotExist(err) { + t.Errorf("Expected open non-existent.json: no such file or directory, got %s", err) + } + if !creds.IsExpired() { + t.Error("Should be expired if not loaded") + } +} + +func TestFileMinioClient(t *testing.T) { + os.Clearenv() + + creds := NewFileMinioClient("config.json.sample", "") + credValues, err := creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "accessKey" { + t.Errorf("Expected 'accessKey', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "secret" { + t.Errorf("Expected 'secret', got %s'", credValues.SecretAccessKey) + } + if credValues.SignerType != SignatureV4 { + t.Errorf("Expected 'S3v4', got %s'", credValues.SignerType) + } + + os.Clearenv() + os.Setenv("MINIO_ALIAS", "play") + + creds = NewFileMinioClient("config.json.sample", "") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "Q3AM3UQ867SPQQA43P2F" { + t.Errorf("Expected 'Q3AM3UQ867SPQQA43P2F', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" { + t.Errorf("Expected 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', got %s'", credValues.SecretAccessKey) + } + if credValues.SignerType != SignatureV2 { + t.Errorf("Expected 'S3v2', got %s'", credValues.SignerType) + } + + os.Clearenv() + + creds = NewFileMinioClient("config.json.sample", "play") + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if credValues.AccessKeyID != "Q3AM3UQ867SPQQA43P2F" { + t.Errorf("Expected 'Q3AM3UQ867SPQQA43P2F', got %s'", credValues.AccessKeyID) + } + if credValues.SecretAccessKey != "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG" { + t.Errorf("Expected 'zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG', got %s'", credValues.SecretAccessKey) + } + if credValues.SignerType != SignatureV2 { + t.Errorf("Expected 'S3v2', got %s'", credValues.SignerType) + } + + creds = NewFileMinioClient("non-existent.json", "play") + _, err = creds.Get() + if !os.IsNotExist(err) { + t.Errorf("Expected open non-existent.json: no such file or directory, got %s", err) + } + if !creds.IsExpired() { + t.Error("Should be expired if not loaded") + } +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go b/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go new file mode 100644 index 000000000..ee24a213b --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws.go @@ -0,0 +1,196 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import ( + "bufio" + "encoding/json" + "errors" + "net/http" + "net/url" + "path" + "time" +) + +// DefaultExpiryWindow - Default expiry window. +// ExpiryWindow will allow the credentials to trigger refreshing +// prior to the credentials actually expiring. This is beneficial +// so race conditions with expiring credentials do not cause +// request to fail unexpectedly due to ExpiredTokenException exceptions. +const DefaultExpiryWindow = time.Second * 10 // 10 secs + +// A IAM retrieves credentials from the EC2 service, and keeps track if +// those credentials are expired. +type IAM struct { + Expiry + + // Required http Client to use when connecting to IAM metadata service. + Client *http.Client + + // Custom endpoint in place of + endpoint string +} + +// redirectHeaders copies all headers when following a redirect URL. +// This won't be needed anymore from go 1.8 (https://github.com/golang/go/issues/4800) +func redirectHeaders(req *http.Request, via []*http.Request) error { + if len(via) == 0 { + return nil + } + for key, val := range via[0].Header { + req.Header[key] = val + } + return nil +} + +// NewIAM returns a pointer to a new Credentials object wrapping +// the IAM. Takes a ConfigProvider to create a EC2Metadata client. +// The ConfigProvider is satisfied by the session.Session type. +func NewIAM(endpoint string) *Credentials { + if endpoint == "" { + // IAM Roles for Amazon EC2 + // http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html + endpoint = "http://169.254.169.254" + } + p := &IAM{ + Client: &http.Client{ + Transport: http.DefaultTransport, + CheckRedirect: redirectHeaders, + }, + endpoint: endpoint, + } + return New(p) +} + +// Retrieve retrieves credentials from the EC2 service. +// Error will be returned if the request fails, or unable to extract +// the desired +func (m *IAM) Retrieve() (Value, error) { + credsList, err := requestCredList(m.Client, m.endpoint) + if err != nil { + return Value{}, err + } + + if len(credsList) == 0 { + return Value{}, errors.New("empty EC2 Role list") + } + credsName := credsList[0] + + roleCreds, err := requestCred(m.Client, m.endpoint, credsName) + if err != nil { + return Value{}, err + } + + // Expiry window is set to 10secs. + m.SetExpiration(roleCreds.Expiration, DefaultExpiryWindow) + + return Value{ + AccessKeyID: roleCreds.AccessKeyID, + SecretAccessKey: roleCreds.SecretAccessKey, + SessionToken: roleCreds.Token, + SignerType: SignatureV4, + }, nil +} + +// A ec2RoleCredRespBody provides the shape for unmarshaling credential +// request responses. +type ec2RoleCredRespBody struct { + // Success State + Expiration time.Time + AccessKeyID string + SecretAccessKey string + Token string + + // Error state + Code string + Message string +} + +const iamSecurityCredsPath = "/latest/meta-data/iam/security-credentials" + +// requestCredList requests a list of credentials from the EC2 service. +// If there are no credentials, or there is an error making or receiving the request +func requestCredList(client *http.Client, endpoint string) ([]string, error) { + u, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + u.Path = iamSecurityCredsPath + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return nil, err + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errors.New(resp.Status) + } + + credsList := []string{} + s := bufio.NewScanner(resp.Body) + for s.Scan() { + credsList = append(credsList, s.Text()) + } + + if err := s.Err(); err != nil { + return nil, err + } + + return credsList, nil +} + +// requestCred requests the credentials for a specific credentials from the EC2 service. +// +// If the credentials cannot be found, or there is an error reading the response +// and error will be returned. +func requestCred(client *http.Client, endpoint string, credsName string) (ec2RoleCredRespBody, error) { + u, err := url.Parse(endpoint) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + u.Path = path.Join(iamSecurityCredsPath, credsName) + req, err := http.NewRequest("GET", u.String(), nil) + if err != nil { + return ec2RoleCredRespBody{}, err + } + + resp, err := client.Do(req) + if err != nil { + return ec2RoleCredRespBody{}, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return ec2RoleCredRespBody{}, errors.New(resp.Status) + } + + respCreds := ec2RoleCredRespBody{} + if err := json.NewDecoder(resp.Body).Decode(&respCreds); err != nil { + return ec2RoleCredRespBody{}, err + } + + if respCreds.Code != "Success" { + // If an error code was returned something failed requesting the role. + return ec2RoleCredRespBody{}, errors.New(respCreds.Message) + } + + return respCreds, nil +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws_test.go b/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws_test.go new file mode 100644 index 000000000..19553945d --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/iam_aws_test.go @@ -0,0 +1,180 @@ +package credentials + +import ( + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" +) + +const credsRespTmpl = `{ + "Code": "Success", + "Type": "AWS-HMAC", + "AccessKeyId" : "accessKey", + "SecretAccessKey" : "secret", + "Token" : "token", + "Expiration" : "%s", + "LastUpdated" : "2009-11-23T0:00:00Z" +}` + +const credsFailRespTmpl = `{ + "Code": "ErrorCode", + "Message": "ErrorMsg", + "LastUpdated": "2009-11-23T0:00:00Z" +}` + +func initTestFailServer() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Not allowed", http.StatusBadRequest) + })) + return server +} + +func initTestServerNoRoles() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("")) + })) + return server +} + +func initTestServer(expireOn string, failAssume bool) *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/latest/meta-data/iam/security-credentials" { + fmt.Fprintln(w, "RoleName") + } else if r.URL.Path == "/latest/meta-data/iam/security-credentials/RoleName" { + if failAssume { + fmt.Fprintf(w, credsFailRespTmpl) + } else { + fmt.Fprintf(w, credsRespTmpl, expireOn) + } + } else { + http.Error(w, "bad request", http.StatusBadRequest) + } + })) + + return server +} + +func TestIAMMalformedEndpoint(t *testing.T) { + creds := NewIAM("%%%%") + _, err := creds.Get() + if err == nil { + t.Fatal("Unexpected should fail here") + } + if err.Error() != `parse %%%%: invalid URL escape "%%%"` { + t.Fatalf("Expected parse %%%%%%%%: invalid URL escape \"%%%%%%\", got %s", err) + } +} + +func TestIAMFailServer(t *testing.T) { + server := initTestFailServer() + defer server.Close() + + creds := NewIAM(server.URL) + + _, err := creds.Get() + if err == nil { + t.Fatal("Unexpected should fail here") + } + if err.Error() != "400 Bad Request" { + t.Fatalf("Expected '400 Bad Request', got %s", err) + } +} + +func TestIAMNoRoles(t *testing.T) { + server := initTestServerNoRoles() + defer server.Close() + + creds := NewIAM(server.URL) + _, err := creds.Get() + if err == nil { + t.Fatal("Unexpected should fail here") + } + if err.Error() != "empty EC2 Role list" { + t.Fatalf("Expected 'empty EC2 Role list', got %s", err) + } +} + +func TestIAM(t *testing.T) { + server := initTestServer("2014-12-16T01:51:37Z", false) + defer server.Close() + + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + + creds, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + if "accessKey" != creds.AccessKeyID { + t.Errorf("Expected \"accessKey\", got %s", creds.AccessKeyID) + } + + if "secret" != creds.SecretAccessKey { + t.Errorf("Expected \"secret\", got %s", creds.SecretAccessKey) + } + + if "token" != creds.SessionToken { + t.Errorf("Expected \"token\", got %s", creds.SessionToken) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired.") + } +} + +func TestIAMFailAssume(t *testing.T) { + server := initTestServer("2014-12-16T01:51:37Z", true) + defer server.Close() + + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + + _, err := p.Retrieve() + if err == nil { + t.Fatal("Unexpected success, should fail") + } + if err.Error() != "ErrorMsg" { + t.Errorf("Expected \"ErrorMsg\", got %s", err) + } +} + +func TestIAMIsExpired(t *testing.T) { + server := initTestServer("2014-12-16T01:51:37Z", false) + defer server.Close() + + p := &IAM{ + Client: http.DefaultClient, + endpoint: server.URL, + } + p.CurrentTime = func() time.Time { + return time.Date(2014, 12, 15, 21, 26, 0, 0, time.UTC) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired before retrieve.") + } + + _, err := p.Retrieve() + if err != nil { + t.Fatal(err) + } + + if p.IsExpired() { + t.Error("Expected creds to not be expired after retrieve.") + } + + p.CurrentTime = func() time.Time { + return time.Date(3014, 12, 15, 21, 26, 0, 0, time.UTC) + } + + if !p.IsExpired() { + t.Error("Expected creds to be expired when curren time has changed") + } +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/signature-type.go b/vendor/github.com/minio/minio-go/pkg/credentials/signature-type.go new file mode 100644 index 000000000..c64ad6c23 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/signature-type.go @@ -0,0 +1,76 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "strings" + +// SignatureType is type of Authorization requested for a given HTTP request. +type SignatureType int + +// Different types of supported signatures - default is SignatureV4 or SignatureDefault. +const ( + // SignatureDefault is always set to v4. + SignatureDefault SignatureType = iota + SignatureV4 + SignatureV2 + SignatureV4Streaming + SignatureAnonymous // Anonymous signature signifies, no signature. +) + +// IsV2 - is signature SignatureV2? +func (s SignatureType) IsV2() bool { + return s == SignatureV2 +} + +// IsV4 - is signature SignatureV4? +func (s SignatureType) IsV4() bool { + return s == SignatureV4 || s == SignatureDefault +} + +// IsStreamingV4 - is signature SignatureV4Streaming? +func (s SignatureType) IsStreamingV4() bool { + return s == SignatureV4Streaming +} + +// IsAnonymous - is signature empty? +func (s SignatureType) IsAnonymous() bool { + return s == SignatureAnonymous +} + +// Stringer humanized version of signature type, +// strings returned here are case insensitive. +func (s SignatureType) String() string { + if s.IsV2() { + return "S3v2" + } else if s.IsV4() { + return "S3v4" + } else if s.IsStreamingV4() { + return "S3v4Streaming" + } + return "Anonymous" +} + +func parseSignatureType(str string) SignatureType { + if strings.EqualFold(str, "S3v4") { + return SignatureV4 + } else if strings.EqualFold(str, "S3v2") { + return SignatureV2 + } else if strings.EqualFold(str, "S3v4Streaming") { + return SignatureV4Streaming + } + return SignatureAnonymous +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/static.go b/vendor/github.com/minio/minio-go/pkg/credentials/static.go new file mode 100644 index 000000000..25aff5696 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/static.go @@ -0,0 +1,67 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +// A Static is a set of credentials which are set programmatically, +// and will never expire. +type Static struct { + Value +} + +// NewStaticV2 returns a pointer to a new Credentials object +// wrapping a static credentials value provider, signature is +// set to v2. If access and secret are not specified then +// regardless of signature type set it Value will return +// as anonymous. +func NewStaticV2(id, secret, token string) *Credentials { + return NewStatic(id, secret, token, SignatureV2) +} + +// NewStaticV4 is similar to NewStaticV2 with similar considerations. +func NewStaticV4(id, secret, token string) *Credentials { + return NewStatic(id, secret, token, SignatureV4) +} + +// NewStatic returns a pointer to a new Credentials object +// wrapping a static credentials value provider. +func NewStatic(id, secret, token string, signerType SignatureType) *Credentials { + return New(&Static{ + Value: Value{ + AccessKeyID: id, + SecretAccessKey: secret, + SessionToken: token, + SignerType: signerType, + }, + }) +} + +// Retrieve returns the static credentials. +func (s *Static) Retrieve() (Value, error) { + if s.AccessKeyID == "" || s.SecretAccessKey == "" { + // Anonymous is not an error + return Value{SignerType: SignatureAnonymous}, nil + } + return s.Value, nil +} + +// IsExpired returns if the credentials are expired. +// +// For Static, the credentials never expired. +func (s *Static) IsExpired() bool { + return false +} diff --git a/vendor/github.com/minio/minio-go/pkg/credentials/static_test.go b/vendor/github.com/minio/minio-go/pkg/credentials/static_test.go new file mode 100644 index 000000000..491b1554b --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/credentials/static_test.go @@ -0,0 +1,68 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package credentials + +import "testing" + +func TestStaticGet(t *testing.T) { + creds := NewStatic("UXHW", "SECRET", "", SignatureV4) + credValues, err := creds.Get() + if err != nil { + t.Fatal(err) + } + + if "UXHW" != credValues.AccessKeyID { + t.Errorf("Expected access key ID to match \"UXHW\", got %s", credValues.AccessKeyID) + } + if "SECRET" != credValues.SecretAccessKey { + t.Errorf("Expected secret access key to match \"SECRET\", got %s", credValues.SecretAccessKey) + } + + if credValues.SessionToken != "" { + t.Error("Expected session token to match") + } + + if credValues.SignerType != SignatureV4 { + t.Errorf("Expected 'S3v4', got %s", credValues.SignerType) + } + + if creds.IsExpired() { + t.Error("Static credentials should never expire") + } + + creds = NewStatic("", "", "", SignatureDefault) + credValues, err = creds.Get() + if err != nil { + t.Fatal(err) + } + + if "" != credValues.AccessKeyID { + t.Errorf("Expected access key ID to match empty string, got %s", credValues.AccessKeyID) + } + if "" != credValues.SecretAccessKey { + t.Errorf("Expected secret access key to match empty string, got %s", credValues.SecretAccessKey) + } + + if !credValues.SignerType.IsAnonymous() { + t.Errorf("Expected 'Anonymous', got %s", credValues.SignerType) + } + + if creds.IsExpired() { + t.Error("Static credentials should never expire") + } +} diff --git a/vendor/github.com/minio/minio-go/pkg/encrypt/cbc.go b/vendor/github.com/minio/minio-go/pkg/encrypt/cbc.go new file mode 100644 index 000000000..7670e68f4 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/encrypt/cbc.go @@ -0,0 +1,284 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package encrypt + +import ( + "bytes" + "crypto/aes" + "crypto/cipher" + "crypto/rand" + "encoding/base64" + "errors" + "io" +) + +// Crypt mode - encryption or decryption +type cryptMode int + +const ( + encryptMode cryptMode = iota + decryptMode +) + +// CBCSecureMaterials encrypts/decrypts data using AES CBC algorithm +type CBCSecureMaterials struct { + + // Data stream to encrypt/decrypt + stream io.Reader + + // Last internal error + err error + + // End of file reached + eof bool + + // Holds initial data + srcBuf *bytes.Buffer + + // Holds transformed data (encrypted or decrypted) + dstBuf *bytes.Buffer + + // Encryption algorithm + encryptionKey Key + + // Key to encrypts/decrypts data + contentKey []byte + + // Encrypted form of contentKey + cryptedKey []byte + + // Initialization vector + iv []byte + + // matDesc - currently unused + matDesc []byte + + // Indicate if we are going to encrypt or decrypt + cryptMode cryptMode + + // Helper that encrypts/decrypts data + blockMode cipher.BlockMode +} + +// NewCBCSecureMaterials builds new CBC crypter module with +// the specified encryption key (symmetric or asymmetric) +func NewCBCSecureMaterials(key Key) (*CBCSecureMaterials, error) { + if key == nil { + return nil, errors.New("Unable to recognize empty encryption properties") + } + return &CBCSecureMaterials{ + srcBuf: bytes.NewBuffer([]byte{}), + dstBuf: bytes.NewBuffer([]byte{}), + encryptionKey: key, + matDesc: []byte("{}"), + }, nil + +} + +// SetupEncryptMode - tells CBC that we are going to encrypt data +func (s *CBCSecureMaterials) SetupEncryptMode(stream io.Reader) error { + // Set mode to encrypt + s.cryptMode = encryptMode + + // Set underlying reader + s.stream = stream + + s.eof = false + s.srcBuf.Reset() + s.dstBuf.Reset() + + var err error + + // Generate random content key + s.contentKey = make([]byte, aes.BlockSize*2) + if _, err := rand.Read(s.contentKey); err != nil { + return err + } + // Encrypt content key + s.cryptedKey, err = s.encryptionKey.Encrypt(s.contentKey) + if err != nil { + return err + } + // Generate random IV + s.iv = make([]byte, aes.BlockSize) + if _, err = rand.Read(s.iv); err != nil { + return err + } + // New cipher + encryptContentBlock, err := aes.NewCipher(s.contentKey) + if err != nil { + return err + } + + s.blockMode = cipher.NewCBCEncrypter(encryptContentBlock, s.iv) + + return nil +} + +// SetupDecryptMode - tells CBC that we are going to decrypt data +func (s *CBCSecureMaterials) SetupDecryptMode(stream io.Reader, iv string, key string) error { + // Set mode to decrypt + s.cryptMode = decryptMode + + // Set underlying reader + s.stream = stream + + // Reset + s.eof = false + s.srcBuf.Reset() + s.dstBuf.Reset() + + var err error + + // Get IV + s.iv, err = base64.StdEncoding.DecodeString(iv) + if err != nil { + return err + } + + // Get encrypted content key + s.cryptedKey, err = base64.StdEncoding.DecodeString(key) + if err != nil { + return err + } + + // Decrypt content key + s.contentKey, err = s.encryptionKey.Decrypt(s.cryptedKey) + if err != nil { + return err + } + + // New cipher + decryptContentBlock, err := aes.NewCipher(s.contentKey) + if err != nil { + return err + } + + s.blockMode = cipher.NewCBCDecrypter(decryptContentBlock, s.iv) + return nil +} + +// GetIV - return randomly generated IV (per S3 object), base64 encoded. +func (s *CBCSecureMaterials) GetIV() string { + return base64.StdEncoding.EncodeToString(s.iv) +} + +// GetKey - return content encrypting key (cek) in encrypted form, base64 encoded. +func (s *CBCSecureMaterials) GetKey() string { + return base64.StdEncoding.EncodeToString(s.cryptedKey) +} + +// GetDesc - user provided encryption material description in JSON (UTF8) format. +func (s *CBCSecureMaterials) GetDesc() string { + return string(s.matDesc) +} + +// Fill buf with encrypted/decrypted data +func (s *CBCSecureMaterials) Read(buf []byte) (n int, err error) { + // Always fill buf from bufChunk at the end of this function + defer func() { + if s.err != nil { + n, err = 0, s.err + } else { + n, err = s.dstBuf.Read(buf) + } + }() + + // Return + if s.eof { + return + } + + // Fill dest buffer if its length is less than buf + for !s.eof && s.dstBuf.Len() < len(buf) { + + srcPart := make([]byte, aes.BlockSize) + dstPart := make([]byte, aes.BlockSize) + + // Fill src buffer + for s.srcBuf.Len() < aes.BlockSize*2 { + _, err = io.CopyN(s.srcBuf, s.stream, aes.BlockSize) + if err != nil { + break + } + } + + // Quit immediately for errors other than io.EOF + if err != nil && err != io.EOF { + s.err = err + return + } + + // Mark current encrypting/decrypting as finished + s.eof = (err == io.EOF) + + if s.eof && s.cryptMode == encryptMode { + if srcPart, err = pkcs5Pad(s.srcBuf.Bytes(), aes.BlockSize); err != nil { + s.err = err + return + } + } else { + _, _ = s.srcBuf.Read(srcPart) + } + + // Crypt srcPart content + for len(srcPart) > 0 { + + // Crypt current part + s.blockMode.CryptBlocks(dstPart, srcPart[:aes.BlockSize]) + + // Unpad when this is the last part and we are decrypting + if s.eof && s.cryptMode == decryptMode { + dstPart, err = pkcs5Unpad(dstPart, aes.BlockSize) + if err != nil { + s.err = err + return + } + } + + // Send crypted data to dstBuf + if _, wErr := s.dstBuf.Write(dstPart); wErr != nil { + s.err = wErr + return + } + // Move to the next part + srcPart = srcPart[aes.BlockSize:] + } + } + return +} + +// Unpad a set of bytes following PKCS5 algorithm +func pkcs5Unpad(buf []byte, blockSize int) ([]byte, error) { + len := len(buf) + if len == 0 { + return nil, errors.New("buffer is empty") + } + pad := int(buf[len-1]) + if pad > len || pad > blockSize { + return nil, errors.New("invalid padding size") + } + return buf[:len-pad], nil +} + +// Pad a set of bytes following PKCS5 algorithm +func pkcs5Pad(buf []byte, blockSize int) ([]byte, error) { + len := len(buf) + pad := blockSize - (len % blockSize) + padText := bytes.Repeat([]byte{byte(pad)}, pad) + return append(buf, padText...), nil +} diff --git a/vendor/github.com/minio/minio-go/pkg/encrypt/interface.go b/vendor/github.com/minio/minio-go/pkg/encrypt/interface.go new file mode 100644 index 000000000..2fd75033f --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/encrypt/interface.go @@ -0,0 +1,50 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package encrypt implements a generic interface to encrypt any stream of data. +// currently this package implements two types of encryption +// - Symmetric encryption using AES. +// - Asymmetric encrytion using RSA. +package encrypt + +import "io" + +// Materials - provides generic interface to encrypt any stream of data. +type Materials interface { + + // Returns encrypted/decrypted data, io.Reader compatible. + Read(b []byte) (int, error) + + // Get randomly generated IV, base64 encoded. + GetIV() (iv string) + + // Get content encrypting key (cek) in encrypted form, base64 encoded. + GetKey() (key string) + + // Get user provided encryption material description in + // JSON (UTF8) format. This is not used, kept for future. + GetDesc() (desc string) + + // Setup encrypt mode, further calls of Read() function + // will return the encrypted form of data streamed + // by the passed reader + SetupEncryptMode(stream io.Reader) error + + // Setup decrypted mode, further calls of Read() function + // will return the decrypted form of data streamed + // by the passed reader + SetupDecryptMode(stream io.Reader, iv string, key string) error +} diff --git a/vendor/github.com/minio/minio-go/pkg/encrypt/keys.go b/vendor/github.com/minio/minio-go/pkg/encrypt/keys.go new file mode 100644 index 000000000..8814845e3 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/encrypt/keys.go @@ -0,0 +1,165 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package encrypt + +import ( + "crypto/aes" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "errors" +) + +// Key - generic interface to encrypt/decrypt a key. +// We use it to encrypt/decrypt content key which is the key +// that encrypt/decrypt object data. +type Key interface { + // Encrypt data using to the set encryption key + Encrypt([]byte) ([]byte, error) + // Decrypt data using to the set encryption key + Decrypt([]byte) ([]byte, error) +} + +// SymmetricKey - encrypts data with a symmetric master key +type SymmetricKey struct { + masterKey []byte +} + +// Encrypt passed bytes +func (s *SymmetricKey) Encrypt(plain []byte) ([]byte, error) { + // Initialize an AES encryptor using a master key + keyBlock, err := aes.NewCipher(s.masterKey) + if err != nil { + return []byte{}, err + } + + // Pad the key before encryption + plain, _ = pkcs5Pad(plain, aes.BlockSize) + + encKey := []byte{} + encPart := make([]byte, aes.BlockSize) + + // Encrypt the passed key by block + for { + if len(plain) < aes.BlockSize { + break + } + // Encrypt the passed key + keyBlock.Encrypt(encPart, plain[:aes.BlockSize]) + // Add the encrypted block to the total encrypted key + encKey = append(encKey, encPart...) + // Pass to the next plain block + plain = plain[aes.BlockSize:] + } + return encKey, nil +} + +// Decrypt passed bytes +func (s *SymmetricKey) Decrypt(cipher []byte) ([]byte, error) { + // Initialize AES decrypter + keyBlock, err := aes.NewCipher(s.masterKey) + if err != nil { + return nil, err + } + + var plain []byte + plainPart := make([]byte, aes.BlockSize) + + // Decrypt the encrypted data block by block + for { + if len(cipher) < aes.BlockSize { + break + } + keyBlock.Decrypt(plainPart, cipher[:aes.BlockSize]) + // Add the decrypted block to the total result + plain = append(plain, plainPart...) + // Pass to the next cipher block + cipher = cipher[aes.BlockSize:] + } + + // Unpad the resulted plain data + plain, err = pkcs5Unpad(plain, aes.BlockSize) + if err != nil { + return nil, err + } + + return plain, nil +} + +// NewSymmetricKey generates a new encrypt/decrypt crypto using +// an AES master key password +func NewSymmetricKey(b []byte) *SymmetricKey { + return &SymmetricKey{masterKey: b} +} + +// AsymmetricKey - struct which encrypts/decrypts data +// using RSA public/private certificates +type AsymmetricKey struct { + publicKey *rsa.PublicKey + privateKey *rsa.PrivateKey +} + +// Encrypt data using public key +func (a *AsymmetricKey) Encrypt(plain []byte) ([]byte, error) { + cipher, err := rsa.EncryptPKCS1v15(rand.Reader, a.publicKey, plain) + if err != nil { + return nil, err + } + return cipher, nil +} + +// Decrypt data using public key +func (a *AsymmetricKey) Decrypt(cipher []byte) ([]byte, error) { + cipher, err := rsa.DecryptPKCS1v15(rand.Reader, a.privateKey, cipher) + if err != nil { + return nil, err + } + return cipher, nil +} + +// NewAsymmetricKey - generates a crypto module able to encrypt/decrypt +// data using a pair for private and public key +func NewAsymmetricKey(privData []byte, pubData []byte) (*AsymmetricKey, error) { + // Parse private key from passed data + priv, err := x509.ParsePKCS8PrivateKey(privData) + if err != nil { + return nil, err + } + privKey, ok := priv.(*rsa.PrivateKey) + if !ok { + return nil, errors.New("not a valid private key") + } + + // Parse public key from passed data + pub, err := x509.ParsePKIXPublicKey(pubData) + if err != nil { + return nil, err + } + + pubKey, ok := pub.(*rsa.PublicKey) + if !ok { + return nil, errors.New("not a valid public key") + } + + // Associate the private key with the passed public key + privKey.PublicKey = *pubKey + + return &AsymmetricKey{ + publicKey: pubKey, + privateKey: privKey, + }, nil +} diff --git a/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go b/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go index cbb889d8d..b2d46e178 100644 --- a/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go +++ b/vendor/github.com/minio/minio-go/pkg/policy/bucket-policy.go @@ -583,7 +583,7 @@ func GetPolicies(statements []Statement, bucketName string) map[string]BucketPol r = r[:len(r)-1] asterisk = "*" } - objectPath := r[len(awsResourcePrefix+bucketName)+1 : len(r)] + objectPath := r[len(awsResourcePrefix+bucketName)+1:] p := GetPolicy(statements, bucketName, objectPath) policyRules[bucketName+"/"+objectPath+asterisk] = p } diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go new file mode 100644 index 000000000..c2f0baee6 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming.go @@ -0,0 +1,290 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bytes" + "encoding/hex" + "fmt" + "io" + "net/http" + "strconv" + "strings" + "time" +) + +// Reference for constants used below - +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html#example-signature-calculations-streaming +const ( + streamingSignAlgorithm = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + streamingEncoding = "aws-chunked" + streamingPayloadHdr = "AWS4-HMAC-SHA256-PAYLOAD" + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + payloadChunkSize = 64 * 1024 + chunkSigConstLen = 17 // ";chunk-signature=" + signatureStrLen = 64 // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2" + crlfLen = 2 // CRLF +) + +// Request headers to be ignored while calculating seed signature for +// a request. +var ignoredStreamingHeaders = map[string]bool{ + "Authorization": true, + "User-Agent": true, + "Content-Type": true, +} + +// getSignedChunkLength - calculates the length of chunk metadata +func getSignedChunkLength(chunkDataSize int64) int64 { + return int64(len(fmt.Sprintf("%x", chunkDataSize))) + + chunkSigConstLen + + signatureStrLen + + crlfLen + + chunkDataSize + + crlfLen +} + +// getStreamLength - calculates the length of the overall stream (data + metadata) +func getStreamLength(dataLen, chunkSize int64) int64 { + if dataLen <= 0 { + return 0 + } + + chunksCount := int64(dataLen / chunkSize) + remainingBytes := int64(dataLen % chunkSize) + streamLen := int64(0) + streamLen += chunksCount * getSignedChunkLength(chunkSize) + if remainingBytes > 0 { + streamLen += getSignedChunkLength(remainingBytes) + } + streamLen += getSignedChunkLength(0) + return streamLen +} + +// buildChunkStringToSign - returns the string to sign given chunk data +// and previous signature. +func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string { + stringToSignParts := []string{ + streamingPayloadHdr, + t.Format(iso8601DateFormat), + getScope(region, t), + previousSig, + emptySHA256, + hex.EncodeToString(sum256(chunkData)), + } + + return strings.Join(stringToSignParts, "\n") +} + +// prepareStreamingRequest - prepares a request with appropriate +// headers before computing the seed signature. +func prepareStreamingRequest(req *http.Request, sessionToken string, dataLen int64, timestamp time.Time) { + // Set x-amz-content-sha256 header. + req.Header.Set("X-Amz-Content-Sha256", streamingSignAlgorithm) + if sessionToken != "" { + req.Header.Set("X-Amz-Security-Token", sessionToken) + } + req.Header.Set("Content-Encoding", streamingEncoding) + req.Header.Set("X-Amz-Date", timestamp.Format(iso8601DateFormat)) + + // Set content length with streaming signature for each chunk included. + req.ContentLength = getStreamLength(dataLen, int64(payloadChunkSize)) + req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLen, 10)) +} + +// buildChunkHeader - returns the chunk header. +// e.g string(IntHexBase(chunk-size)) + ";chunk-signature=" + signature + \r\n + chunk-data + \r\n +func buildChunkHeader(chunkLen int64, signature string) []byte { + return []byte(strconv.FormatInt(chunkLen, 16) + ";chunk-signature=" + signature + "\r\n") +} + +// buildChunkSignature - returns chunk signature for a given chunk and previous signature. +func buildChunkSignature(chunkData []byte, reqTime time.Time, region, + previousSignature, secretAccessKey string) string { + + chunkStringToSign := buildChunkStringToSign(reqTime, region, + previousSignature, chunkData) + signingKey := getSigningKey(secretAccessKey, region, reqTime) + return getSignature(signingKey, chunkStringToSign) +} + +// getSeedSignature - returns the seed signature for a given request. +func (s *StreamingReader) setSeedSignature(req *http.Request) { + // Get canonical request + canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders) + + // Get string to sign from canonical request. + stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest) + + signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime) + + // Calculate signature. + s.seedSignature = getSignature(signingKey, stringToSign) +} + +// StreamingReader implements chunked upload signature as a reader on +// top of req.Body's ReaderCloser chunk header;data;... repeat +type StreamingReader struct { + accessKeyID string + secretAccessKey string + sessionToken string + region string + prevSignature string + seedSignature string + contentLen int64 // Content-Length from req header + baseReadCloser io.ReadCloser // underlying io.Reader + bytesRead int64 // bytes read from underlying io.Reader + buf bytes.Buffer // holds signed chunk + chunkBuf []byte // holds raw data read from req Body + chunkBufLen int // no. of bytes read so far into chunkBuf + done bool // done reading the underlying reader to EOF + reqTime time.Time + chunkNum int + totalChunks int + lastChunkSize int +} + +// signChunk - signs a chunk read from s.baseReader of chunkLen size. +func (s *StreamingReader) signChunk(chunkLen int) { + // Compute chunk signature for next header + signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime, + s.region, s.prevSignature, s.secretAccessKey) + + // For next chunk signature computation + s.prevSignature = signature + + // Write chunk header into streaming buffer + chunkHdr := buildChunkHeader(int64(chunkLen), signature) + s.buf.Write(chunkHdr) + + // Write chunk data into streaming buffer + s.buf.Write(s.chunkBuf[:chunkLen]) + + // Write the chunk trailer. + s.buf.Write([]byte("\r\n")) + + // Reset chunkBufLen for next chunk read. + s.chunkBufLen = 0 + s.chunkNum++ +} + +// setStreamingAuthHeader - builds and sets authorization header value +// for streaming signature. +func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { + credential := GetCredential(s.accessKeyID, s.region, s.reqTime) + authParts := []string{ + signV4Algorithm + " Credential=" + credential, + "SignedHeaders=" + getSignedHeaders(*req, ignoredStreamingHeaders), + "Signature=" + s.seedSignature, + } + + // Set authorization header. + auth := strings.Join(authParts, ",") + req.Header.Set("Authorization", auth) +} + +// StreamingSignV4 - provides chunked upload signatureV4 support by +// implementing io.Reader. +func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken, + region string, dataLen int64, reqTime time.Time) *http.Request { + + // Set headers needed for streaming signature. + prepareStreamingRequest(req, sessionToken, dataLen, reqTime) + + stReader := &StreamingReader{ + baseReadCloser: req.Body, + accessKeyID: accessKeyID, + secretAccessKey: secretAccessKey, + sessionToken: sessionToken, + region: region, + reqTime: reqTime, + chunkBuf: make([]byte, payloadChunkSize), + contentLen: dataLen, + chunkNum: 1, + totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, + lastChunkSize: int(dataLen % payloadChunkSize), + } + + // Add the request headers required for chunk upload signing. + + // Compute the seed signature. + stReader.setSeedSignature(req) + + // Set the authorization header with the seed signature. + stReader.setStreamingAuthHeader(req) + + // Set seed signature as prevSignature for subsequent + // streaming signing process. + stReader.prevSignature = stReader.seedSignature + req.Body = stReader + + return req +} + +// Read - this method performs chunk upload signature providing a +// io.Reader interface. +func (s *StreamingReader) Read(buf []byte) (int, error) { + switch { + // After the last chunk is read from underlying reader, we + // never re-fill s.buf. + case s.done: + + // s.buf will be (re-)filled with next chunk when has lesser + // bytes than asked for. + case s.buf.Len() < len(buf): + s.chunkBufLen = 0 + for { + n1, err := s.baseReadCloser.Read(s.chunkBuf[s.chunkBufLen:]) + if err == nil || err == io.ErrUnexpectedEOF { + s.chunkBufLen += n1 + s.bytesRead += int64(n1) + + if s.chunkBufLen == payloadChunkSize || + (s.chunkNum == s.totalChunks-1 && + s.chunkBufLen == s.lastChunkSize) { + // Sign the chunk and write it to s.buf. + s.signChunk(s.chunkBufLen) + break + } + + } else if err == io.EOF { + // No more data left in baseReader - last chunk. + // Done reading the last chunk from baseReader. + s.done = true + + // bytes read from baseReader different than + // content length provided. + if s.bytesRead != s.contentLen { + return 0, io.ErrUnexpectedEOF + } + + // Sign the chunk and write it to s.buf. + s.signChunk(0) + break + + } else { + return 0, err + } + } + } + return s.buf.Read(buf) +} + +// Close - this method makes underlying io.ReadCloser's Close method available. +func (s *StreamingReader) Close() error { + return s.baseReadCloser.Close() +} diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go new file mode 100644 index 000000000..1f49f2234 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-streaming_test.go @@ -0,0 +1,106 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bytes" + "io/ioutil" + "testing" + "time" +) + +func TestGetSeedSignature(t *testing.T) { + accessKeyID := "AKIAIOSFODNN7EXAMPLE" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + dataLen := 66560 + data := bytes.Repeat([]byte("a"), dataLen) + body := ioutil.NopCloser(bytes.NewReader(data)) + + req := NewRequest("PUT", "/examplebucket/chunkObject.txt", body) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.URL.Host = "s3.amazonaws.com" + + reqTime, err := time.Parse("20060102T150405Z", "20130524T000000Z") + if err != nil { + t.Fatalf("Failed to parse time - %v", err) + } + + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime) + actualSeedSignature := req.Body.(*StreamingReader).seedSignature + + expectedSeedSignature := "007480502de61457e955731b0f5d191f7e6f54a8a0f6cc7974a5ebd887965686" + if actualSeedSignature != expectedSeedSignature { + t.Errorf("Expected %s but received %s", expectedSeedSignature, actualSeedSignature) + } +} + +func TestChunkSignature(t *testing.T) { + chunkData := bytes.Repeat([]byte("a"), 65536) + reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") + previousSignature := "4f232c4386841ef735655705268965c44a0e4690baa4adea153f7db9fa80a0a9" + location := "us-east-1" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + expectedSignature := "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648" + actualSignature := buildChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID) + if actualSignature != expectedSignature { + t.Errorf("Expected %s but received %s", expectedSignature, actualSignature) + } +} + +func TestSetStreamingAuthorization(t *testing.T) { + location := "us-east-1" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + accessKeyID := "AKIAIOSFODNN7EXAMPLE" + + req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.URL.Host = "s3.amazonaws.com" + + dataLen := int64(65 * 1024) + reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + + expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=007480502de61457e955731b0f5d191f7e6f54a8a0f6cc7974a5ebd887965686" + + actualAuthorization := req.Header.Get("Authorization") + if actualAuthorization != expectedAuthorization { + t.Errorf("Expected %s but received %s", expectedAuthorization, actualAuthorization) + } +} + +func TestStreamingReader(t *testing.T) { + reqTime, _ := time.Parse("20060102T150405Z", "20130524T000000Z") + location := "us-east-1" + secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" + accessKeyID := "AKIAIOSFODNN7EXAMPLE" + dataLen := int64(65 * 1024) + + req := NewRequest("PUT", "/examplebucket/chunkObject.txt", nil) + req.Header.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") + req.ContentLength = 65 * 1024 + req.URL.Host = "s3.amazonaws.com" + + baseReader := ioutil.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024))) + req.Body = baseReader + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + + b, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("Expected no error but received %v %d", err, len(b)) + } + req.Body.Close() +} diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go index e1ec6c02c..af0e91515 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v2.go @@ -316,7 +316,7 @@ func writeCanonicalizedResource(buf *bytes.Buffer, req http.Request, isPreSign b // Request parameters if len(vv[0]) > 0 { buf.WriteByte('=') - buf.WriteString(strings.Replace(url.QueryEscape(vv[0]), "+", "%20", -1)) + buf.WriteString(vv[0]) } } } diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go index 3322b67cc..0d75dc162 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature-v4.go @@ -70,7 +70,7 @@ const ( /// /// Is skipped for obvious reasons /// -var ignoredHeaders = map[string]bool{ +var v4IgnoredHeaders = map[string]bool{ "Authorization": true, "Content-Type": true, "Content-Length": true, @@ -122,7 +122,7 @@ func getHashedPayload(req http.Request) string { // getCanonicalHeaders generate a list of request headers for // signature. -func getCanonicalHeaders(req http.Request) string { +func getCanonicalHeaders(req http.Request, ignoredHeaders map[string]bool) string { var headers []string vals := make(map[string][]string) for k, vv := range req.Header { @@ -161,7 +161,7 @@ func getCanonicalHeaders(req http.Request) string { // getSignedHeaders generate all signed request headers. // i.e lexically sorted, semicolon-separated list of lowercase // request header names. -func getSignedHeaders(req http.Request) string { +func getSignedHeaders(req http.Request, ignoredHeaders map[string]bool) string { var headers []string for k := range req.Header { if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; ok { @@ -183,14 +183,14 @@ func getSignedHeaders(req http.Request) string { // <CanonicalHeaders>\n // <SignedHeaders>\n // <HashedPayload> -func getCanonicalRequest(req http.Request) string { +func getCanonicalRequest(req http.Request, ignoredHeaders map[string]bool) string { req.URL.RawQuery = strings.Replace(req.URL.Query().Encode(), "+", "%20", -1) canonicalRequest := strings.Join([]string{ req.Method, s3utils.EncodePath(req.URL.Path), req.URL.RawQuery, - getCanonicalHeaders(req), - getSignedHeaders(req), + getCanonicalHeaders(req, ignoredHeaders), + getSignedHeaders(req, ignoredHeaders), getHashedPayload(req), }, "\n") return canonicalRequest @@ -206,7 +206,7 @@ func getStringToSignV4(t time.Time, location, canonicalRequest string) string { // PreSignV4 presign the request, in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. -func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, expires int64) *http.Request { +func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request { // Presign is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req @@ -219,7 +219,7 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, credential := GetCredential(accessKeyID, location, t) // Get all signed headers. - signedHeaders := getSignedHeaders(req) + signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) // Set URL query. query := req.URL.Query() @@ -228,10 +228,14 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, location string, query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10)) query.Set("X-Amz-SignedHeaders", signedHeaders) query.Set("X-Amz-Credential", credential) + // Set session token if available. + if sessionToken != "" { + query.Set("X-Amz-Security-Token", sessionToken) + } req.URL.RawQuery = query.Encode() // Get canonical request. - canonicalRequest := getCanonicalRequest(req) + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) // Get string to sign from canonical request. stringToSign := getStringToSignV4(t, location, canonicalRequest) @@ -260,7 +264,7 @@ func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, l // SignV4 sign the request before Do(), in accordance with // http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html. -func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *http.Request { +func SignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request { // Signature calculation is not needed for anonymous credentials. if accessKeyID == "" || secretAccessKey == "" { return &req @@ -272,8 +276,13 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *ht // Set x-amz-date. req.Header.Set("X-Amz-Date", t.Format(iso8601DateFormat)) + // Set session token if available. + if sessionToken != "" { + req.Header.Set("X-Amz-Security-Token", sessionToken) + } + // Get canonical request. - canonicalRequest := getCanonicalRequest(req) + canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders) // Get string to sign from canonical request. stringToSign := getStringToSignV4(t, location, canonicalRequest) @@ -285,7 +294,7 @@ func SignV4(req http.Request, accessKeyID, secretAccessKey, location string) *ht credential := GetCredential(accessKeyID, location, t) // Get all signed headers. - signedHeaders := getSignedHeaders(req) + signedHeaders := getSignedHeaders(req, v4IgnoredHeaders) // Calculate signature. signature := getSignature(signingKey, stringToSign) diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go index 6f5ba1895..85ff063df 100644 --- a/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/request-signature_test.go @@ -28,12 +28,12 @@ func TestSignatureCalculation(t *testing.T) { if err != nil { t.Fatal("Error:", err) } - req = SignV4(*req, "", "", "us-east-1") + req = SignV4(*req, "", "", "", "us-east-1") if req.Header.Get("Authorization") != "" { t.Fatal("Error: anonymous credentials should not have Authorization header.") } - req = PreSignV4(*req, "", "", "us-east-1", 0) + req = PreSignV4(*req, "", "", "", "us-east-1", 0) if strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { t.Fatal("Error: anonymous credentials should not have Signature query resource.") } @@ -48,12 +48,12 @@ func TestSignatureCalculation(t *testing.T) { t.Fatal("Error: anonymous credentials should not have Signature query resource.") } - req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1") + req = SignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1") if req.Header.Get("Authorization") == "" { t.Fatal("Error: normal credentials should have Authorization header.") } - req = PreSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "us-east-1", 0) + req = PreSignV4(*req, "ACCESS-KEY", "SECRET-KEY", "", "us-east-1", 0) if !strings.Contains(req.URL.RawQuery, "X-Amz-Signature") { t.Fatal("Error: normal credentials should have Signature query resource.") } diff --git a/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go b/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go new file mode 100644 index 000000000..049e5813d --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/s3signer/test-utils_test.go @@ -0,0 +1,103 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3signer + +import ( + "bufio" + "bytes" + "crypto/tls" + "io" + "io/ioutil" + "net/http" + "strings" +) + +// N B minio-go should compile on go1.5.3 onwards and httptest package is +// available only from go.1.7.x. The following function is taken from +// Go httptest package to be able to build on older versions of Go. + +// NewRequest returns a new incoming server Request, suitable +// for passing to an http.Handler for testing. +// +// The target is the RFC 7230 "request-target": it may be either a +// path or an absolute URL. If target is an absolute URL, the host name +// from the URL is used. Otherwise, "example.com" is used. +// +// The TLS field is set to a non-nil dummy value if target has scheme +// "https". +// +// The Request.Proto is always HTTP/1.1. +// +// An empty method means "GET". +// +// The provided body may be nil. If the body is of type *bytes.Reader, +// *strings.Reader, or *bytes.Buffer, the Request.ContentLength is +// set. +// +// NewRequest panics on error for ease of use in testing, where a +// panic is acceptable. +func NewRequest(method, target string, body io.Reader) *http.Request { + if method == "" { + method = "GET" + } + req, err := http.ReadRequest(bufio.NewReader(strings.NewReader(method + " " + target + " HTTP/1.0\r\n\r\n"))) + if err != nil { + panic("invalid NewRequest arguments; " + err.Error()) + } + + // HTTP/1.0 was used above to avoid needing a Host field. Change it to 1.1 here. + req.Proto = "HTTP/1.1" + req.ProtoMinor = 1 + req.Close = false + + if body != nil { + switch v := body.(type) { + case *bytes.Buffer: + req.ContentLength = int64(v.Len()) + case *bytes.Reader: + req.ContentLength = int64(v.Len()) + case *strings.Reader: + req.ContentLength = int64(v.Len()) + default: + req.ContentLength = -1 + } + if rc, ok := body.(io.ReadCloser); ok { + req.Body = rc + } else { + req.Body = ioutil.NopCloser(body) + } + } + + // 192.0.2.0/24 is "TEST-NET" in RFC 5737 for use solely in + // documentation and example source code and should not be + // used publicly. + req.RemoteAddr = "192.0.2.1:1234" + + if req.Host == "" { + req.Host = "example.com" + } + + if strings.HasPrefix(target, "https://") { + req.TLS = &tls.ConnectionState{ + Version: tls.VersionTLS12, + HandshakeComplete: true, + ServerName: req.Host, + } + } + + return req +} diff --git a/vendor/github.com/minio/minio-go/request-headers.go b/vendor/github.com/minio/minio-go/request-headers.go new file mode 100644 index 000000000..fa23b2fe3 --- /dev/null +++ b/vendor/github.com/minio/minio-go/request-headers.go @@ -0,0 +1,111 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016-17 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "net/http" + "time" +) + +// RequestHeaders - implement methods for setting special +// request headers for GET, HEAD object operations. +// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html +type RequestHeaders struct { + http.Header +} + +// NewGetReqHeaders - initializes a new request headers for GET request. +func NewGetReqHeaders() RequestHeaders { + return RequestHeaders{ + Header: make(http.Header), + } +} + +// NewHeadReqHeaders - initializes a new request headers for HEAD request. +func NewHeadReqHeaders() RequestHeaders { + return RequestHeaders{ + Header: make(http.Header), + } +} + +// SetMatchETag - set match etag. +func (c RequestHeaders) SetMatchETag(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + c.Set("If-Match", etag) + return nil +} + +// SetMatchETagExcept - set match etag except. +func (c RequestHeaders) SetMatchETagExcept(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + c.Set("If-None-Match", etag) + return nil +} + +// SetUnmodified - set unmodified time since. +func (c RequestHeaders) SetUnmodified(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Modified since cannot be empty.") + } + c.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetModified - set modified time since. +func (c RequestHeaders) SetModified(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Modified since cannot be empty.") + } + c.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetRange - set the start and end offset of the object to be read. +// See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. +func (c RequestHeaders) SetRange(start, end int64) error { + switch { + case start == 0 && end < 0: + // Read last '-end' bytes. `bytes=-N`. + c.Set("Range", fmt.Sprintf("bytes=%d", end)) + case 0 < start && end == 0: + // Read everything starting from offset + // 'start'. `bytes=N-`. + c.Set("Range", fmt.Sprintf("bytes=%d-", start)) + case 0 <= start && start <= end: + // Read everything starting at 'start' till the + // 'end'. `bytes=N-M` + c.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + default: + // All other cases such as + // bytes=-3- + // bytes=5-3 + // bytes=-2-4 + // bytes=-3-0 + // bytes=-3--2 + // are invalid. + return ErrInvalidArgument( + fmt.Sprintf( + "Invalid range specified: start=%d end=%d", + start, end)) + } + return nil +} diff --git a/vendor/github.com/minio/minio-go/request-headers_test.go b/vendor/github.com/minio/minio-go/request-headers_test.go new file mode 100644 index 000000000..f026cd0a2 --- /dev/null +++ b/vendor/github.com/minio/minio-go/request-headers_test.go @@ -0,0 +1,56 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "testing" +) + +func TestSetHeader(t *testing.T) { + testCases := []struct { + start int64 + end int64 + errVal error + expected string + }{ + {0, 10, nil, "bytes=0-10"}, + {1, 10, nil, "bytes=1-10"}, + {5, 0, nil, "bytes=5-"}, + {0, -5, nil, "bytes=-5"}, + {0, 0, nil, "bytes=0-0"}, + {11, 10, fmt.Errorf("Invalid range specified: start=11 end=10"), + ""}, + {-1, 10, fmt.Errorf("Invalid range specified: start=-1 end=10"), ""}, + {-1, 0, fmt.Errorf("Invalid range specified: start=-1 end=0"), ""}, + {1, -5, fmt.Errorf("Invalid range specified: start=1 end=-5"), ""}, + } + for i, testCase := range testCases { + rh := NewGetReqHeaders() + err := rh.SetRange(testCase.start, testCase.end) + if err == nil && testCase.errVal != nil { + t.Errorf("Test %d: Expected to fail with '%v' but it passed", + i+1, testCase.errVal) + } else if err != nil && testCase.errVal.Error() != err.Error() { + t.Errorf("Test %d: Expected error '%v' but got error '%v'", + i+1, testCase.errVal, err) + } else if err == nil && rh.Get("Range") != testCase.expected { + t.Errorf("Test %d: Expected range header '%s', but got '%s'", + i+1, testCase.expected, rh.Get("Range")) + } + } +} diff --git a/vendor/github.com/minio/minio-go/retry.go b/vendor/github.com/minio/minio-go/retry.go index 41b70e474..1de5107e4 100644 --- a/vendor/github.com/minio/minio-go/retry.go +++ b/vendor/github.com/minio/minio-go/retry.go @@ -33,8 +33,16 @@ const MaxJitter = 1.0 // NoJitter disables the use of jitter for randomizing the exponential backoff time const NoJitter = 0.0 -// newRetryTimer creates a timer with exponentially increasing delays -// until the maximum retry attempts are reached. +// DefaultRetryUnit - default unit multiplicative per retry. +// defaults to 1 second. +const DefaultRetryUnit = time.Second + +// DefaultRetryCap - Each retry attempt never waits no longer than +// this maximum time duration. +const DefaultRetryCap = time.Second * 30 + +// newRetryTimer creates a timer with exponentially increasing +// delays until the maximum retry attempts are reached. func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duration, jitter float64, doneCh chan struct{}) <-chan int { attemptCh := make(chan int) @@ -78,6 +86,9 @@ func (c Client) newRetryTimer(maxRetry int, unit time.Duration, cap time.Duratio // isNetErrorRetryable - is network error retryable. func isNetErrorRetryable(err error) bool { + if err == nil { + return false + } switch err.(type) { case net.Error: switch err.(type) { @@ -96,6 +107,9 @@ func isNetErrorRetryable(err error) bool { } else if strings.Contains(err.Error(), "i/o timeout") { // If error is - tcp timeoutError, retry. return true + } else if strings.Contains(err.Error(), "connection timed out") { + // If err is a net.Dial timeout, retry. + return true } } } diff --git a/vendor/github.com/minio/minio-go/s3-error.go b/vendor/github.com/minio/minio-go/s3-error.go new file mode 100644 index 000000000..11b40a0f8 --- /dev/null +++ b/vendor/github.com/minio/minio-go/s3-error.go @@ -0,0 +1,60 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +// Non exhaustive list of AWS S3 standard error responses - +// http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html +var s3ErrorResponseMap = map[string]string{ + "AccessDenied": "Access Denied.", + "BadDigest": "The Content-Md5 you specified did not match what we received.", + "EntityTooSmall": "Your proposed upload is smaller than the minimum allowed object size.", + "EntityTooLarge": "Your proposed upload exceeds the maximum allowed object size.", + "IncompleteBody": "You did not provide the number of bytes specified by the Content-Length HTTP header.", + "InternalError": "We encountered an internal error, please try again.", + "InvalidAccessKeyID": "The access key ID you provided does not exist in our records.", + "InvalidBucketName": "The specified bucket is not valid.", + "InvalidDigest": "The Content-Md5 you specified is not valid.", + "InvalidRange": "The requested range is not satisfiable", + "MalformedXML": "The XML you provided was not well-formed or did not validate against our published schema.", + "MissingContentLength": "You must provide the Content-Length HTTP header.", + "MissingContentMD5": "Missing required header for this request: Content-Md5.", + "MissingRequestBodyError": "Request body is empty.", + "NoSuchBucket": "The specified bucket does not exist", + "NoSuchBucketPolicy": "The bucket policy does not exist", + "NoSuchKey": "The specified key does not exist.", + "NoSuchUpload": "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", + "NotImplemented": "A header you provided implies functionality that is not implemented", + "PreconditionFailed": "At least one of the pre-conditions you specified did not hold", + "RequestTimeTooSkewed": "The difference between the request time and the server's time is too large.", + "SignatureDoesNotMatch": "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + "MethodNotAllowed": "The specified method is not allowed against this resource.", + "InvalidPart": "One or more of the specified parts could not be found.", + "InvalidPartOrder": "The list of parts was not in ascending order. The parts list must be specified in order by part number.", + "InvalidObjectState": "The operation is not valid for the current state of the object.", + "AuthorizationHeaderMalformed": "The authorization header is malformed; the region is wrong.", + "MalformedPOSTRequest": "The body of your POST request is not well-formed multipart/form-data.", + "BucketNotEmpty": "The bucket you tried to delete is not empty", + "AllAccessDisabled": "All access to this bucket has been disabled.", + "MalformedPolicy": "Policy has invalid resource.", + "MissingFields": "Missing fields in request.", + "AuthorizationQueryParametersError": "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"<YOUR-AKID>/YYYYMMDD/REGION/SERVICE/aws4_request\".", + "MalformedDate": "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.", + "BucketAlreadyOwnedByYou": "Your previous request to create the named bucket succeeded and you already own it.", + "InvalidDuration": "Duration provided in the request is invalid.", + "XAmzContentSHA256Mismatch": "The provided 'x-amz-content-sha256' header does not match what was computed.", + // Add new API errors here. +} diff --git a/vendor/github.com/minio/minio-go/signature-type.go b/vendor/github.com/minio/minio-go/signature-type.go deleted file mode 100644 index cae74cd01..000000000 --- a/vendor/github.com/minio/minio-go/signature-type.go +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015 Minio, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package minio - -// SignatureType is type of Authorization requested for a given HTTP request. -type SignatureType int - -// Different types of supported signatures - default is Latest i.e SignatureV4. -const ( - Latest SignatureType = iota - SignatureV4 - SignatureV2 -) - -// isV2 - is signature SignatureV2? -func (s SignatureType) isV2() bool { - return s == SignatureV2 -} - -// isV4 - is signature SignatureV4? -func (s SignatureType) isV4() bool { - return s == SignatureV4 || s == Latest -} diff --git a/vendor/github.com/minio/minio-go/utils.go b/vendor/github.com/minio/minio-go/utils.go index 93cd1712f..4fa43b1ff 100644 --- a/vendor/github.com/minio/minio-go/utils.go +++ b/vendor/github.com/minio/minio-go/utils.go @@ -110,6 +110,8 @@ func closeResponse(resp *http.Response) { } } +var emptySHA256 = sum256(nil) + // Sentinel URL is the default url value which is invalid. var sentinelURL = url.URL{} diff --git a/vendor/github.com/minio/minio-go/utils_test.go b/vendor/github.com/minio/minio-go/utils_test.go index 99bdea329..4e015c855 100644 --- a/vendor/github.com/minio/minio-go/utils_test.go +++ b/vendor/github.com/minio/minio-go/utils_test.go @@ -57,13 +57,12 @@ func TestGetEndpointURL(t *testing.T) { {"s3.cn-north-1.amazonaws.com.cn", false, "http://s3.cn-north-1.amazonaws.com.cn", nil, true}, {"192.168.1.1:9000", false, "http://192.168.1.1:9000", nil, true}, {"192.168.1.1:9000", true, "https://192.168.1.1:9000", nil, true}, - {"192.168.1.1::9000", false, "", fmt.Errorf("too many colons in address %s", "192.168.1.1::9000"), false}, - {"13333.123123.-", true, "", fmt.Errorf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-"), false}, - {"13333.123123.-", true, "", fmt.Errorf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-"), false}, - {"s3.amazonaws.com:443", true, "", fmt.Errorf("Amazon S3 endpoint should be 's3.amazonaws.com'."), false}, - {"storage.googleapis.com:4000", true, "", fmt.Errorf("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, - {"s3.aamzza.-", true, "", fmt.Errorf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-"), false}, - {"", true, "", fmt.Errorf("Endpoint: does not follow ip address or domain name standards."), false}, + {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, + {"13333.123123.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "13333.123123.-")), false}, + {"s3.amazonaws.com:443", true, "", ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'."), false}, + {"storage.googleapis.com:4000", true, "", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, + {"s3.aamzza.-", true, "", ErrInvalidArgument(fmt.Sprintf("Endpoint: %s does not follow ip address or domain name standards.", "s3.aamzza.-")), false}, + {"", true, "", ErrInvalidArgument("Endpoint: does not follow ip address or domain name standards."), false}, } for i, testCase := range testCases { @@ -98,17 +97,17 @@ func TestIsValidEndpointURL(t *testing.T) { // Flag indicating whether the test is expected to pass or not. shouldPass bool }{ - {"", fmt.Errorf("Endpoint url cannot be empty."), false}, + {"", ErrInvalidArgument("Endpoint url cannot be empty."), false}, {"/", nil, true}, {"https://s3.am1;4205;0cazonaws.com", nil, true}, {"https://s3.cn-north-1.amazonaws.com.cn", nil, true}, {"https://s3.amazonaws.com/", nil, true}, {"https://storage.googleapis.com/", nil, true}, - {"192.168.1.1", fmt.Errorf("Endpoint url cannot have fully qualified paths."), false}, - {"https://amazon.googleapis.com/", fmt.Errorf("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, - {"https://storage.googleapis.com/bucket/", fmt.Errorf("Endpoint url cannot have fully qualified paths."), false}, - {"https://z3.amazonaws.com", fmt.Errorf("Amazon S3 endpoint should be 's3.amazonaws.com'."), false}, - {"https://s3.amazonaws.com/bucket/object", fmt.Errorf("Endpoint url cannot have fully qualified paths."), false}, + {"192.168.1.1", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"https://amazon.googleapis.com/", ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'."), false}, + {"https://storage.googleapis.com/bucket/", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, + {"https://z3.amazonaws.com", ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'."), false}, + {"https://s3.amazonaws.com/bucket/object", ErrInvalidArgument("Endpoint url cannot have fully qualified paths."), false}, } for i, testCase := range testCases { @@ -149,9 +148,9 @@ func TestIsValidExpiry(t *testing.T) { // Flag to indicate whether the test should pass. shouldPass bool }{ - {100 * time.Millisecond, fmt.Errorf("Expires cannot be lesser than 1 second."), false}, - {604801 * time.Second, fmt.Errorf("Expires cannot be greater than 7 days."), false}, - {0 * time.Second, fmt.Errorf("Expires cannot be lesser than 1 second."), false}, + {100 * time.Millisecond, ErrInvalidArgument("Expires cannot be lesser than 1 second."), false}, + {604801 * time.Second, ErrInvalidArgument("Expires cannot be greater than 7 days."), false}, + {0 * time.Second, ErrInvalidArgument("Expires cannot be lesser than 1 second."), false}, {1 * time.Second, nil, true}, {10000 * time.Second, nil, true}, {999 * time.Second, nil, true}, |