diff options
Diffstat (limited to 'vendor/github.com/minio/minio-go/api.go')
| -rw-r--r-- | vendor/github.com/minio/minio-go/api.go | 297 |
1 files changed, 177 insertions, 120 deletions
diff --git a/vendor/github.com/minio/minio-go/api.go b/vendor/github.com/minio/minio-go/api.go index a563a18d4..6fe508aa1 100644 --- a/vendor/github.com/minio/minio-go/api.go +++ b/vendor/github.com/minio/minio-go/api.go @@ -1,5 +1,6 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage + * (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,23 +19,27 @@ package minio import ( "bytes" + "crypto/md5" + "crypto/sha256" "encoding/base64" "encoding/hex" "errors" "fmt" + "hash" "io" "io/ioutil" "math/rand" + "net" "net/http" "net/http/httputil" "net/url" "os" - "regexp" "runtime" "strings" "sync" "time" + "github.com/minio/minio-go/pkg/credentials" "github.com/minio/minio-go/pkg/s3signer" "github.com/minio/minio-go/pkg/s3utils" ) @@ -46,14 +51,11 @@ type Client struct { // Parsed endpoint url provided by the user. endpointURL url.URL - // AccessKeyID required for authorized requests. - accessKeyID string - // SecretAccessKey required for authorized requests. - secretAccessKey string - // Choose a signature type if necessary. - signature SignatureType - // Set to 'true' if Client has no access and secret keys. - anonymous bool + // Holds various credential providers. + credsProvider *credentials.Credentials + + // Custom signerType value overrides all credentials. + overrideSignerType credentials.SignatureType // User supplied. appInfo struct { @@ -85,7 +87,7 @@ type Client struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "2.1.0" + libraryVersion = "3.0.0" ) // User Agent should always following the below style. @@ -100,58 +102,58 @@ const ( // NewV2 - instantiate minio client with Amazon S3 signature version // '2' compatibility. func NewV2(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, secure) + creds := credentials.NewStaticV2(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "") if err != nil { return nil, err } - - // Set to use signature version '2'. - clnt.signature = SignatureV2 + clnt.overrideSignerType = credentials.SignatureV2 return clnt, nil } // NewV4 - instantiate minio client with Amazon S3 signature version // '4' compatibility. func NewV4(endpoint string, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, secure) + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "") if err != nil { return nil, err } - - // Set to use signature version '4'. - clnt.signature = SignatureV4 + clnt.overrideSignerType = credentials.SignatureV4 return clnt, nil } -// New - instantiate minio client Client, adds automatic verification of signature. +// New - instantiate minio client, adds automatic verification of signature. func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { - return NewWithRegion(endpoint, accessKeyID, secretAccessKey, secure, "") -} - -// NewWithRegion - instantiate minio client, with region configured. Unlike New(), -// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. -// Use this function when if your application deals with single region. -func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { - clnt, err := privateNew(endpoint, accessKeyID, secretAccessKey, secure) + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + clnt, err := privateNew(endpoint, creds, secure, "") if err != nil { return nil, err } - // Google cloud storage should be set to signature V2, force it if not. if s3utils.IsGoogleEndpoint(clnt.endpointURL) { - clnt.signature = SignatureV2 + clnt.overrideSignerType = credentials.SignatureV2 } - - // If Amazon S3 set to signature v2.n + // If Amazon S3 set to signature v4. if s3utils.IsAmazonEndpoint(clnt.endpointURL) { - clnt.signature = SignatureV4 + clnt.overrideSignerType = credentials.SignatureV4 } + return clnt, nil +} - // Sets custom region, if region is empty bucket location cache is used automatically. - clnt.region = region +// NewWithCredentials - instantiate minio client with credentials provider +// for retrieving credentials from various credentials provider such as +// IAM, File, Env etc. +func NewWithCredentials(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { + return privateNew(endpoint, creds, secure, region) +} - // Success.. - return clnt, nil +// NewWithRegion - instantiate minio client, with region configured. Unlike New(), +// NewWithRegion avoids bucket-location lookup operations and it is slightly faster. +// Use this function when if your application deals with single region. +func NewWithRegion(endpoint, accessKeyID, secretAccessKey string, secure bool, region string) (*Client, error) { + creds := credentials.NewStaticV4(accessKeyID, secretAccessKey, "") + return privateNew(endpoint, creds, secure, region) } // lockedRandSource provides protected rand source, implements rand.Source interface. @@ -188,7 +190,7 @@ func redirectHeaders(req *http.Request, via []*http.Request) error { return nil } -func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, error) { +func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { // construct endpoint. endpointURL, err := getEndpointURL(endpoint, secure) if err != nil { @@ -197,8 +199,9 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Cl // instantiate new Client. clnt := new(Client) - clnt.accessKeyID = accessKeyID - clnt.secretAccessKey = secretAccessKey + + // Save the credentials. + clnt.credsProvider = creds // Remember whether we are using https or not clnt.secure = secure @@ -212,7 +215,10 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Cl CheckRedirect: redirectHeaders, } - // Instantiae bucket location cache. + // Sets custom region, if region is empty bucket location cache is used automatically. + clnt.region = region + + // Instantiate bucket location cache. clnt.bucketLocCache = newBucketLocationCache() // Introduce a new locked random seed. @@ -286,6 +292,29 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { } } +// Hash materials provides relevant initialized hash algo writers +// based on the expected signature type. +// +// - For signature v4 request if the connection is insecure compute only sha256. +// - For signature v4 request if the connection is secure compute only md5. +// - For anonymous request compute md5. +func (c *Client) hashMaterials() (hashAlgos map[string]hash.Hash, hashSums map[string][]byte) { + hashSums = make(map[string][]byte) + hashAlgos = make(map[string]hash.Hash) + if c.overrideSignerType.IsV4() { + if c.secure { + hashAlgos["md5"] = md5.New() + } else { + hashAlgos["sha256"] = sha256.New() + } + } else { + if c.overrideSignerType.IsAnonymous() { + hashAlgos["md5"] = md5.New() + } + } + return hashAlgos, hashSums +} + // requestMetadata - is container for all the values to make a request. type requestMetadata struct { // If set newRequest presigns the URL. @@ -306,40 +335,6 @@ type requestMetadata struct { contentMD5Bytes []byte } -// regCred matches credential string in HTTP header -var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/") - -// regCred matches signature string in HTTP header -var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)") - -// Filter out signature value from Authorization header. -func (c Client) filterSignature(req *http.Request) { - if _, ok := req.Header["Authorization"]; !ok { - return - } - // Handle if Signature V2. - if c.signature.isV2() { - // Set a temporary redacted auth - req.Header.Set("Authorization", "AWS **REDACTED**:**REDACTED**") - return - } - - /// Signature V4 authorization header. - - // Save the original auth. - origAuth := req.Header.Get("Authorization") - // Strip out accessKeyID from: - // Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request - newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/") - - // Strip out 256-bit signature from: Signature=<256-bit signature> - newAuth = regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**") - - // Set a temporary redacted auth - req.Header.Set("Authorization", newAuth) - return -} - // dumpHTTP - dump HTTP request and response. func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { // Starts http dump. @@ -349,7 +344,10 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { } // Filter out Signature field from Authorization header. - c.filterSignature(req) + origAuth := req.Header.Get("Authorization") + if origAuth != "" { + req.Header.Set("Authorization", redactSignature(origAuth)) + } // Only display request header. reqTrace, err := httputil.DumpRequestOut(req, false) @@ -478,6 +476,13 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt case os.Stdin, os.Stdout, os.Stderr: isRetryable = false } + // Figure out if the body can be closed - if yes + // we will definitely close it upon the function + // return. + bodyCloser, ok := metadata.contentBody.(io.Closer) + if ok { + defer bodyCloser.Close() + } } // Create a done channel to control 'newRetryTimer' go routine. @@ -553,9 +558,14 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Bucket region if set in error response and the error // code dictates invalid region, we can retry the request // with the new region. - if errResponse.Code == "InvalidRegion" && errResponse.Region != "" { - c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) - continue // Retry. + // + // Additionally we should only retry if bucketLocation and custom + // region is empty. + if metadata.bucketLocation == "" && c.region == "" { + if res.StatusCode == http.StatusBadRequest && errResponse.Region != "" { + c.bucketLocCache.Set(metadata.bucketName, errResponse.Region) + continue // Retry. + } } // Verify if error response code is retryable. @@ -581,53 +591,72 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R method = "POST" } - // Default all requests to "us-east-1" or "cn-north-1" (china region) - location := "us-east-1" - if s3utils.IsAmazonChinaEndpoint(c.endpointURL) { - // For china specifically we need to set everything to - // cn-north-1 for now, there is no easier way until AWS S3 - // provides a cleaner compatible API across "us-east-1" and - // China region. - location = "cn-north-1" - } - - // Gather location only if bucketName is present. - if metadata.bucketName != "" { - location, err = c.getBucketLocation(metadata.bucketName) - if err != nil { - return nil, err + location := metadata.bucketLocation + if location == "" { + if metadata.bucketName != "" { + // Gather location only if bucketName is present. + location, err = c.getBucketLocation(metadata.bucketName) + if err != nil { + if ToErrorResponse(err).Code != "AccessDenied" { + return nil, err + } + } + // Upon AccessDenied error on fetching bucket location, default + // to possible locations based on endpoint URL. This can usually + // happen when GetBucketLocation() is disabled using IAM policies. + } + if location == "" { + location = getDefaultLocation(c.endpointURL, c.region) } } - // Save location. - metadata.bucketLocation = location - // Construct a new target URL. - targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, metadata.bucketLocation, metadata.queryValues) + targetURL, err := c.makeTargetURL(metadata.bucketName, metadata.objectName, location, metadata.queryValues) if err != nil { return nil, err } // Initialize a new HTTP request for the method. - req, err = http.NewRequest(method, targetURL.String(), metadata.contentBody) + req, err = http.NewRequest(method, targetURL.String(), nil) if err != nil { return nil, err } - // Anonymous request. - anonymous := c.accessKeyID == "" || c.secretAccessKey == "" + // Get credentials from the configured credentials provider. + value, err := c.credsProvider.Get() + if err != nil { + return nil, err + } + + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous + } // Generate presign url if needed, return right here. if metadata.expires != 0 && metadata.presignURL { - if anonymous { - return nil, ErrInvalidArgument("Requests cannot be presigned with anonymous credentials.") + if signerType.IsAnonymous() { + return nil, ErrInvalidArgument("Presigned URLs cannot be generated with anonymous credentials.") } - if c.signature.isV2() { + if signerType.IsV2() { // Presign URL with signature v2. - req = s3signer.PreSignV2(*req, c.accessKeyID, c.secretAccessKey, metadata.expires) - } else if c.signature.isV4() { + req = s3signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires) + } else if signerType.IsV4() { // Presign URL with signature v4. - req = s3signer.PreSignV4(*req, c.accessKeyID, c.secretAccessKey, location, metadata.expires) + req = s3signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires) } return req, nil } @@ -640,9 +669,21 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R req.Header.Set(k, v[0]) } - // set incoming content-length. - if metadata.contentLength > 0 { - req.ContentLength = metadata.contentLength + // Go net/http notoriously closes the request body. + // - The request Body, if non-nil, will be closed by the underlying Transport, even on errors. + // This can cause underlying *os.File seekers to fail, avoid that + // by making sure to wrap the closer as a nop. + if metadata.contentLength == 0 { + req.Body = nil + } else { + req.Body = ioutil.NopCloser(metadata.contentBody) + } + + // Set incoming content-length. + req.ContentLength = metadata.contentLength + if req.ContentLength <= -1 { + // For unknown content length, we upload using transfer-encoding: chunked. + req.TransferEncoding = []string{"chunked"} } // set md5Sum for content protection. @@ -650,17 +691,18 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(metadata.contentMD5Bytes)) } - if anonymous { + // For anonymous requests just return. + if signerType.IsAnonymous() { return req, nil - } // Sign the request for all authenticated requests. + } switch { - case c.signature.isV2(): + case signerType.IsV2(): // Add signature version '2' authorization header. - req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey) - case c.signature.isStreamingV4() && method == "PUT": - req = s3signer.StreamingSignV4(req, c.accessKeyID, - c.secretAccessKey, location, metadata.contentLength, time.Now().UTC()) + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + case signerType.IsStreamingV4() && method == "PUT": + req = s3signer.StreamingSignV4(req, accessKeyID, + secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) default: // Set sha256 sum for signature calculation only with signature version '4'. shaHeader := unsignedPayload @@ -670,7 +712,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R req.Header.Set("X-Amz-Content-Sha256", shaHeader) // Add signature version '4' authorization header. - req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, location) + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, location) } // Return request. @@ -701,14 +743,26 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html host = c.s3AccelerateEndpoint } else { - // Fetch new host based on the bucket location. - host = getS3Endpoint(bucketLocation) + // Do not change the host if the endpoint URL is a FIPS S3 endpoint. + if !s3utils.IsAmazonFIPSGovCloudEndpoint(c.endpointURL) { + // Fetch new host based on the bucket location. + host = getS3Endpoint(bucketLocation) + } } } // Save scheme. scheme := c.endpointURL.Scheme + // Strip port 80 and 443 so we won't send these ports in Host header. + // The reason is that browsers and curl automatically remove :80 and :443 + // with the generated presigned urls, then a signature mismatch error. + if h, p, err := net.SplitHostPort(host); err == nil { + if scheme == "http" && p == "80" || scheme == "https" && p == "443" { + host = h + } + } + urlStr := scheme + "://" + host + "/" // Make URL only if bucketName is available, otherwise use the // endpoint URL. @@ -732,13 +786,16 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que } } } + // If there are any query values, add them to the end. if len(queryValues) > 0 { urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) } + u, err := url.Parse(urlStr) if err != nil { return nil, err } + return u, nil } |