From 961c04cae992eadb42d286d2f85f8a675bdc68c8 Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Mon, 29 Jan 2018 14:17:40 -0800 Subject: Upgrading server dependancies (#8154) --- vendor/github.com/minio/minio-go/api.go | 185 +++++++++++++++++--------------- 1 file changed, 100 insertions(+), 85 deletions(-) (limited to 'vendor/github.com/minio/minio-go/api.go') diff --git a/vendor/github.com/minio/minio-go/api.go b/vendor/github.com/minio/minio-go/api.go index 946a58869..681853849 100644 --- a/vendor/github.com/minio/minio-go/api.go +++ b/vendor/github.com/minio/minio-go/api.go @@ -1,6 +1,6 @@ /* * Minio Go Library for Amazon S3 Compatible Cloud Storage - * (C) 2015, 2016, 2017 Minio, Inc. + * Copyright 2015-2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -19,10 +19,9 @@ package minio import ( "bytes" + "context" "crypto/md5" "crypto/sha256" - "encoding/base64" - "encoding/hex" "errors" "fmt" "hash" @@ -49,7 +48,7 @@ type Client struct { /// Standard options. // Parsed endpoint url provided by the user. - endpointURL url.URL + endpointURL *url.URL // Holds various credential providers. credsProvider *credentials.Credentials @@ -87,7 +86,7 @@ type Client struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "3.0.3" + libraryVersion = "4.0.6" ) // User Agent should always following the below style. @@ -131,11 +130,11 @@ func New(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Client, e return nil, err } // Google cloud storage should be set to signature V2, force it if not. - if s3utils.IsGoogleEndpoint(clnt.endpointURL) { + if s3utils.IsGoogleEndpoint(*clnt.endpointURL) { clnt.overrideSignerType = credentials.SignatureV2 } // If Amazon S3 set to signature v4. - if s3utils.IsAmazonEndpoint(clnt.endpointURL) { + if s3utils.IsAmazonEndpoint(*clnt.endpointURL) { clnt.overrideSignerType = credentials.SignatureV4 } return clnt, nil @@ -178,41 +177,66 @@ func (r *lockedRandSource) Seed(seed int64) { r.lk.Unlock() } -// redirectHeaders copies all headers when following a redirect URL. -// This won't be needed anymore from go 1.8 (https://github.com/golang/go/issues/4800) -func redirectHeaders(req *http.Request, via []*http.Request) error { +// Redirect requests by re signing the request. +func (c *Client) redirectHeaders(req *http.Request, via []*http.Request) error { + if len(via) >= 5 { + return errors.New("stopped after 5 redirects") + } if len(via) == 0 { return nil } - for key, val := range via[0].Header { - req.Header[key] = val + lastRequest := via[len(via)-1] + var reAuth bool + for attr, val := range lastRequest.Header { + // if hosts do not match do not copy Authorization header + if attr == "Authorization" && req.Host != lastRequest.Host { + reAuth = true + continue + } + if _, ok := req.Header[attr]; !ok { + req.Header[attr] = val + } + } + + *c.endpointURL = *req.URL + + value, err := c.credsProvider.Get() + if err != nil { + return err + } + var ( + signerType = value.SignerType + accessKeyID = value.AccessKeyID + secretAccessKey = value.SecretAccessKey + sessionToken = value.SessionToken + region = c.region + ) + + // Custom signer set then override the behavior. + if c.overrideSignerType != credentials.SignatureDefault { + signerType = c.overrideSignerType + } + + // If signerType returned by credentials helper is anonymous, + // then do not sign regardless of signerType override. + if value.SignerType == credentials.SignatureAnonymous { + signerType = credentials.SignatureAnonymous } - return nil -} -// getRegionFromURL - parse region from URL if present. -func getRegionFromURL(u url.URL) (region string) { - region = "" - if s3utils.IsGoogleEndpoint(u) { - return - } else if s3utils.IsAmazonChinaEndpoint(u) { - // For china specifically we need to set everything to - // cn-north-1 for now, there is no easier way until AWS S3 - // provides a cleaner compatible API across "us-east-1" and - // China region. - return "cn-north-1" - } else if s3utils.IsAmazonGovCloudEndpoint(u) { - // For us-gov specifically we need to set everything to - // us-gov-west-1 for now, there is no easier way until AWS S3 - // provides a cleaner compatible API across "us-east-1" and - // Gov cloud region. - return "us-gov-west-1" - } - parts := s3utils.AmazonS3Host.FindStringSubmatch(u.Host) - if len(parts) > 1 { - region = parts[1] - } - return region + if reAuth { + // Check if there is no region override, if not get it from the URL if possible. + if region == "" { + region = s3utils.GetRegionFromURL(*c.endpointURL) + } + switch { + case signerType.IsV2(): + // Add signature version '2' authorization header. + req = s3signer.SignV2(*req, accessKeyID, secretAccessKey) + case signerType.IsV4(): + req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, getDefaultLocation(*c.endpointURL, region)) + } + } + return nil } func privateNew(endpoint string, creds *credentials.Credentials, secure bool, region string) (*Client, error) { @@ -232,17 +256,17 @@ func privateNew(endpoint string, creds *credentials.Credentials, secure bool, re clnt.secure = secure // Save endpoint URL, user agent for future uses. - clnt.endpointURL = *endpointURL + clnt.endpointURL = endpointURL // Instantiate http client and bucket location cache. clnt.httpClient = &http.Client{ Transport: defaultMinioTransport, - CheckRedirect: redirectHeaders, + CheckRedirect: clnt.redirectHeaders, } // Sets custom region, if region is empty bucket location cache is used automatically. if region == "" { - region = getRegionFromURL(clnt.endpointURL) + region = s3utils.GetRegionFromURL(*clnt.endpointURL) } clnt.region = region @@ -315,7 +339,7 @@ func (c *Client) TraceOff() { // please vist - // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) { - if s3utils.IsAmazonEndpoint(c.endpointURL) { + if s3utils.IsAmazonEndpoint(*c.endpointURL) { c.s3AccelerateEndpoint = accelerateEndpoint } } @@ -356,11 +380,11 @@ type requestMetadata struct { expires int64 // Generated by our internal code. - bucketLocation string - contentBody io.Reader - contentLength int64 - contentSHA256Bytes []byte - contentMD5Bytes []byte + bucketLocation string + contentBody io.Reader + contentLength int64 + contentMD5Base64 string // carries base64 encoded md5sum + contentSHA256Hex string // carries hex encoded sha256sum } // dumpHTTP - dump HTTP request and response. @@ -419,6 +443,7 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { } } } + // Write response to trace output. _, err = fmt.Fprint(c.traceOutput, strings.TrimSuffix(string(respTrace), "\r\n")) if err != nil { @@ -437,38 +462,22 @@ func (c Client) dumpHTTP(req *http.Request, resp *http.Response) error { // do - execute http request. func (c Client) do(req *http.Request) (*http.Response, error) { - var resp *http.Response - var err error - // Do the request in a loop in case of 307 http is met since golang still doesn't - // handle properly this situation (https://github.com/golang/go/issues/7912) - for { - resp, err = c.httpClient.Do(req) - if err != nil { - // Handle this specifically for now until future Golang - // versions fix this issue properly. - urlErr, ok := err.(*url.Error) - if ok && strings.Contains(urlErr.Err.Error(), "EOF") { + resp, err := c.httpClient.Do(req) + if err != nil { + // Handle this specifically for now until future Golang versions fix this issue properly. + if urlErr, ok := err.(*url.Error); ok { + if strings.Contains(urlErr.Err.Error(), "EOF") { return nil, &url.Error{ Op: urlErr.Op, URL: urlErr.URL, Err: errors.New("Connection closed by foreign host " + urlErr.URL + ". Retry again."), } } - return nil, err - } - // Redo the request with the new redirect url if http 307 is returned, quit the loop otherwise - if resp != nil && resp.StatusCode == http.StatusTemporaryRedirect { - newURL, err := url.Parse(resp.Header.Get("Location")) - if err != nil { - break - } - req.URL = newURL - } else { - break } + return nil, err } - // Response cannot be non-nil, report if its the case. + // Response cannot be non-nil, report error if thats the case. if resp == nil { msg := "Response is empty. " + reportIssue return nil, ErrInvalidArgument(msg) @@ -481,6 +490,7 @@ func (c Client) do(req *http.Request) (*http.Response, error) { return nil, err } } + return resp, nil } @@ -494,9 +504,11 @@ var successStatus = []int{ // executeMethod - instantiates a given method, and retries the // request upon any error up to maxRetries attempts in a binomially // delayed manner using a standard back off algorithm. -func (c Client) executeMethod(method string, metadata requestMetadata) (res *http.Response, err error) { +func (c Client) executeMethod(ctx context.Context, method string, metadata requestMetadata) (res *http.Response, err error) { var isRetryable bool // Indicates if request can be retried. var bodySeeker io.Seeker // Extracted seeker from io.Reader. + var reqRetry = MaxRetry // Indicates how many times we can retry the request + if metadata.contentBody != nil { // Check if body is seekable then it is retryable. bodySeeker, isRetryable = metadata.contentBody.(io.Seeker) @@ -504,6 +516,11 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt case os.Stdin, os.Stdout, os.Stderr: isRetryable = false } + // Retry only when reader is seekable + if !isRetryable { + reqRetry = 1 + } + // Figure out if the body can be closed - if yes // we will definitely close it upon the function // return. @@ -522,7 +539,7 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt // Blank indentifier is kept here on purpose since 'range' without // blank identifiers is only supported since go1.4 // https://golang.org/doc/go1.4#forrange. - for range c.newRetryTimer(MaxRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { + for range c.newRetryTimer(reqRetry, DefaultRetryUnit, DefaultRetryCap, MaxJitter, doneCh) { // Retry executes the following function body if request has an // error until maxRetries have been exhausted, retry attempts are // performed after waiting for a given period of time in a @@ -546,6 +563,9 @@ func (c Client) executeMethod(method string, metadata requestMetadata) (res *htt return nil, err } + // Add context to request + req = req.WithContext(ctx) + // Initiate the request. res, err = c.do(req) if err != nil { @@ -639,7 +659,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R // happen when GetBucketLocation() is disabled using IAM policies. } if location == "" { - location = getDefaultLocation(c.endpointURL, c.region) + location = getDefaultLocation(*c.endpointURL, c.region) } } @@ -720,8 +740,8 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R } // set md5Sum for content protection. - if metadata.contentMD5Bytes != nil { - req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(metadata.contentMD5Bytes)) + if len(metadata.contentMD5Base64) > 0 { + req.Header.Set("Content-Md5", metadata.contentMD5Base64) } // For anonymous requests just return. @@ -742,8 +762,8 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R default: // Set sha256 sum for signature calculation only with signature version '4'. shaHeader := unsignedPayload - if len(metadata.contentSHA256Bytes) > 0 { - shaHeader = hex.EncodeToString(metadata.contentSHA256Bytes) + if metadata.contentSHA256Hex != "" { + shaHeader = metadata.contentSHA256Hex } req.Header.Set("X-Amz-Content-Sha256", shaHeader) @@ -767,7 +787,7 @@ func (c Client) setUserAgent(req *http.Request) { func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, queryValues url.Values) (*url.URL, error) { host := c.endpointURL.Host // For Amazon S3 endpoint, try to fetch location based endpoint. - if s3utils.IsAmazonEndpoint(c.endpointURL) { + if s3utils.IsAmazonEndpoint(*c.endpointURL) { if c.s3AccelerateEndpoint != "" && bucketName != "" { // http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html // Disable transfer acceleration for non-compliant bucket names. @@ -780,7 +800,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que host = c.s3AccelerateEndpoint } else { // Do not change the host if the endpoint URL is a FIPS S3 endpoint. - if !s3utils.IsAmazonFIPSGovCloudEndpoint(c.endpointURL) { + if !s3utils.IsAmazonFIPSGovCloudEndpoint(*c.endpointURL) { // Fetch new host based on the bucket location. host = getS3Endpoint(bucketLocation) } @@ -804,7 +824,7 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que // endpoint URL. if bucketName != "" { // Save if target url will have buckets which suppport virtual host. - isVirtualHostStyle := s3utils.IsVirtualHostSupported(c.endpointURL, bucketName) + isVirtualHostStyle := s3utils.IsVirtualHostSupported(*c.endpointURL, bucketName) // If endpoint supports virtual host style use that always. // Currently only S3 and Google Cloud Storage would support @@ -828,10 +848,5 @@ func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, que urlStr = urlStr + "?" + s3utils.QueryEncode(queryValues) } - u, err := url.Parse(urlStr) - if err != nil { - return nil, err - } - - return u, nil + return url.Parse(urlStr) } -- cgit v1.2.3-1-g7c22