From 84d2482ddbff9564c9ad75b2d30af66e3ddfd44d Mon Sep 17 00:00:00 2001 From: Christopher Speller Date: Thu, 12 May 2016 15:08:58 -0400 Subject: Updating go depencancies. Switching to go1.6 vendoring (#2949) --- vendor/github.com/goamz/goamz/LICENSE | 185 ++++ vendor/github.com/goamz/goamz/aws/attempt.go | 74 ++ vendor/github.com/goamz/goamz/aws/aws.go | 432 ++++++++++ vendor/github.com/goamz/goamz/aws/client.go | 124 +++ vendor/github.com/goamz/goamz/aws/regions.go | 254 ++++++ vendor/github.com/goamz/goamz/aws/sign.go | 357 ++++++++ vendor/github.com/goamz/goamz/s3/multi.go | 439 ++++++++++ vendor/github.com/goamz/goamz/s3/s3.go | 1161 ++++++++++++++++++++++++++ vendor/github.com/goamz/goamz/s3/sign.go | 141 ++++ 9 files changed, 3167 insertions(+) create mode 100644 vendor/github.com/goamz/goamz/LICENSE create mode 100644 vendor/github.com/goamz/goamz/aws/attempt.go create mode 100644 vendor/github.com/goamz/goamz/aws/aws.go create mode 100644 vendor/github.com/goamz/goamz/aws/client.go create mode 100644 vendor/github.com/goamz/goamz/aws/regions.go create mode 100644 vendor/github.com/goamz/goamz/aws/sign.go create mode 100644 vendor/github.com/goamz/goamz/s3/multi.go create mode 100644 vendor/github.com/goamz/goamz/s3/s3.go create mode 100644 vendor/github.com/goamz/goamz/s3/sign.go (limited to 'vendor/github.com/goamz') diff --git a/vendor/github.com/goamz/goamz/LICENSE b/vendor/github.com/goamz/goamz/LICENSE new file mode 100644 index 000000000..53320c352 --- /dev/null +++ b/vendor/github.com/goamz/goamz/LICENSE @@ -0,0 +1,185 @@ +This software is licensed under the LGPLv3, included below. + +As a special exception to the GNU Lesser General Public License version 3 +("LGPL3"), the copyright holders of this Library give you permission to +convey to a third party a Combined Work that links statically or dynamically +to this Library without providing any Minimal Corresponding Source or +Minimal Application Code as set out in 4d or providing the installation +information set out in section 4e, provided that you comply with the other +provisions of LGPL3 and provided that you meet, for the Application the +terms and conditions of the license(s) which apply to the Application. + +Except as stated in this special exception, the provisions of LGPL3 will +continue to comply in full to this Library. If you modify this Library, you +may apply this exception to your version of this Library, but you are not +obliged to do so. If you do not wish to do so, delete this exception +statement from your version. This exception does not (and cannot) modify any +license terms which apply to the Application, with which you must still +comply. + + + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. diff --git a/vendor/github.com/goamz/goamz/aws/attempt.go b/vendor/github.com/goamz/goamz/aws/attempt.go new file mode 100644 index 000000000..c0654f5d8 --- /dev/null +++ b/vendor/github.com/goamz/goamz/aws/attempt.go @@ -0,0 +1,74 @@ +package aws + +import ( + "time" +) + +// AttemptStrategy represents a strategy for waiting for an action +// to complete successfully. This is an internal type used by the +// implementation of other goamz packages. +type AttemptStrategy struct { + Total time.Duration // total duration of attempt. + Delay time.Duration // interval between each try in the burst. + Min int // minimum number of retries; overrides Total +} + +type Attempt struct { + strategy AttemptStrategy + last time.Time + end time.Time + force bool + count int +} + +// Start begins a new sequence of attempts for the given strategy. +func (s AttemptStrategy) Start() *Attempt { + now := time.Now() + return &Attempt{ + strategy: s, + last: now, + end: now.Add(s.Total), + force: true, + } +} + +// Next waits until it is time to perform the next attempt or returns +// false if it is time to stop trying. +func (a *Attempt) Next() bool { + now := time.Now() + sleep := a.nextSleep(now) + if !a.force && !now.Add(sleep).Before(a.end) && a.strategy.Min <= a.count { + return false + } + a.force = false + if sleep > 0 && a.count > 0 { + time.Sleep(sleep) + now = time.Now() + } + a.count++ + a.last = now + return true +} + +func (a *Attempt) nextSleep(now time.Time) time.Duration { + sleep := a.strategy.Delay - now.Sub(a.last) + if sleep < 0 { + return 0 + } + return sleep +} + +// HasNext returns whether another attempt will be made if the current +// one fails. If it returns true, the following call to Next is +// guaranteed to return true. +func (a *Attempt) HasNext() bool { + if a.force || a.strategy.Min > a.count { + return true + } + now := time.Now() + if now.Add(a.nextSleep(now)).Before(a.end) { + a.force = true + return true + } + return false +} diff --git a/vendor/github.com/goamz/goamz/aws/aws.go b/vendor/github.com/goamz/goamz/aws/aws.go new file mode 100644 index 000000000..77bf563d6 --- /dev/null +++ b/vendor/github.com/goamz/goamz/aws/aws.go @@ -0,0 +1,432 @@ +// +// goamz - Go packages to interact with the Amazon Web Services. +// +// https://wiki.ubuntu.com/goamz +// +// Copyright (c) 2011 Canonical Ltd. +// +// Written by Gustavo Niemeyer +// +package aws + +import ( + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "os" + "time" + + "github.com/vaughan0/go-ini" +) + +// Defines the valid signers +const ( + V2Signature = iota + V4Signature = iota + Route53Signature = iota +) + +// Defines the service endpoint and correct Signer implementation to use +// to sign requests for this endpoint +type ServiceInfo struct { + Endpoint string + Signer uint +} + +// Region defines the URLs where AWS services may be accessed. +// +// See http://goo.gl/d8BP1 for more details. +type Region struct { + Name string // the canonical name of this region. + EC2Endpoint string + S3Endpoint string + S3BucketEndpoint string // Not needed by AWS S3. Use ${bucket} for bucket name. + S3LocationConstraint bool // true if this region requires a LocationConstraint declaration. + S3LowercaseBucket bool // true if the region requires bucket names to be lower case. + SDBEndpoint string + SESEndpoint string + SNSEndpoint string + SQSEndpoint string + IAMEndpoint string + ELBEndpoint string + DynamoDBEndpoint string + CloudWatchServicepoint ServiceInfo + AutoScalingEndpoint string + RDSEndpoint ServiceInfo + STSEndpoint string + CloudFormationEndpoint string + ECSEndpoint string + DynamoDBStreamsEndpoint string +} + +var Regions = map[string]Region{ + APNortheast.Name: APNortheast, + APSoutheast.Name: APSoutheast, + APSoutheast2.Name: APSoutheast2, + EUCentral.Name: EUCentral, + EUWest.Name: EUWest, + USEast.Name: USEast, + USWest.Name: USWest, + USWest2.Name: USWest2, + USGovWest.Name: USGovWest, + SAEast.Name: SAEast, + CNNorth.Name: CNNorth, +} + +// Designates a signer interface suitable for signing AWS requests, params +// should be appropriately encoded for the request before signing. +// +// A signer should be initialized with Auth and the appropriate endpoint. +type Signer interface { + Sign(method, path string, params map[string]string) +} + +// An AWS Service interface with the API to query the AWS service +// +// Supplied as an easy way to mock out service calls during testing. +type AWSService interface { + // Queries the AWS service at a given method/path with the params and + // returns an http.Response and error + Query(method, path string, params map[string]string) (*http.Response, error) + // Builds an error given an XML payload in the http.Response, can be used + // to process an error if the status code is not 200 for example. + BuildError(r *http.Response) error +} + +// Implements a Server Query/Post API to easily query AWS services and build +// errors when desired +type Service struct { + service ServiceInfo + signer Signer +} + +// Create a base set of params for an action +func MakeParams(action string) map[string]string { + params := make(map[string]string) + params["Action"] = action + return params +} + +// Create a new AWS server to handle making requests +func NewService(auth Auth, service ServiceInfo) (s *Service, err error) { + var signer Signer + switch service.Signer { + case V2Signature: + signer, err = NewV2Signer(auth, service) + // case V4Signature: + // signer, err = NewV4Signer(auth, service, Regions["eu-west-1"]) + default: + err = fmt.Errorf("Unsupported signer for service") + } + if err != nil { + return + } + s = &Service{service: service, signer: signer} + return +} + +func (s *Service) Query(method, path string, params map[string]string) (resp *http.Response, err error) { + params["Timestamp"] = time.Now().UTC().Format(time.RFC3339) + u, err := url.Parse(s.service.Endpoint) + if err != nil { + return nil, err + } + u.Path = path + + s.signer.Sign(method, path, params) + if method == "GET" { + u.RawQuery = multimap(params).Encode() + resp, err = http.Get(u.String()) + } else if method == "POST" { + resp, err = http.PostForm(u.String(), multimap(params)) + } + + return +} + +func (s *Service) BuildError(r *http.Response) error { + errors := ErrorResponse{} + xml.NewDecoder(r.Body).Decode(&errors) + var err Error + err = errors.Errors + err.RequestId = errors.RequestId + err.StatusCode = r.StatusCode + if err.Message == "" { + err.Message = r.Status + } + return &err +} + +type ErrorResponse struct { + Errors Error `xml:"Error"` + RequestId string // A unique ID for tracking the request +} + +type Error struct { + StatusCode int + Type string + Code string + Message string + RequestId string +} + +func (err *Error) Error() string { + return fmt.Sprintf("Type: %s, Code: %s, Message: %s", + err.Type, err.Code, err.Message, + ) +} + +type Auth struct { + AccessKey, SecretKey string + token string + expiration time.Time +} + +func (a *Auth) Token() string { + if a.token == "" { + return "" + } + if time.Since(a.expiration) >= -30*time.Second { //in an ideal world this should be zero assuming the instance is synching it's clock + *a, _ = GetAuth("", "", "", time.Time{}) + } + return a.token +} + +func (a *Auth) Expiration() time.Time { + return a.expiration +} + +// To be used with other APIs that return auth credentials such as STS +func NewAuth(accessKey, secretKey, token string, expiration time.Time) *Auth { + return &Auth{ + AccessKey: accessKey, + SecretKey: secretKey, + token: token, + expiration: expiration, + } +} + +// ResponseMetadata +type ResponseMetadata struct { + RequestId string // A unique ID for tracking the request +} + +type BaseResponse struct { + ResponseMetadata ResponseMetadata +} + +var unreserved = make([]bool, 128) +var hex = "0123456789ABCDEF" + +func init() { + // RFC3986 + u := "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz01234567890-_.~" + for _, c := range u { + unreserved[c] = true + } +} + +func multimap(p map[string]string) url.Values { + q := make(url.Values, len(p)) + for k, v := range p { + q[k] = []string{v} + } + return q +} + +type credentials struct { + Code string + LastUpdated string + Type string + AccessKeyId string + SecretAccessKey string + Token string + Expiration string +} + +// GetMetaData retrieves instance metadata about the current machine. +// +// See http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html for more details. +func GetMetaData(path string) (contents []byte, err error) { + url := "http://169.254.169.254/latest/meta-data/" + path + + resp, err := RetryingClient.Get(url) + if err != nil { + return + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + err = fmt.Errorf("Code %d returned for url %s", resp.StatusCode, url) + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return + } + return []byte(body), err +} + +func getInstanceCredentials() (cred credentials, err error) { + credentialPath := "iam/security-credentials/" + + // Get the instance role + role, err := GetMetaData(credentialPath) + if err != nil { + return + } + + // Get the instance role credentials + credentialJSON, err := GetMetaData(credentialPath + string(role)) + if err != nil { + return + } + + err = json.Unmarshal([]byte(credentialJSON), &cred) + return +} + +// GetAuth creates an Auth based on either passed in credentials, +// environment information or instance based role credentials. +func GetAuth(accessKey string, secretKey, token string, expiration time.Time) (auth Auth, err error) { + // First try passed in credentials + if accessKey != "" && secretKey != "" { + return Auth{accessKey, secretKey, token, expiration}, nil + } + + // Next try to get auth from the shared credentials file + auth, err = SharedAuth() + if err == nil { + // Found auth, return + return + } + + // Next try to get auth from the environment + auth, err = EnvAuth() + if err == nil { + // Found auth, return + return + } + + // Next try getting auth from the instance role + cred, err := getInstanceCredentials() + if err == nil { + // Found auth, return + auth.AccessKey = cred.AccessKeyId + auth.SecretKey = cred.SecretAccessKey + auth.token = cred.Token + exptdate, err := time.Parse("2006-01-02T15:04:05Z", cred.Expiration) + if err != nil { + err = fmt.Errorf("Error Parseing expiration date: cred.Expiration :%s , error: %s \n", cred.Expiration, err) + } + auth.expiration = exptdate + return auth, err + } + err = errors.New("No valid AWS authentication found") + return auth, err +} + +// EnvAuth creates an Auth based on environment information. +// The AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY environment +// variables are used. +// AWS_SESSION_TOKEN is used if present. +func EnvAuth() (auth Auth, err error) { + auth.AccessKey = os.Getenv("AWS_ACCESS_KEY_ID") + if auth.AccessKey == "" { + auth.AccessKey = os.Getenv("AWS_ACCESS_KEY") + } + + auth.SecretKey = os.Getenv("AWS_SECRET_ACCESS_KEY") + if auth.SecretKey == "" { + auth.SecretKey = os.Getenv("AWS_SECRET_KEY") + } + if auth.AccessKey == "" { + err = errors.New("AWS_ACCESS_KEY_ID or AWS_ACCESS_KEY not found in environment") + } + if auth.SecretKey == "" { + err = errors.New("AWS_SECRET_ACCESS_KEY or AWS_SECRET_KEY not found in environment") + } + + auth.token = os.Getenv("AWS_SESSION_TOKEN") + return +} + +// SharedAuth creates an Auth based on shared credentials stored in +// $HOME/.aws/credentials. The AWS_PROFILE environment variables is used to +// select the profile. +func SharedAuth() (auth Auth, err error) { + var profileName = os.Getenv("AWS_PROFILE") + + if profileName == "" { + profileName = "default" + } + + var credentialsFile = os.Getenv("AWS_CREDENTIAL_FILE") + if credentialsFile == "" { + var homeDir = os.Getenv("HOME") + if homeDir == "" { + err = errors.New("Could not get HOME") + return + } + credentialsFile = homeDir + "/.aws/credentials" + } + + file, err := ini.LoadFile(credentialsFile) + if err != nil { + err = errors.New("Couldn't parse AWS credentials file") + return + } + + var profile = file[profileName] + if profile == nil { + err = errors.New("Couldn't find profile in AWS credentials file") + return + } + + auth.AccessKey = profile["aws_access_key_id"] + auth.SecretKey = profile["aws_secret_access_key"] + + if auth.AccessKey == "" { + err = errors.New("AWS_ACCESS_KEY_ID not found in environment in credentials file") + } + if auth.SecretKey == "" { + err = errors.New("AWS_SECRET_ACCESS_KEY not found in credentials file") + } + return +} + +// Encode takes a string and URI-encodes it in a way suitable +// to be used in AWS signatures. +func Encode(s string) string { + encode := false + for i := 0; i != len(s); i++ { + c := s[i] + if c > 127 || !unreserved[c] { + encode = true + break + } + } + if !encode { + return s + } + e := make([]byte, len(s)*3) + ei := 0 + for i := 0; i != len(s); i++ { + c := s[i] + if c > 127 || !unreserved[c] { + e[ei] = '%' + e[ei+1] = hex[c>>4] + e[ei+2] = hex[c&0xF] + ei += 3 + } else { + e[ei] = c + ei += 1 + } + } + return string(e[:ei]) +} diff --git a/vendor/github.com/goamz/goamz/aws/client.go b/vendor/github.com/goamz/goamz/aws/client.go new file mode 100644 index 000000000..86d2ccec8 --- /dev/null +++ b/vendor/github.com/goamz/goamz/aws/client.go @@ -0,0 +1,124 @@ +package aws + +import ( + "math" + "net" + "net/http" + "time" +) + +type RetryableFunc func(*http.Request, *http.Response, error) bool +type WaitFunc func(try int) +type DeadlineFunc func() time.Time + +type ResilientTransport struct { + // Timeout is the maximum amount of time a dial will wait for + // a connect to complete. + // + // The default is no timeout. + // + // With or without a timeout, the operating system may impose + // its own earlier timeout. For instance, TCP timeouts are + // often around 3 minutes. + DialTimeout time.Duration + + // MaxTries, if non-zero, specifies the number of times we will retry on + // failure. Retries are only attempted for temporary network errors or known + // safe failures. + MaxTries int + Deadline DeadlineFunc + ShouldRetry RetryableFunc + Wait WaitFunc + transport *http.Transport +} + +// Convenience method for creating an http client +func NewClient(rt *ResilientTransport) *http.Client { + rt.transport = &http.Transport{ + Dial: func(netw, addr string) (net.Conn, error) { + c, err := net.DialTimeout(netw, addr, rt.DialTimeout) + if err != nil { + return nil, err + } + c.SetDeadline(rt.Deadline()) + return c, nil + }, + Proxy: http.ProxyFromEnvironment, + } + // TODO: Would be nice is ResilientTransport allowed clients to initialize + // with http.Transport attributes. + return &http.Client{ + Transport: rt, + } +} + +var retryingTransport = &ResilientTransport{ + Deadline: func() time.Time { + return time.Now().Add(5 * time.Second) + }, + DialTimeout: 10 * time.Second, + MaxTries: 3, + ShouldRetry: awsRetry, + Wait: ExpBackoff, +} + +// Exported default client +var RetryingClient = NewClient(retryingTransport) + +func (t *ResilientTransport) RoundTrip(req *http.Request) (*http.Response, error) { + return t.tries(req) +} + +// Retry a request a maximum of t.MaxTries times. +// We'll only retry if the proper criteria are met. +// If a wait function is specified, wait that amount of time +// In between requests. +func (t *ResilientTransport) tries(req *http.Request) (res *http.Response, err error) { + for try := 0; try < t.MaxTries; try += 1 { + res, err = t.transport.RoundTrip(req) + + if !t.ShouldRetry(req, res, err) { + break + } + if res != nil { + res.Body.Close() + } + if t.Wait != nil { + t.Wait(try) + } + } + + return +} + +func ExpBackoff(try int) { + time.Sleep(100 * time.Millisecond * + time.Duration(math.Exp2(float64(try)))) +} + +func LinearBackoff(try int) { + time.Sleep(time.Duration(try*100) * time.Millisecond) +} + +// Decide if we should retry a request. +// In general, the criteria for retrying a request is described here +// http://docs.aws.amazon.com/general/latest/gr/api-retries.html +func awsRetry(req *http.Request, res *http.Response, err error) bool { + retry := false + + // Retry if there's a temporary network error. + if neterr, ok := err.(net.Error); ok { + if neterr.Temporary() { + retry = true + } + } + + // Retry if we get a 5xx series error. + if res != nil { + if res.StatusCode >= 500 && res.StatusCode < 600 { + retry = true + } + } + + return retry +} diff --git a/vendor/github.com/goamz/goamz/aws/regions.go b/vendor/github.com/goamz/goamz/aws/regions.go new file mode 100644 index 000000000..5e18f023d --- /dev/null +++ b/vendor/github.com/goamz/goamz/aws/regions.go @@ -0,0 +1,254 @@ +package aws + +var USGovWest = Region{ + "us-gov-west-1", + "https://ec2.us-gov-west-1.amazonaws.com", + "https://s3-fips-us-gov-west-1.amazonaws.com", + "", + true, + true, + "", + "", + "https://sns.us-gov-west-1.amazonaws.com", + "https://sqs.us-gov-west-1.amazonaws.com", + "https://iam.us-gov.amazonaws.com", + "https://elasticloadbalancing.us-gov-west-1.amazonaws.com", + "https://dynamodb.us-gov-west-1.amazonaws.com", + ServiceInfo{"https://monitoring.us-gov-west-1.amazonaws.com", V2Signature}, + "https://autoscaling.us-gov-west-1.amazonaws.com", + ServiceInfo{"https://rds.us-gov-west-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.us-gov-west-1.amazonaws.com", + "https://ecs.us-gov-west-1.amazonaws.com", + "https://streams.dynamodb.us-gov-west-1.amazonaws.com", +} + +var USEast = Region{ + "us-east-1", + "https://ec2.us-east-1.amazonaws.com", + "https://s3.amazonaws.com", + "", + false, + false, + "https://sdb.amazonaws.com", + "https://email.us-east-1.amazonaws.com", + "https://sns.us-east-1.amazonaws.com", + "https://sqs.us-east-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.us-east-1.amazonaws.com", + "https://dynamodb.us-east-1.amazonaws.com", + ServiceInfo{"https://monitoring.us-east-1.amazonaws.com", V2Signature}, + "https://autoscaling.us-east-1.amazonaws.com", + ServiceInfo{"https://rds.us-east-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.us-east-1.amazonaws.com", + "https://ecs.us-east-1.amazonaws.com", + "https://streams.dynamodb.us-east-1.amazonaws.com", +} + +var USWest = Region{ + "us-west-1", + "https://ec2.us-west-1.amazonaws.com", + "https://s3-us-west-1.amazonaws.com", + "", + true, + true, + "https://sdb.us-west-1.amazonaws.com", + "", + "https://sns.us-west-1.amazonaws.com", + "https://sqs.us-west-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.us-west-1.amazonaws.com", + "https://dynamodb.us-west-1.amazonaws.com", + ServiceInfo{"https://monitoring.us-west-1.amazonaws.com", V2Signature}, + "https://autoscaling.us-west-1.amazonaws.com", + ServiceInfo{"https://rds.us-west-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.us-west-1.amazonaws.com", + "https://ecs.us-west-1.amazonaws.com", + "https://streams.dynamodb.us-west-1.amazonaws.com", +} + +var USWest2 = Region{ + "us-west-2", + "https://ec2.us-west-2.amazonaws.com", + "https://s3-us-west-2.amazonaws.com", + "", + true, + true, + "https://sdb.us-west-2.amazonaws.com", + "https://email.us-west-2.amazonaws.com", + "https://sns.us-west-2.amazonaws.com", + "https://sqs.us-west-2.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.us-west-2.amazonaws.com", + "https://dynamodb.us-west-2.amazonaws.com", + ServiceInfo{"https://monitoring.us-west-2.amazonaws.com", V2Signature}, + "https://autoscaling.us-west-2.amazonaws.com", + ServiceInfo{"https://rds.us-west-2.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.us-west-2.amazonaws.com", + "https://ecs.us-west-2.amazonaws.com", + "https://streams.dynamodb.us-west-2.amazonaws.com", +} + +var EUWest = Region{ + "eu-west-1", + "https://ec2.eu-west-1.amazonaws.com", + "https://s3-eu-west-1.amazonaws.com", + "", + true, + true, + "https://sdb.eu-west-1.amazonaws.com", + "https://email.eu-west-1.amazonaws.com", + "https://sns.eu-west-1.amazonaws.com", + "https://sqs.eu-west-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.eu-west-1.amazonaws.com", + "https://dynamodb.eu-west-1.amazonaws.com", + ServiceInfo{"https://monitoring.eu-west-1.amazonaws.com", V2Signature}, + "https://autoscaling.eu-west-1.amazonaws.com", + ServiceInfo{"https://rds.eu-west-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.eu-west-1.amazonaws.com", + "https://ecs.eu-west-1.amazonaws.com", + "https://streams.dynamodb.eu-west-1.amazonaws.com", +} + +var EUCentral = Region{ + "eu-central-1", + "https://ec2.eu-central-1.amazonaws.com", + "https://s3-eu-central-1.amazonaws.com", + "", + true, + true, + "https://sdb.eu-central-1.amazonaws.com", + "https://email.eu-central-1.amazonaws.com", + "https://sns.eu-central-1.amazonaws.com", + "https://sqs.eu-central-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.eu-central-1.amazonaws.com", + "https://dynamodb.eu-central-1.amazonaws.com", + ServiceInfo{"https://monitoring.eu-central-1.amazonaws.com", V2Signature}, + "https://autoscaling.eu-central-1.amazonaws.com", + ServiceInfo{"https://rds.eu-central-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.eu-central-1.amazonaws.com", + "https://ecs.eu-central-1.amazonaws.com", + "https://streams.dynamodb.eu-central-1.amazonaws.com", +} + +var APSoutheast = Region{ + "ap-southeast-1", + "https://ec2.ap-southeast-1.amazonaws.com", + "https://s3-ap-southeast-1.amazonaws.com", + "", + true, + true, + "https://sdb.ap-southeast-1.amazonaws.com", + "", + "https://sns.ap-southeast-1.amazonaws.com", + "https://sqs.ap-southeast-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.ap-southeast-1.amazonaws.com", + "https://dynamodb.ap-southeast-1.amazonaws.com", + ServiceInfo{"https://monitoring.ap-southeast-1.amazonaws.com", V2Signature}, + "https://autoscaling.ap-southeast-1.amazonaws.com", + ServiceInfo{"https://rds.ap-southeast-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.ap-southeast-1.amazonaws.com", + "https://ecs.ap-southeast-1.amazonaws.com", + "https://streams.dynamodb.ap-southeast-1.amazonaws.com", +} + +var APSoutheast2 = Region{ + "ap-southeast-2", + "https://ec2.ap-southeast-2.amazonaws.com", + "https://s3-ap-southeast-2.amazonaws.com", + "", + true, + true, + "https://sdb.ap-southeast-2.amazonaws.com", + "", + "https://sns.ap-southeast-2.amazonaws.com", + "https://sqs.ap-southeast-2.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.ap-southeast-2.amazonaws.com", + "https://dynamodb.ap-southeast-2.amazonaws.com", + ServiceInfo{"https://monitoring.ap-southeast-2.amazonaws.com", V2Signature}, + "https://autoscaling.ap-southeast-2.amazonaws.com", + ServiceInfo{"https://rds.ap-southeast-2.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.ap-southeast-2.amazonaws.com", + "https://ecs.ap-southeast-2.amazonaws.com", + "https://streams.dynamodb.ap-southeast-2.amazonaws.com", +} + +var APNortheast = Region{ + "ap-northeast-1", + "https://ec2.ap-northeast-1.amazonaws.com", + "https://s3-ap-northeast-1.amazonaws.com", + "", + true, + true, + "https://sdb.ap-northeast-1.amazonaws.com", + "", + "https://sns.ap-northeast-1.amazonaws.com", + "https://sqs.ap-northeast-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.ap-northeast-1.amazonaws.com", + "https://dynamodb.ap-northeast-1.amazonaws.com", + ServiceInfo{"https://monitoring.ap-northeast-1.amazonaws.com", V2Signature}, + "https://autoscaling.ap-northeast-1.amazonaws.com", + ServiceInfo{"https://rds.ap-northeast-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.ap-northeast-1.amazonaws.com", + "https://ecs.ap-northeast-1.amazonaws.com", + "https://streams.dynamodb.ap-northeast-1.amazonaws.com", +} + +var SAEast = Region{ + "sa-east-1", + "https://ec2.sa-east-1.amazonaws.com", + "https://s3-sa-east-1.amazonaws.com", + "", + true, + true, + "https://sdb.sa-east-1.amazonaws.com", + "", + "https://sns.sa-east-1.amazonaws.com", + "https://sqs.sa-east-1.amazonaws.com", + "https://iam.amazonaws.com", + "https://elasticloadbalancing.sa-east-1.amazonaws.com", + "https://dynamodb.sa-east-1.amazonaws.com", + ServiceInfo{"https://monitoring.sa-east-1.amazonaws.com", V2Signature}, + "https://autoscaling.sa-east-1.amazonaws.com", + ServiceInfo{"https://rds.sa-east-1.amazonaws.com", V2Signature}, + "https://sts.amazonaws.com", + "https://cloudformation.sa-east-1.amazonaws.com", + "https://ecs.sa-east-1.amazonaws.com", + "https://streams.dynamodb.sa-east-1.amazonaws.com", +} + +var CNNorth = Region{ + "cn-north-1", + "https://ec2.cn-north-1.amazonaws.com.cn", + "https://s3.cn-north-1.amazonaws.com.cn", + "", + true, + true, + "https://sdb.cn-north-1.amazonaws.com.cn", + "", + "https://sns.cn-north-1.amazonaws.com.cn", + "https://sqs.cn-north-1.amazonaws.com.cn", + "https://iam.cn-north-1.amazonaws.com.cn", + "https://elasticloadbalancing.cn-north-1.amazonaws.com.cn", + "https://dynamodb.cn-north-1.amazonaws.com.cn", + ServiceInfo{"https://monitoring.cn-north-1.amazonaws.com.cn", V4Signature}, + "https://autoscaling.cn-north-1.amazonaws.com.cn", + ServiceInfo{"https://rds.cn-north-1.amazonaws.com.cn", V4Signature}, + "https://sts.cn-north-1.amazonaws.com.cn", + "https://cloudformation.cn-north-1.amazonaws.com.cn", + "https://ecs.cn-north-1.amazonaws.com.cn", + "https://streams.dynamodb.cn-north-1.amazonaws.com.cn", +} diff --git a/vendor/github.com/goamz/goamz/aws/sign.go b/vendor/github.com/goamz/goamz/aws/sign.go new file mode 100644 index 000000000..22ce0781b --- /dev/null +++ b/vendor/github.com/goamz/goamz/aws/sign.go @@ -0,0 +1,357 @@ +package aws + +import ( + "bytes" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "fmt" + "io/ioutil" + "net/http" + "net/url" + "path" + "sort" + "strings" + "time" +) + +type V2Signer struct { + auth Auth + service ServiceInfo + host string +} + +var b64 = base64.StdEncoding + +func NewV2Signer(auth Auth, service ServiceInfo) (*V2Signer, error) { + u, err := url.Parse(service.Endpoint) + if err != nil { + return nil, err + } + return &V2Signer{auth: auth, service: service, host: u.Host}, nil +} + +func (s *V2Signer) Sign(method, path string, params map[string]string) { + params["AWSAccessKeyId"] = s.auth.AccessKey + params["SignatureVersion"] = "2" + params["SignatureMethod"] = "HmacSHA256" + if s.auth.Token() != "" { + params["SecurityToken"] = s.auth.Token() + } + + // AWS specifies that the parameters in a signed request must + // be provided in the natural order of the keys. This is distinct + // from the natural order of the encoded value of key=value. + // Percent and Equals affect the sorting order. + var keys, sarray []string + for k, _ := range params { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + sarray = append(sarray, Encode(k)+"="+Encode(params[k])) + } + joined := strings.Join(sarray, "&") + payload := method + "\n" + s.host + "\n" + path + "\n" + joined + hash := hmac.New(sha256.New, []byte(s.auth.SecretKey)) + hash.Write([]byte(payload)) + signature := make([]byte, b64.EncodedLen(hash.Size())) + b64.Encode(signature, hash.Sum(nil)) + + params["Signature"] = string(signature) +} + +// Common date formats for signing requests +const ( + ISO8601BasicFormat = "20060102T150405Z" + ISO8601BasicFormatShort = "20060102" +) + +type Route53Signer struct { + auth Auth +} + +func NewRoute53Signer(auth Auth) *Route53Signer { + return &Route53Signer{auth: auth} +} + +// getCurrentDate fetches the date stamp from the aws servers to +// ensure the auth headers are within 5 minutes of the server time +func (s *Route53Signer) getCurrentDate() string { + response, err := http.Get("https://route53.amazonaws.com/date") + if err != nil { + fmt.Print("Unable to get date from amazon: ", err) + return "" + } + + response.Body.Close() + return response.Header.Get("Date") +} + +// Creates the authorize signature based on the date stamp and secret key +func (s *Route53Signer) getHeaderAuthorize(message string) string { + hmacSha256 := hmac.New(sha256.New, []byte(s.auth.SecretKey)) + hmacSha256.Write([]byte(message)) + cryptedString := hmacSha256.Sum(nil) + + return base64.StdEncoding.EncodeToString(cryptedString) +} + +// Adds all the required headers for AWS Route53 API to the request +// including the authorization +func (s *Route53Signer) Sign(req *http.Request) { + date := s.getCurrentDate() + authHeader := fmt.Sprintf("AWS3-HTTPS AWSAccessKeyId=%s,Algorithm=%s,Signature=%s", + s.auth.AccessKey, "HmacSHA256", s.getHeaderAuthorize(date)) + + req.Header.Set("Host", req.Host) + req.Header.Set("X-Amzn-Authorization", authHeader) + req.Header.Set("X-Amz-Date", date) + req.Header.Set("Content-Type", "application/xml") +} + +/* +The V4Signer encapsulates all of the functionality to sign a request with the AWS +Signature Version 4 Signing Process. (http://goo.gl/u1OWZz) +*/ +type V4Signer struct { + auth Auth + serviceName string + region Region +} + +/* +Return a new instance of a V4Signer capable of signing AWS requests. +*/ +func NewV4Signer(auth Auth, serviceName string, region Region) *V4Signer { + return &V4Signer{auth: auth, serviceName: serviceName, region: region} +} + +/* +Sign a request according to the AWS Signature Version 4 Signing Process. (http://goo.gl/u1OWZz) + +The signed request will include an "x-amz-date" header with a current timestamp if a valid "x-amz-date" +or "date" header was not available in the original request. In addition, AWS Signature Version 4 requires +the "host" header to be a signed header, therefor the Sign method will manually set a "host" header from +the request.Host. + +The signed request will include a new "Authorization" header indicating that the request has been signed. + +Any changes to the request after signing the request will invalidate the signature. +*/ +func (s *V4Signer) Sign(req *http.Request) { + req.Header.Set("host", req.Host) // host header must be included as a signed header + t := s.requestTime(req) // Get requst time + creq := s.canonicalRequest(req) // Build canonical request + sts := s.stringToSign(t, creq) // Build string to sign + signature := s.signature(t, sts) // Calculate the AWS Signature Version 4 + auth := s.authorization(req.Header, t, signature) // Create Authorization header value + req.Header.Set("Authorization", auth) // Add Authorization header to request + return +} + +/* +requestTime method will parse the time from the request "x-amz-date" or "date" headers. +If the "x-amz-date" header is present, that will take priority over the "date" header. +If neither header is defined or we are unable to parse either header as a valid date +then we will create a new "x-amz-date" header with the current time. +*/ +func (s *V4Signer) requestTime(req *http.Request) time.Time { + + // Get "x-amz-date" header + date := req.Header.Get("x-amz-date") + + // Attempt to parse as ISO8601BasicFormat + t, err := time.Parse(ISO8601BasicFormat, date) + if err == nil { + return t + } + + // Attempt to parse as http.TimeFormat + t, err = time.Parse(http.TimeFormat, date) + if err == nil { + req.Header.Set("x-amz-date", t.Format(ISO8601BasicFormat)) + return t + } + + // Get "date" header + date = req.Header.Get("date") + + // Attempt to parse as http.TimeFormat + t, err = time.Parse(http.TimeFormat, date) + if err == nil { + return t + } + + // Create a current time header to be used + t = time.Now().UTC() + req.Header.Set("x-amz-date", t.Format(ISO8601BasicFormat)) + return t +} + +/* +canonicalRequest method creates the canonical request according to Task 1 of the AWS Signature Version 4 Signing Process. (http://goo.gl/eUUZ3S) + + CanonicalRequest = + HTTPRequestMethod + '\n' + + CanonicalURI + '\n' + + CanonicalQueryString + '\n' + + CanonicalHeaders + '\n' + + SignedHeaders + '\n' + + HexEncode(Hash(Payload)) +*/ +func (s *V4Signer) canonicalRequest(req *http.Request) string { + c := new(bytes.Buffer) + fmt.Fprintf(c, "%s\n", req.Method) + fmt.Fprintf(c, "%s\n", s.canonicalURI(req.URL)) + fmt.Fprintf(c, "%s\n", s.canonicalQueryString(req.URL)) + fmt.Fprintf(c, "%s\n\n", s.canonicalHeaders(req.Header)) + fmt.Fprintf(c, "%s\n", s.signedHeaders(req.Header)) + fmt.Fprintf(c, "%s", s.payloadHash(req)) + return c.String() +} + +func (s *V4Signer) canonicalURI(u *url.URL) string { + canonicalPath := u.RequestURI() + if u.RawQuery != "" { + canonicalPath = canonicalPath[:len(canonicalPath)-len(u.RawQuery)-1] + } + slash := strings.HasSuffix(canonicalPath, "/") + canonicalPath = path.Clean(canonicalPath) + if canonicalPath != "/" && slash { + canonicalPath += "/" + } + return canonicalPath +} + +func (s *V4Signer) canonicalQueryString(u *url.URL) string { + var a []string + for k, vs := range u.Query() { + k = Encode(k) + for _, v := range vs { + if v == "" { + a = append(a, k+"=") + } else { + v = Encode(v) + a = append(a, k+"="+v) + } + } + } + sort.Strings(a) + return strings.Join(a, "&") +} + +func (s *V4Signer) canonicalHeaders(h http.Header) string { + i, a := 0, make([]string, len(h)) + for k, v := range h { + for j, w := range v { + v[j] = strings.Trim(w, " ") + } + sort.Strings(v) + a[i] = strings.ToLower(k) + ":" + strings.Join(v, ",") + i++ + } + sort.Strings(a) + return strings.Join(a, "\n") +} + +func (s *V4Signer) signedHeaders(h http.Header) string { + i, a := 0, make([]string, len(h)) + for k, _ := range h { + a[i] = strings.ToLower(k) + i++ + } + sort.Strings(a) + return strings.Join(a, ";") +} + +func (s *V4Signer) payloadHash(req *http.Request) string { + var b []byte + if req.Body == nil { + b = []byte("") + } else { + var err error + b, err = ioutil.ReadAll(req.Body) + if err != nil { + // TODO: I REALLY DON'T LIKE THIS PANIC!!!! + panic(err) + } + } + req.Body = ioutil.NopCloser(bytes.NewBuffer(b)) + return s.hash(string(b)) +} + +/* +stringToSign method creates the string to sign accorting to Task 2 of the AWS Signature Version 4 Signing Process. (http://goo.gl/es1PAu) + + StringToSign = + Algorithm + '\n' + + RequestDate + '\n' + + CredentialScope + '\n' + + HexEncode(Hash(CanonicalRequest)) +*/ +func (s *V4Signer) stringToSign(t time.Time, creq string) string { + w := new(bytes.Buffer) + fmt.Fprint(w, "AWS4-HMAC-SHA256\n") + fmt.Fprintf(w, "%s\n", t.Format(ISO8601BasicFormat)) + fmt.Fprintf(w, "%s\n", s.credentialScope(t)) + fmt.Fprintf(w, "%s", s.hash(creq)) + return w.String() +} + +func (s *V4Signer) credentialScope(t time.Time) string { + return fmt.Sprintf("%s/%s/%s/aws4_request", t.Format(ISO8601BasicFormatShort), s.region.Name, s.serviceName) +} + +/* +signature method calculates the AWS Signature Version 4 according to Task 3 of the AWS Signature Version 4 Signing Process. (http://goo.gl/j0Yqe1) + + signature = HexEncode(HMAC(derived-signing-key, string-to-sign)) +*/ +func (s *V4Signer) signature(t time.Time, sts string) string { + h := s.hmac(s.derivedKey(t), []byte(sts)) + return fmt.Sprintf("%x", h) +} + +/* +derivedKey method derives a signing key to be used for signing a request. + + kSecret = Your AWS Secret Access Key + kDate = HMAC("AWS4" + kSecret, Date) + kRegion = HMAC(kDate, Region) + kService = HMAC(kRegion, Service) + kSigning = HMAC(kService, "aws4_request") +*/ +func (s *V4Signer) derivedKey(t time.Time) []byte { + h := s.hmac([]byte("AWS4"+s.auth.SecretKey), []byte(t.Format(ISO8601BasicFormatShort))) + h = s.hmac(h, []byte(s.region.Name)) + h = s.hmac(h, []byte(s.serviceName)) + h = s.hmac(h, []byte("aws4_request")) + return h +} + +/* +authorization method generates the authorization header value. +*/ +func (s *V4Signer) authorization(header http.Header, t time.Time, signature string) string { + w := new(bytes.Buffer) + fmt.Fprint(w, "AWS4-HMAC-SHA256 ") + fmt.Fprintf(w, "Credential=%s/%s, ", s.auth.AccessKey, s.credentialScope(t)) + fmt.Fprintf(w, "SignedHeaders=%s, ", s.signedHeaders(header)) + fmt.Fprintf(w, "Signature=%s", signature) + return w.String() +} + +// hash method calculates the sha256 hash for a given string +func (s *V4Signer) hash(in string) string { + h := sha256.New() + fmt.Fprintf(h, "%s", in) + return fmt.Sprintf("%x", h.Sum(nil)) +} + +// hmac method calculates the sha256 hmac for a given slice of bytes +func (s *V4Signer) hmac(key, data []byte) []byte { + h := hmac.New(sha256.New, key) + h.Write(data) + return h.Sum(nil) +} diff --git a/vendor/github.com/goamz/goamz/s3/multi.go b/vendor/github.com/goamz/goamz/s3/multi.go new file mode 100644 index 000000000..348ead300 --- /dev/null +++ b/vendor/github.com/goamz/goamz/s3/multi.go @@ -0,0 +1,439 @@ +package s3 + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "errors" + "io" + "sort" + "strconv" +) + +// Multi represents an unfinished multipart upload. +// +// Multipart uploads allow sending big objects in smaller chunks. +// After all parts have been sent, the upload must be explicitly +// completed by calling Complete with the list of parts. +// +// See http://goo.gl/vJfTG for an overview of multipart uploads. +type Multi struct { + Bucket *Bucket + Key string + UploadId string +} + +// That's the default. Here just for testing. +var listMultiMax = 1000 + +type listMultiResp struct { + NextKeyMarker string + NextUploadIdMarker string + IsTruncated bool + Upload []Multi + CommonPrefixes []string `xml:"CommonPrefixes>Prefix"` +} + +// ListMulti returns the list of unfinished multipart uploads in b. +// +// The prefix parameter limits the response to keys that begin with the +// specified prefix. You can use prefixes to separate a bucket into different +// groupings of keys (to get the feeling of folders, for example). +// +// The delim parameter causes the response to group all of the keys that +// share a common prefix up to the next delimiter in a single entry within +// the CommonPrefixes field. You can use delimiters to separate a bucket +// into different groupings of keys, similar to how folders would work. +// +// See http://goo.gl/ePioY for details. +func (b *Bucket) ListMulti(prefix, delim string) (multis []*Multi, prefixes []string, err error) { + params := map[string][]string{ + "uploads": {""}, + "max-uploads": {strconv.FormatInt(int64(listMultiMax), 10)}, + "prefix": {prefix}, + "delimiter": {delim}, + } + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + req := &request{ + method: "GET", + bucket: b.Name, + params: params, + } + var resp listMultiResp + err := b.S3.query(req, &resp) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, nil, err + } + for i := range resp.Upload { + multi := &resp.Upload[i] + multi.Bucket = b + multis = append(multis, multi) + } + prefixes = append(prefixes, resp.CommonPrefixes...) + if !resp.IsTruncated { + return multis, prefixes, nil + } + params["key-marker"] = []string{resp.NextKeyMarker} + params["upload-id-marker"] = []string{resp.NextUploadIdMarker} + attempt = b.S3.AttemptStrategy.Start() // Last request worked. + } + panic("unreachable") +} + +// Multi returns a multipart upload handler for the provided key +// inside b. If a multipart upload exists for key, it is returned, +// otherwise a new multipart upload is initiated with contType and perm. +func (b *Bucket) Multi(key, contType string, perm ACL) (*Multi, error) { + multis, _, err := b.ListMulti(key, "") + if err != nil && !hasCode(err, "NoSuchUpload") { + return nil, err + } + for _, m := range multis { + if m.Key == key { + return m, nil + } + } + return b.InitMulti(key, contType, perm) +} + +// InitMulti initializes a new multipart upload at the provided +// key inside b and returns a value for manipulating it. +// +// See http://goo.gl/XP8kL for details. +func (b *Bucket) InitMulti(key string, contType string, perm ACL) (*Multi, error) { + headers := map[string][]string{ + "Content-Type": {contType}, + "Content-Length": {"0"}, + "x-amz-acl": {string(perm)}, + } + params := map[string][]string{ + "uploads": {""}, + } + req := &request{ + method: "POST", + bucket: b.Name, + path: key, + headers: headers, + params: params, + } + var err error + var resp struct { + UploadId string `xml:"UploadId"` + } + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + err = b.S3.query(req, &resp) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + return &Multi{Bucket: b, Key: key, UploadId: resp.UploadId}, nil +} + +// PutPart sends part n of the multipart upload, reading all the content from r. +// Each part, except for the last one, must be at least 5MB in size. +// +// See http://goo.gl/pqZer for details. +func (m *Multi) PutPart(n int, r io.ReadSeeker) (Part, error) { + partSize, _, md5b64, err := seekerInfo(r) + if err != nil { + return Part{}, err + } + return m.putPart(n, r, partSize, md5b64) +} + +func (m *Multi) putPart(n int, r io.ReadSeeker, partSize int64, md5b64 string) (Part, error) { + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(partSize, 10)}, + "Content-MD5": {md5b64}, + } + params := map[string][]string{ + "uploadId": {m.UploadId}, + "partNumber": {strconv.FormatInt(int64(n), 10)}, + } + for attempt := m.Bucket.S3.AttemptStrategy.Start(); attempt.Next(); { + _, err := r.Seek(0, 0) + if err != nil { + return Part{}, err + } + req := &request{ + method: "PUT", + bucket: m.Bucket.Name, + path: m.Key, + headers: headers, + params: params, + payload: r, + } + err = m.Bucket.S3.prepare(req) + if err != nil { + return Part{}, err + } + resp, err := m.Bucket.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return Part{}, err + } + etag := resp.Header.Get("ETag") + if etag == "" { + return Part{}, errors.New("part upload succeeded with no ETag") + } + return Part{n, etag, partSize}, nil + } + panic("unreachable") +} + +func seekerInfo(r io.ReadSeeker) (size int64, md5hex string, md5b64 string, err error) { + _, err = r.Seek(0, 0) + if err != nil { + return 0, "", "", err + } + digest := md5.New() + size, err = io.Copy(digest, r) + if err != nil { + return 0, "", "", err + } + sum := digest.Sum(nil) + md5hex = hex.EncodeToString(sum) + md5b64 = base64.StdEncoding.EncodeToString(sum) + return size, md5hex, md5b64, nil +} + +type Part struct { + N int `xml:"PartNumber"` + ETag string + Size int64 +} + +type partSlice []Part + +func (s partSlice) Len() int { return len(s) } +func (s partSlice) Less(i, j int) bool { return s[i].N < s[j].N } +func (s partSlice) Swap(i, j int) { s[i], s[j] = s[j], s[i] } + +type listPartsResp struct { + NextPartNumberMarker string + IsTruncated bool + Part []Part +} + +// That's the default. Here just for testing. +var listPartsMax = 1000 + +// ListParts returns the list of previously uploaded parts in m, +// ordered by part number. +// +// See http://goo.gl/ePioY for details. +func (m *Multi) ListParts() ([]Part, error) { + params := map[string][]string{ + "uploadId": {m.UploadId}, + "max-parts": {strconv.FormatInt(int64(listPartsMax), 10)}, + } + var parts partSlice + for attempt := m.Bucket.S3.AttemptStrategy.Start(); attempt.Next(); { + req := &request{ + method: "GET", + bucket: m.Bucket.Name, + path: m.Key, + params: params, + } + var resp listPartsResp + err := m.Bucket.S3.query(req, &resp) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + parts = append(parts, resp.Part...) + if !resp.IsTruncated { + sort.Sort(parts) + return parts, nil + } + params["part-number-marker"] = []string{resp.NextPartNumberMarker} + attempt = m.Bucket.S3.AttemptStrategy.Start() // Last request worked. + } + panic("unreachable") +} + +type ReaderAtSeeker interface { + io.ReaderAt + io.ReadSeeker +} + +// PutAll sends all of r via a multipart upload with parts no larger +// than partSize bytes, which must be set to at least 5MB. +// Parts previously uploaded are either reused if their checksum +// and size match the new part, or otherwise overwritten with the +// new content. +// PutAll returns all the parts of m (reused or not). +func (m *Multi) PutAll(r ReaderAtSeeker, partSize int64) ([]Part, error) { + old, err := m.ListParts() + if err != nil && !hasCode(err, "NoSuchUpload") { + return nil, err + } + reuse := 0 // Index of next old part to consider reusing. + current := 1 // Part number of latest good part handled. + totalSize, err := r.Seek(0, 2) + if err != nil { + return nil, err + } + first := true // Must send at least one empty part if the file is empty. + var result []Part +NextSection: + for offset := int64(0); offset < totalSize || first; offset += partSize { + first = false + if offset+partSize > totalSize { + partSize = totalSize - offset + } + section := io.NewSectionReader(r, offset, partSize) + _, md5hex, md5b64, err := seekerInfo(section) + if err != nil { + return nil, err + } + for reuse < len(old) && old[reuse].N <= current { + // Looks like this part was already sent. + part := &old[reuse] + etag := `"` + md5hex + `"` + if part.N == current && part.Size == partSize && part.ETag == etag { + // Checksum matches. Reuse the old part. + result = append(result, *part) + current++ + continue NextSection + } + reuse++ + } + + // Part wasn't found or doesn't match. Send it. + part, err := m.putPart(current, section, partSize, md5b64) + if err != nil { + return nil, err + } + result = append(result, part) + current++ + } + return result, nil +} + +type completeUpload struct { + XMLName xml.Name `xml:"CompleteMultipartUpload"` + Parts completeParts `xml:"Part"` +} + +type completePart struct { + PartNumber int + ETag string +} + +type completeParts []completePart + +func (p completeParts) Len() int { return len(p) } +func (p completeParts) Less(i, j int) bool { return p[i].PartNumber < p[j].PartNumber } +func (p completeParts) Swap(i, j int) { p[i], p[j] = p[j], p[i] } + +type completeResponse struct { + // The element name: should be either CompleteMultipartUploadResult or Error. + XMLName xml.Name + // If the element was error, then it should have the following: + Code string + Message string + RequestId string + HostId string +} + +// Complete assembles the given previously uploaded parts into the +// final object. This operation may take several minutes. +// +// The complete call to AMZ may still fail after returning HTTP 200, +// so even though it's unusued, the body of the reply must be demarshalled +// and checked to see whether or not the complete succeeded. +// +// See http://goo.gl/2Z7Tw for details. +func (m *Multi) Complete(parts []Part) error { + params := map[string][]string{ + "uploadId": {m.UploadId}, + } + c := completeUpload{} + for _, p := range parts { + c.Parts = append(c.Parts, completePart{p.N, p.ETag}) + } + sort.Sort(c.Parts) + data, err := xml.Marshal(&c) + if err != nil { + return err + } + + // Setting Content-Length prevents breakage on DreamObjects + for attempt := m.Bucket.S3.AttemptStrategy.Start(); attempt.Next(); { + req := &request{ + method: "POST", + bucket: m.Bucket.Name, + path: m.Key, + params: params, + payload: bytes.NewReader(data), + headers: map[string][]string{ + "Content-Length": []string{strconv.Itoa(len(data))}, + }, + } + + resp := &completeResponse{} + err := m.Bucket.S3.query(req, resp) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err == nil && resp.XMLName.Local == "Error" { + err = &Error{ + StatusCode: 200, + Code: resp.Code, + Message: resp.Message, + RequestId: resp.RequestId, + HostId: resp.HostId, + } + } + return err + } + panic("unreachable") +} + +// Abort deletes an unifinished multipart upload and any previously +// uploaded parts for it. +// +// After a multipart upload is aborted, no additional parts can be +// uploaded using it. However, if any part uploads are currently in +// progress, those part uploads might or might not succeed. As a result, +// it might be necessary to abort a given multipart upload multiple +// times in order to completely free all storage consumed by all parts. +// +// NOTE: If the described scenario happens to you, please report back to +// the goamz authors with details. In the future such retrying should be +// handled internally, but it's not clear what happens precisely (Is an +// error returned? Is the issue completely undetectable?). +// +// See http://goo.gl/dnyJw for details. +func (m *Multi) Abort() error { + params := map[string][]string{ + "uploadId": {m.UploadId}, + } + for attempt := m.Bucket.S3.AttemptStrategy.Start(); attempt.Next(); { + req := &request{ + method: "DELETE", + bucket: m.Bucket.Name, + path: m.Key, + params: params, + } + err := m.Bucket.S3.query(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + return err + } + panic("unreachable") +} diff --git a/vendor/github.com/goamz/goamz/s3/s3.go b/vendor/github.com/goamz/goamz/s3/s3.go new file mode 100644 index 000000000..c659aa6ba --- /dev/null +++ b/vendor/github.com/goamz/goamz/s3/s3.go @@ -0,0 +1,1161 @@ +// +// goamz - Go packages to interact with the Amazon Web Services. +// +// https://wiki.ubuntu.com/goamz +// +// Copyright (c) 2011 Canonical Ltd. +// +// Written by Gustavo Niemeyer +// + +package s3 + +import ( + "bytes" + "crypto/hmac" + "crypto/md5" + "crypto/sha1" + "encoding/base64" + "encoding/xml" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strconv" + "strings" + "time" + + "github.com/goamz/goamz/aws" +) + +const debug = false + +// The S3 type encapsulates operations with an S3 region. +type S3 struct { + aws.Auth + aws.Region + + // ConnectTimeout is the maximum time a request attempt will + // wait for a successful connection to be made. + // + // A value of zero means no timeout. + ConnectTimeout time.Duration + + // ReadTimeout is the maximum time a request attempt will wait + // for an individual read to complete. + // + // A value of zero means no timeout. + ReadTimeout time.Duration + + // WriteTimeout is the maximum time a request attempt will + // wait for an individual write to complete. + // + // A value of zero means no timeout. + WriteTimeout time.Duration + + // RequestTimeout is the maximum time a request attempt can + // take before operations return a timeout error. + // + // This includes connection time, any redirects, and reading + // the response body. The timer remains running after the request + // is made so it can interrupt reading of the response data. + // + // A Timeout of zero means no timeout. + RequestTimeout time.Duration + + // AttemptStrategy is the attempt strategy used for requests. + aws.AttemptStrategy + + // Reserve the right of using private data. + private byte + + // client used for requests + client *http.Client +} + +// The Bucket type encapsulates operations with an S3 bucket. +type Bucket struct { + *S3 + Name string +} + +// The Owner type represents the owner of the object in an S3 bucket. +type Owner struct { + ID string + DisplayName string +} + +// Fold options into an Options struct +// +type Options struct { + SSE bool + Meta map[string][]string + ContentEncoding string + CacheControl string + RedirectLocation string + ContentMD5 string + // What else? + // Content-Disposition string + //// The following become headers so they are []strings rather than strings... I think + // x-amz-storage-class []string +} + +type CopyOptions struct { + Options + MetadataDirective string + ContentType string +} + +// CopyObjectResult is the output from a Copy request +type CopyObjectResult struct { + ETag string + LastModified string +} + +// DefaultAttemptStrategy is the default AttemptStrategy used by S3 objects created by New. +var DefaultAttemptStrategy = aws.AttemptStrategy{ + Min: 5, + Total: 5 * time.Second, + Delay: 200 * time.Millisecond, +} + +// New creates a new S3. Optional client argument allows for custom http.clients to be used. +func New(auth aws.Auth, region aws.Region, client ...*http.Client) *S3 { + + var httpclient *http.Client + + if len(client) > 0 { + httpclient = client[0] + } + + return &S3{Auth: auth, Region: region, AttemptStrategy: DefaultAttemptStrategy, client: httpclient} +} + +// Bucket returns a Bucket with the given name. +func (s3 *S3) Bucket(name string) *Bucket { + if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket { + name = strings.ToLower(name) + } + return &Bucket{s3, name} +} + +var createBucketConfiguration = ` + %s +` + +// locationConstraint returns an io.Reader specifying a LocationConstraint if +// required for the region. +// +// See http://goo.gl/bh9Kq for details. +func (s3 *S3) locationConstraint() io.Reader { + constraint := "" + if s3.Region.S3LocationConstraint { + constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name) + } + return strings.NewReader(constraint) +} + +type ACL string + +const ( + Private = ACL("private") + PublicRead = ACL("public-read") + PublicReadWrite = ACL("public-read-write") + AuthenticatedRead = ACL("authenticated-read") + BucketOwnerRead = ACL("bucket-owner-read") + BucketOwnerFull = ACL("bucket-owner-full-control") +) + +// PutBucket creates a new bucket. +// +// See http://goo.gl/ndjnR for details. +func (b *Bucket) PutBucket(perm ACL) error { + headers := map[string][]string{ + "x-amz-acl": {string(perm)}, + } + req := &request{ + method: "PUT", + bucket: b.Name, + path: "/", + headers: headers, + payload: b.locationConstraint(), + } + return b.S3.query(req, nil) +} + +// DelBucket removes an existing S3 bucket. All objects in the bucket must +// be removed before the bucket itself can be removed. +// +// See http://goo.gl/GoBrY for details. +func (b *Bucket) DelBucket() (err error) { + req := &request{ + method: "DELETE", + bucket: b.Name, + path: "/", + } + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + err = b.S3.query(req, nil) + if !shouldRetry(err) { + break + } + } + return err +} + +// Get retrieves an object from an S3 bucket. +// +// See http://goo.gl/isCO7 for details. +func (b *Bucket) Get(path string) (data []byte, err error) { + body, err := b.GetReader(path) + defer func() { + if body != nil { + body.Close() + } + }() + if err != nil { + return nil, err + } + data, err = ioutil.ReadAll(body) + return data, err +} + +// GetReader retrieves an object from an S3 bucket, +// returning the body of the HTTP response. +// It is the caller's responsibility to call Close on rc when +// finished reading. +func (b *Bucket) GetReader(path string) (rc io.ReadCloser, err error) { + resp, err := b.GetResponse(path) + if resp != nil { + return resp.Body, err + } + return nil, err +} + +// GetResponse retrieves an object from an S3 bucket, +// returning the HTTP response. +// It is the caller's responsibility to call Close on rc when +// finished reading +func (b *Bucket) GetResponse(path string) (resp *http.Response, err error) { + return b.GetResponseWithHeaders(path, make(http.Header)) +} + +// GetReaderWithHeaders retrieves an object from an S3 bucket +// Accepts custom headers to be sent as the second parameter +// returning the body of the HTTP response. +// It is the caller's responsibility to call Close on rc when +// finished reading +func (b *Bucket) GetResponseWithHeaders(path string, headers map[string][]string) (resp *http.Response, err error) { + req := &request{ + bucket: b.Name, + path: path, + headers: headers, + } + err = b.S3.prepare(req) + if err != nil { + return nil, err + } + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + resp, err := b.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + return resp, nil + } + panic("unreachable") +} + +// Exists checks whether or not an object exists on an S3 bucket using a HEAD request. +func (b *Bucket) Exists(path string) (exists bool, err error) { + req := &request{ + method: "HEAD", + bucket: b.Name, + path: path, + } + err = b.S3.prepare(req) + if err != nil { + return + } + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + resp, err := b.S3.run(req, nil) + + if shouldRetry(err) && attempt.HasNext() { + continue + } + + if err != nil { + // We can treat a 403 or 404 as non existance + if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) { + return false, nil + } + return false, err + } + + if resp.StatusCode/100 == 2 { + exists = true + } + return exists, err + } + return false, fmt.Errorf("S3 Currently Unreachable") +} + +// Head HEADs an object in the S3 bucket, returns the response with +// no body see http://bit.ly/17K1ylI +func (b *Bucket) Head(path string, headers map[string][]string) (*http.Response, error) { + req := &request{ + method: "HEAD", + bucket: b.Name, + path: path, + headers: headers, + } + err := b.S3.prepare(req) + if err != nil { + return nil, err + } + + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + resp, err := b.S3.run(req, nil) + if shouldRetry(err) && attempt.HasNext() { + continue + } + if err != nil { + return nil, err + } + return resp, err + } + return nil, fmt.Errorf("S3 Currently Unreachable") +} + +// Put inserts an object into the S3 bucket. +// +// See http://goo.gl/FEBPD for details. +func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, options Options) error { + body := bytes.NewBuffer(data) + return b.PutReader(path, body, int64(len(data)), contType, perm, options) +} + +// PutCopy puts a copy of an object given by the key path into bucket b using b.Path as the target key +func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (result *CopyObjectResult, err error) { + headers := map[string][]string{ + "x-amz-acl": {string(perm)}, + "x-amz-copy-source": {source}, + } + options.addHeaders(headers) + req := &request{ + method: "PUT", + bucket: b.Name, + path: path, + headers: headers, + } + result = &CopyObjectResult{} + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + err = b.S3.query(req, result) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + return result, nil +} + +/* +PutHeader - like Put, inserts an object into the S3 bucket. +Instead of Content-Type string, pass in custom headers to override defaults. +*/ +func (b *Bucket) PutHeader(path string, data []byte, customHeaders map[string][]string, perm ACL) error { + body := bytes.NewBuffer(data) + return b.PutReaderHeader(path, body, int64(len(data)), customHeaders, perm) +} + +// PutReader inserts an object into the S3 bucket by consuming data +// from r until EOF. +func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType string, perm ACL, options Options) error { + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(length, 10)}, + "Content-Type": {contType}, + "x-amz-acl": {string(perm)}, + } + options.addHeaders(headers) + req := &request{ + method: "PUT", + bucket: b.Name, + path: path, + headers: headers, + payload: r, + } + return b.S3.query(req, nil) +} + +/* +PutReaderHeader - like PutReader, inserts an object into S3 from a reader. +Instead of Content-Type string, pass in custom headers to override defaults. +*/ +func (b *Bucket) PutReaderHeader(path string, r io.Reader, length int64, customHeaders map[string][]string, perm ACL) error { + // Default headers + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(length, 10)}, + "Content-Type": {"application/text"}, + "x-amz-acl": {string(perm)}, + } + + // Override with custom headers + for key, value := range customHeaders { + headers[key] = value + } + + req := &request{ + method: "PUT", + bucket: b.Name, + path: path, + headers: headers, + payload: r, + } + return b.S3.query(req, nil) +} + +// addHeaders adds o's specified fields to headers +func (o Options) addHeaders(headers map[string][]string) { + if o.SSE { + headers["x-amz-server-side-encryption"] = []string{"AES256"} + } + if len(o.ContentEncoding) != 0 { + headers["Content-Encoding"] = []string{o.ContentEncoding} + } + if len(o.CacheControl) != 0 { + headers["Cache-Control"] = []string{o.CacheControl} + } + if len(o.ContentMD5) != 0 { + headers["Content-MD5"] = []string{o.ContentMD5} + } + if len(o.RedirectLocation) != 0 { + headers["x-amz-website-redirect-location"] = []string{o.RedirectLocation} + } + for k, v := range o.Meta { + headers["x-amz-meta-"+k] = v + } +} + +// addHeaders adds o's specified fields to headers +func (o CopyOptions) addHeaders(headers map[string][]string) { + o.Options.addHeaders(headers) + if len(o.MetadataDirective) != 0 { + headers["x-amz-metadata-directive"] = []string{o.MetadataDirective} + } + if len(o.ContentType) != 0 { + headers["Content-Type"] = []string{o.ContentType} + } +} + +func makeXmlBuffer(doc []byte) *bytes.Buffer { + buf := new(bytes.Buffer) + buf.WriteString(xml.Header) + buf.Write(doc) + return buf +} + +type RoutingRule struct { + ConditionKeyPrefixEquals string `xml:"Condition>KeyPrefixEquals"` + RedirectReplaceKeyPrefixWith string `xml:"Redirect>ReplaceKeyPrefixWith,omitempty"` + RedirectReplaceKeyWith string `xml:"Redirect>ReplaceKeyWith,omitempty"` +} + +type WebsiteConfiguration struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration"` + IndexDocumentSuffix string `xml:"IndexDocument>Suffix"` + ErrorDocumentKey string `xml:"ErrorDocument>Key"` + RoutingRules *[]RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` +} + +func (b *Bucket) PutBucketWebsite(configuration WebsiteConfiguration) error { + + doc, err := xml.Marshal(configuration) + if err != nil { + return err + } + + buf := makeXmlBuffer(doc) + + return b.PutBucketSubresource("website", buf, int64(buf.Len())) +} + +func (b *Bucket) PutBucketSubresource(subresource string, r io.Reader, length int64) error { + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(length, 10)}, + } + req := &request{ + path: "/", + method: "PUT", + bucket: b.Name, + headers: headers, + payload: r, + params: url.Values{subresource: {""}}, + } + + return b.S3.query(req, nil) +} + +// Del removes an object from the S3 bucket. +// +// See http://goo.gl/APeTt for details. +func (b *Bucket) Del(path string) error { + req := &request{ + method: "DELETE", + bucket: b.Name, + path: path, + } + return b.S3.query(req, nil) +} + +type Delete struct { + Quiet bool `xml:"Quiet,omitempty"` + Objects []Object `xml:"Object"` +} + +type Object struct { + Key string `xml:"Key"` + VersionId string `xml:"VersionId,omitempty"` +} + +// DelMulti removes up to 1000 objects from the S3 bucket. +// +// See http://goo.gl/jx6cWK for details. +func (b *Bucket) DelMulti(objects Delete) error { + doc, err := xml.Marshal(objects) + if err != nil { + return err + } + + buf := makeXmlBuffer(doc) + digest := md5.New() + size, err := digest.Write(buf.Bytes()) + if err != nil { + return err + } + + headers := map[string][]string{ + "Content-Length": {strconv.FormatInt(int64(size), 10)}, + "Content-MD5": {base64.StdEncoding.EncodeToString(digest.Sum(nil))}, + "Content-Type": {"text/xml"}, + } + req := &request{ + path: "/", + method: "POST", + params: url.Values{"delete": {""}}, + bucket: b.Name, + headers: headers, + payload: buf, + } + + return b.S3.query(req, nil) +} + +// The ListResp type holds the results of a List bucket operation. +type ListResp struct { + Name string + Prefix string + Delimiter string + Marker string + NextMarker string + MaxKeys int + + // IsTruncated is true if the results have been truncated because + // there are more keys and prefixes than can fit in MaxKeys. + // N.B. this is the opposite sense to that documented (incorrectly) in + // http://goo.gl/YjQTc + IsTruncated bool + Contents []Key + CommonPrefixes []string `xml:">Prefix"` +} + +// The Key type represents an item stored in an S3 bucket. +type Key struct { + Key string + LastModified string + Size int64 + // ETag gives the hex-encoded MD5 sum of the contents, + // surrounded with double-quotes. + ETag string + StorageClass string + Owner Owner +} + +// List returns information about objects in an S3 bucket. +// +// The prefix parameter limits the response to keys that begin with the +// specified prefix. +// +// The delim parameter causes the response to group all of the keys that +// share a common prefix up to the next delimiter in a single entry within +// the CommonPrefixes field. You can use delimiters to separate a bucket +// into different groupings of keys, similar to how folders would work. +// +// The marker parameter specifies the key to start with when listing objects +// in a bucket. Amazon S3 lists objects in alphabetical order and +// will return keys alphabetically greater than the marker. +// +// The max parameter specifies how many keys + common prefixes to return in +// the response. The default is 1000. +// +// For example, given these keys in a bucket: +// +// index.html +// index2.html +// photos/2006/January/sample.jpg +// photos/2006/February/sample2.jpg +// photos/2006/February/sample3.jpg +// photos/2006/February/sample4.jpg +// +// Listing this bucket with delimiter set to "/" would yield the +// following result: +// +// &ListResp{ +// Name: "sample-bucket", +// MaxKeys: 1000, +// Delimiter: "/", +// Contents: []Key{ +// {Key: "index.html", "index2.html"}, +// }, +// CommonPrefixes: []string{ +// "photos/", +// }, +// } +// +// Listing the same bucket with delimiter set to "/" and prefix set to +// "photos/2006/" would yield the following result: +// +// &ListResp{ +// Name: "sample-bucket", +// MaxKeys: 1000, +// Delimiter: "/", +// Prefix: "photos/2006/", +// CommonPrefixes: []string{ +// "photos/2006/February/", +// "photos/2006/January/", +// }, +// } +// +// See http://goo.gl/YjQTc for details. +func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) { + params := map[string][]string{ + "prefix": {prefix}, + "delimiter": {delim}, + "marker": {marker}, + } + if max != 0 { + params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} + } + req := &request{ + bucket: b.Name, + params: params, + } + result = &ListResp{} + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + err = b.S3.query(req, result) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + return result, nil +} + +// The VersionsResp type holds the results of a list bucket Versions operation. +type VersionsResp struct { + Name string + Prefix string + KeyMarker string + VersionIdMarker string + MaxKeys int + Delimiter string + IsTruncated bool + Versions []Version + CommonPrefixes []string `xml:">Prefix"` +} + +// The Version type represents an object version stored in an S3 bucket. +type Version struct { + Key string + VersionId string + IsLatest bool + LastModified string + // ETag gives the hex-encoded MD5 sum of the contents, + // surrounded with double-quotes. + ETag string + Size int64 + Owner Owner + StorageClass string +} + +func (b *Bucket) Versions(prefix, delim, keyMarker string, versionIdMarker string, max int) (result *VersionsResp, err error) { + params := map[string][]string{ + "versions": {""}, + "prefix": {prefix}, + "delimiter": {delim}, + } + + if len(versionIdMarker) != 0 { + params["version-id-marker"] = []string{versionIdMarker} + } + if len(keyMarker) != 0 { + params["key-marker"] = []string{keyMarker} + } + + if max != 0 { + params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} + } + req := &request{ + bucket: b.Name, + params: params, + } + result = &VersionsResp{} + for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { + err = b.S3.query(req, result) + if !shouldRetry(err) { + break + } + } + if err != nil { + return nil, err + } + return result, nil +} + +// Returns a mapping of all key names in this bucket to Key objects +func (b *Bucket) GetBucketContents() (*map[string]Key, error) { + bucket_contents := map[string]Key{} + prefix := "" + path_separator := "" + marker := "" + for { + contents, err := b.List(prefix, path_separator, marker, 1000) + if err != nil { + return &bucket_contents, err + } + for _, key := range contents.Contents { + bucket_contents[key.Key] = key + } + if contents.IsTruncated { + marker = contents.NextMarker + } else { + break + } + } + + return &bucket_contents, nil +} + +// URL returns a non-signed URL that allows retriving the +// object at path. It only works if the object is publicly +// readable (see SignedURL). +func (b *Bucket) URL(path string) string { + req := &request{ + bucket: b.Name, + path: path, + } + err := b.S3.prepare(req) + if err != nil { + panic(err) + } + u, err := req.url() + if err != nil { + panic(err) + } + u.RawQuery = "" + return u.String() +} + +// SignedURL returns a signed URL that allows anyone holding the URL +// to retrieve the object at path. The signature is valid until expires. +func (b *Bucket) SignedURL(path string, expires time.Time) string { + req := &request{ + bucket: b.Name, + path: path, + params: url.Values{"Expires": {strconv.FormatInt(expires.Unix(), 10)}}, + } + err := b.S3.prepare(req) + if err != nil { + panic(err) + } + u, err := req.url() + if err != nil { + panic(err) + } + if b.S3.Auth.Token() != "" { + return u.String() + "&x-amz-security-token=" + url.QueryEscape(req.headers["X-Amz-Security-Token"][0]) + } else { + return u.String() + } +} + +// UploadSignedURL returns a signed URL that allows anyone holding the URL +// to upload the object at path. The signature is valid until expires. +// contenttype is a string like image/png +// path is the resource name in s3 terminalogy like images/ali.png [obviously exclusing the bucket name itself] +func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time.Time) string { + expire_date := expires.Unix() + if method != "POST" { + method = "PUT" + } + stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n/" + b.Name + "/" + path + fmt.Println("String to sign:\n", stringToSign) + a := b.S3.Auth + secretKey := a.SecretKey + accessId := a.AccessKey + mac := hmac.New(sha1.New, []byte(secretKey)) + mac.Write([]byte(stringToSign)) + macsum := mac.Sum(nil) + signature := base64.StdEncoding.EncodeToString([]byte(macsum)) + signature = strings.TrimSpace(signature) + + signedurl, err := url.Parse("https://" + b.Name + ".s3.amazonaws.com/") + if err != nil { + log.Println("ERROR sining url for S3 upload", err) + return "" + } + signedurl.Path += path + params := url.Values{} + params.Add("AWSAccessKeyId", accessId) + params.Add("Expires", strconv.FormatInt(expire_date, 10)) + params.Add("Signature", signature) + if a.Token() != "" { + params.Add("token", a.Token()) + } + + signedurl.RawQuery = params.Encode() + return signedurl.String() +} + +// PostFormArgs returns the action and input fields needed to allow anonymous +// uploads to a bucket within the expiration limit +func (b *Bucket) PostFormArgs(path string, expires time.Time, redirect string) (action string, fields map[string]string) { + conditions := make([]string, 0) + fields = map[string]string{ + "AWSAccessKeyId": b.Auth.AccessKey, + "key": path, + } + + conditions = append(conditions, fmt.Sprintf("{\"key\": \"%s\"}", path)) + conditions = append(conditions, fmt.Sprintf("{\"bucket\": \"%s\"}", b.Name)) + if redirect != "" { + conditions = append(conditions, fmt.Sprintf("{\"success_action_redirect\": \"%s\"}", redirect)) + fields["success_action_redirect"] = redirect + } + + vExpiration := expires.Format("2006-01-02T15:04:05Z") + vConditions := strings.Join(conditions, ",") + policy := fmt.Sprintf("{\"expiration\": \"%s\", \"conditions\": [%s]}", vExpiration, vConditions) + policy64 := base64.StdEncoding.EncodeToString([]byte(policy)) + fields["policy"] = policy64 + + signer := hmac.New(sha1.New, []byte(b.Auth.SecretKey)) + signer.Write([]byte(policy64)) + fields["signature"] = base64.StdEncoding.EncodeToString(signer.Sum(nil)) + + action = fmt.Sprintf("%s/%s/", b.S3.Region.S3Endpoint, b.Name) + return +} + +type request struct { + method string + bucket string + path string + signpath string + params url.Values + headers http.Header + baseurl string + payload io.Reader + prepared bool +} + +func (req *request) url() (*url.URL, error) { + u, err := url.Parse(req.baseurl) + if err != nil { + return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) + } + u.RawQuery = req.params.Encode() + u.Path = req.path + return u, nil +} + +// query prepares and runs the req request. +// If resp is not nil, the XML data contained in the response +// body will be unmarshalled on it. +func (s3 *S3) query(req *request, resp interface{}) error { + err := s3.prepare(req) + if err == nil { + var httpResponse *http.Response + httpResponse, err = s3.run(req, resp) + if resp == nil && httpResponse != nil { + httpResponse.Body.Close() + } + } + return err +} + +// prepare sets up req to be delivered to S3. +func (s3 *S3) prepare(req *request) error { + var signpath = req.path + + if !req.prepared { + req.prepared = true + if req.method == "" { + req.method = "GET" + } + // Copy so they can be mutated without affecting on retries. + params := make(url.Values) + headers := make(http.Header) + for k, v := range req.params { + params[k] = v + } + for k, v := range req.headers { + headers[k] = v + } + req.params = params + req.headers = headers + if !strings.HasPrefix(req.path, "/") { + req.path = "/" + req.path + } + signpath = req.path + if req.bucket != "" { + req.baseurl = s3.Region.S3BucketEndpoint + if req.baseurl == "" { + // Use the path method to address the bucket. + req.baseurl = s3.Region.S3Endpoint + req.path = "/" + req.bucket + req.path + } else { + // Just in case, prevent injection. + if strings.IndexAny(req.bucket, "/:@") >= 0 { + return fmt.Errorf("bad S3 bucket: %q", req.bucket) + } + req.baseurl = strings.Replace(req.baseurl, "${bucket}", req.bucket, -1) + } + signpath = "/" + req.bucket + signpath + } + } + + // Always sign again as it's not clear how far the + // server has handled a previous attempt. + u, err := url.Parse(req.baseurl) + if err != nil { + return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) + } + reqSignpathSpaceFix := (&url.URL{Path: signpath}).String() + req.headers["Host"] = []string{u.Host} + req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} + if s3.Auth.Token() != "" { + req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()} + } + sign(s3.Auth, req.method, reqSignpathSpaceFix, req.params, req.headers) + return nil +} + +// run sends req and returns the http response from the server. +// If resp is not nil, the XML data contained in the response +// body will be unmarshalled on it. +func (s3 *S3) run(req *request, resp interface{}) (*http.Response, error) { + if debug { + log.Printf("Running S3 request: %#v", req) + } + + u, err := req.url() + if err != nil { + return nil, err + } + + hreq := http.Request{ + URL: u, + Method: req.method, + ProtoMajor: 1, + ProtoMinor: 1, + Close: true, + Header: req.headers, + } + + if v, ok := req.headers["Content-Length"]; ok { + hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64) + delete(req.headers, "Content-Length") + } + if req.payload != nil { + hreq.Body = ioutil.NopCloser(req.payload) + } + + if s3.client == nil { + s3.client = &http.Client{ + Transport: &http.Transport{ + Dial: func(netw, addr string) (c net.Conn, err error) { + c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout) + if err != nil { + return + } + + var deadline time.Time + if s3.RequestTimeout > 0 { + deadline = time.Now().Add(s3.RequestTimeout) + c.SetDeadline(deadline) + } + + if s3.ReadTimeout > 0 || s3.WriteTimeout > 0 { + c = &ioTimeoutConn{ + TCPConn: c.(*net.TCPConn), + readTimeout: s3.ReadTimeout, + writeTimeout: s3.WriteTimeout, + requestDeadline: deadline, + } + } + return + }, + }, + } + } + + hresp, err := s3.client.Do(&hreq) + if err != nil { + return nil, err + } + if debug { + dump, _ := httputil.DumpResponse(hresp, true) + log.Printf("} -> %s\n", dump) + } + if hresp.StatusCode != 200 && hresp.StatusCode != 204 && hresp.StatusCode != 206 { + defer hresp.Body.Close() + return nil, buildError(hresp) + } + if resp != nil { + err = xml.NewDecoder(hresp.Body).Decode(resp) + hresp.Body.Close() + if debug { + log.Printf("goamz.s3> decoded xml into %#v", resp) + } + } + return hresp, err +} + +// Error represents an error in an operation with S3. +type Error struct { + StatusCode int // HTTP status code (200, 403, ...) + Code string // EC2 error code ("UnsupportedOperation", ...) + Message string // The human-oriented error message + BucketName string + RequestId string + HostId string +} + +func (e *Error) Error() string { + return e.Message +} + +func buildError(r *http.Response) error { + if debug { + log.Printf("got error (status code %v)", r.StatusCode) + data, err := ioutil.ReadAll(r.Body) + if err != nil { + log.Printf("\tread error: %v", err) + } else { + log.Printf("\tdata:\n%s\n\n", data) + } + r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) + } + + err := Error{} + // TODO return error if Unmarshal fails? + xml.NewDecoder(r.Body).Decode(&err) + r.Body.Close() + err.StatusCode = r.StatusCode + if err.Message == "" { + err.Message = r.Status + } + if debug { + log.Printf("err: %#v\n", err) + } + return &err +} + +func shouldRetry(err error) bool { + if err == nil { + return false + } + if e, ok := err.(*url.Error); ok { + // Transport returns this string if it detects a write on a connection which + // has already had an error + if e.Err.Error() == "http: can't write HTTP request on broken connection" { + return true + } + err = e.Err + } + + switch err { + case io.ErrUnexpectedEOF, io.EOF: + return true + } + switch e := err.(type) { + case *net.DNSError: + return true + case *net.OpError: + switch e.Op { + case "read", "write", "WSARecv", "WSASend", "ConnectEx": + return true + } + case *Error: + switch e.Code { + case "InternalError", "NoSuchUpload", "NoSuchBucket", "RequestTimeout": + return true + } + // let's handle tls handshake timeout issues and similar temporary errors + case net.Error: + return e.Temporary() + } + + return false +} + +func hasCode(err error, code string) bool { + s3err, ok := err.(*Error) + return ok && s3err.Code == code +} + +// ioTimeoutConn is a net.Conn which sets a deadline for each Read or Write operation +type ioTimeoutConn struct { + *net.TCPConn + readTimeout time.Duration + writeTimeout time.Duration + requestDeadline time.Time +} + +func (c *ioTimeoutConn) deadline(timeout time.Duration) time.Time { + dl := time.Now().Add(timeout) + if c.requestDeadline.IsZero() || dl.Before(c.requestDeadline) { + return dl + } + + return c.requestDeadline +} + +func (c *ioTimeoutConn) Read(b []byte) (int, error) { + if c.readTimeout > 0 { + err := c.TCPConn.SetReadDeadline(c.deadline(c.readTimeout)) + if err != nil { + return 0, err + } + } + return c.TCPConn.Read(b) +} + +func (c *ioTimeoutConn) Write(b []byte) (int, error) { + if c.writeTimeout > 0 { + err := c.TCPConn.SetWriteDeadline(c.deadline(c.writeTimeout)) + if err != nil { + return 0, err + } + } + return c.TCPConn.Write(b) +} diff --git a/vendor/github.com/goamz/goamz/s3/sign.go b/vendor/github.com/goamz/goamz/s3/sign.go new file mode 100644 index 000000000..722d97d29 --- /dev/null +++ b/vendor/github.com/goamz/goamz/s3/sign.go @@ -0,0 +1,141 @@ +package s3 + +import ( + "crypto/hmac" + "crypto/sha1" + "encoding/base64" + "github.com/goamz/goamz/aws" + "log" + "sort" + "strings" +) + +var b64 = base64.StdEncoding + +// ---------------------------------------------------------------------------- +// S3 signing (http://goo.gl/G1LrK) + +var s3ParamsToSign = map[string]bool{ + "acl": true, + "location": true, + "logging": true, + "notification": true, + "partNumber": true, + "policy": true, + "requestPayment": true, + "torrent": true, + "uploadId": true, + "uploads": true, + "versionId": true, + "versioning": true, + "versions": true, + "response-content-type": true, + "response-content-language": true, + "response-expires": true, + "response-cache-control": true, + "response-content-disposition": true, + "response-content-encoding": true, + "website": true, + "delete": true, +} + +type keySortableTupleList []keySortableTuple + +type keySortableTuple struct { + Key string + TupleString string +} + +func (l keySortableTupleList) StringSlice() []string { + slice := make([]string, len(l)) + for i, v := range l { + slice[i] = v.TupleString + } + return slice +} + +func (l keySortableTupleList) Len() int { + return len(l) +} + +func (l keySortableTupleList) Less(i, j int) bool { + return l[i].Key < l[j].Key +} + +func (l keySortableTupleList) Swap(i, j int) { + l[i], l[j] = l[j], l[i] +} + +func sign(auth aws.Auth, method, canonicalPath string, params, headers map[string][]string) { + var md5, ctype, date, xamz string + var xamzDate bool + var sarray keySortableTupleList + for k, v := range headers { + k = strings.ToLower(k) + switch k { + case "content-md5": + md5 = v[0] + case "content-type": + ctype = v[0] + case "date": + if !xamzDate { + date = v[0] + } + default: + if strings.HasPrefix(k, "x-amz-") { + vall := strings.Join(v, ",") + sarray = append(sarray, keySortableTuple{k, k + ":" + vall}) + if k == "x-amz-date" { + xamzDate = true + date = "" + } + } + } + } + if len(sarray) > 0 { + sort.Sort(sarray) + xamz = strings.Join(sarray.StringSlice(), "\n") + "\n" + } + + expires := false + if v, ok := params["Expires"]; ok { + // Query string request authentication alternative. + expires = true + date = v[0] + params["AWSAccessKeyId"] = []string{auth.AccessKey} + } + + sarray = sarray[0:0] + for k, v := range params { + if s3ParamsToSign[k] { + for _, vi := range v { + if vi == "" { + sarray = append(sarray, keySortableTuple{k, k}) + } else { + // "When signing you do not encode these values." + sarray = append(sarray, keySortableTuple{k, k + "=" + vi}) + } + } + } + } + if len(sarray) > 0 { + sort.Sort(sarray) + canonicalPath = canonicalPath + "?" + strings.Join(sarray.StringSlice(), "&") + } + + payload := method + "\n" + md5 + "\n" + ctype + "\n" + date + "\n" + xamz + canonicalPath + hash := hmac.New(sha1.New, []byte(auth.SecretKey)) + hash.Write([]byte(payload)) + signature := make([]byte, b64.EncodedLen(hash.Size())) + b64.Encode(signature, hash.Sum(nil)) + + if expires { + params["Signature"] = []string{string(signature)} + } else { + headers["Authorization"] = []string{"AWS " + auth.AccessKey + ":" + string(signature)} + } + if debug { + log.Printf("Signature payload: %q", payload) + log.Printf("Signature: %q", signature) + } +} -- cgit v1.2.3-1-g7c22