diff options
Diffstat (limited to 'vendor/github.com/goamz/goamz/s3/s3.go')
| -rw-r--r-- | vendor/github.com/goamz/goamz/s3/s3.go | 1164 |
1 files changed, 0 insertions, 1164 deletions
diff --git a/vendor/github.com/goamz/goamz/s3/s3.go b/vendor/github.com/goamz/goamz/s3/s3.go deleted file mode 100644 index f27479cb4..000000000 --- a/vendor/github.com/goamz/goamz/s3/s3.go +++ /dev/null @@ -1,1164 +0,0 @@ -// -// goamz - Go packages to interact with the Amazon Web Services. -// -// https://wiki.ubuntu.com/goamz -// -// Copyright (c) 2011 Canonical Ltd. -// -// Written by Gustavo Niemeyer <gustavo.niemeyer@canonical.com> -// - -package s3 - -import ( - "bytes" - "crypto/hmac" - "crypto/md5" - "crypto/sha1" - "encoding/base64" - "encoding/xml" - "fmt" - "io" - "io/ioutil" - "log" - "net" - "net/http" - "net/http/httputil" - "net/url" - "strconv" - "strings" - "time" - - "github.com/goamz/goamz/aws" -) - -const debug = false - -// The S3 type encapsulates operations with an S3 region. -type S3 struct { - aws.Auth - aws.Region - - // ConnectTimeout is the maximum time a request attempt will - // wait for a successful connection to be made. - // - // A value of zero means no timeout. - ConnectTimeout time.Duration - - // ReadTimeout is the maximum time a request attempt will wait - // for an individual read to complete. - // - // A value of zero means no timeout. - ReadTimeout time.Duration - - // WriteTimeout is the maximum time a request attempt will - // wait for an individual write to complete. - // - // A value of zero means no timeout. - WriteTimeout time.Duration - - // RequestTimeout is the maximum time a request attempt can - // take before operations return a timeout error. - // - // This includes connection time, any redirects, and reading - // the response body. The timer remains running after the request - // is made so it can interrupt reading of the response data. - // - // A Timeout of zero means no timeout. - RequestTimeout time.Duration - - // AttemptStrategy is the attempt strategy used for requests. - aws.AttemptStrategy - - // Reserve the right of using private data. - private byte - - // client used for requests - client *http.Client -} - -// The Bucket type encapsulates operations with an S3 bucket. -type Bucket struct { - *S3 - Name string -} - -// The Owner type represents the owner of the object in an S3 bucket. -type Owner struct { - ID string - DisplayName string -} - -// Fold options into an Options struct -// -type Options struct { - SSE bool - Meta map[string][]string - ContentEncoding string - CacheControl string - RedirectLocation string - ContentMD5 string - // What else? - // Content-Disposition string - //// The following become headers so they are []strings rather than strings... I think - // x-amz-storage-class []string -} - -type CopyOptions struct { - Options - MetadataDirective string - ContentType string -} - -// CopyObjectResult is the output from a Copy request -type CopyObjectResult struct { - ETag string - LastModified string -} - -// DefaultAttemptStrategy is the default AttemptStrategy used by S3 objects created by New. -var DefaultAttemptStrategy = aws.AttemptStrategy{ - Min: 5, - Total: 5 * time.Second, - Delay: 200 * time.Millisecond, -} - -// New creates a new S3. Optional client argument allows for custom http.clients to be used. -func New(auth aws.Auth, region aws.Region, client ...*http.Client) *S3 { - - var httpclient *http.Client - - if len(client) > 0 { - httpclient = client[0] - } - - return &S3{Auth: auth, Region: region, AttemptStrategy: DefaultAttemptStrategy, client: httpclient} -} - -// Bucket returns a Bucket with the given name. -func (s3 *S3) Bucket(name string) *Bucket { - if s3.Region.S3BucketEndpoint != "" || s3.Region.S3LowercaseBucket { - name = strings.ToLower(name) - } - return &Bucket{s3, name} -} - -var createBucketConfiguration = `<CreateBucketConfiguration xmlns="http://s3.amazonaws.com/doc/2006-03-01/"> - <LocationConstraint>%s</LocationConstraint> -</CreateBucketConfiguration>` - -// locationConstraint returns an io.Reader specifying a LocationConstraint if -// required for the region. -// -// See http://goo.gl/bh9Kq for details. -func (s3 *S3) locationConstraint() io.Reader { - constraint := "" - if s3.Region.S3LocationConstraint { - constraint = fmt.Sprintf(createBucketConfiguration, s3.Region.Name) - } - return strings.NewReader(constraint) -} - -type ACL string - -const ( - Private = ACL("private") - PublicRead = ACL("public-read") - PublicReadWrite = ACL("public-read-write") - AuthenticatedRead = ACL("authenticated-read") - BucketOwnerRead = ACL("bucket-owner-read") - BucketOwnerFull = ACL("bucket-owner-full-control") -) - -// PutBucket creates a new bucket. -// -// See http://goo.gl/ndjnR for details. -func (b *Bucket) PutBucket(perm ACL) error { - headers := map[string][]string{ - "x-amz-acl": {string(perm)}, - } - req := &request{ - method: "PUT", - bucket: b.Name, - path: "/", - headers: headers, - payload: b.locationConstraint(), - } - return b.S3.query(req, nil) -} - -// DelBucket removes an existing S3 bucket. All objects in the bucket must -// be removed before the bucket itself can be removed. -// -// See http://goo.gl/GoBrY for details. -func (b *Bucket) DelBucket() (err error) { - req := &request{ - method: "DELETE", - bucket: b.Name, - path: "/", - } - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - err = b.S3.query(req, nil) - if !shouldRetry(err) { - break - } - } - return err -} - -// Get retrieves an object from an S3 bucket. -// -// See http://goo.gl/isCO7 for details. -func (b *Bucket) Get(path string) (data []byte, err error) { - body, err := b.GetReader(path) - defer func() { - if body != nil { - body.Close() - } - }() - if err != nil { - return nil, err - } - data, err = ioutil.ReadAll(body) - return data, err -} - -// GetReader retrieves an object from an S3 bucket, -// returning the body of the HTTP response. -// It is the caller's responsibility to call Close on rc when -// finished reading. -func (b *Bucket) GetReader(path string) (rc io.ReadCloser, err error) { - resp, err := b.GetResponse(path) - if resp != nil { - return resp.Body, err - } - return nil, err -} - -// GetResponse retrieves an object from an S3 bucket, -// returning the HTTP response. -// It is the caller's responsibility to call Close on rc when -// finished reading -func (b *Bucket) GetResponse(path string) (resp *http.Response, err error) { - return b.GetResponseWithHeaders(path, make(http.Header)) -} - -// GetReaderWithHeaders retrieves an object from an S3 bucket -// Accepts custom headers to be sent as the second parameter -// returning the body of the HTTP response. -// It is the caller's responsibility to call Close on rc when -// finished reading -func (b *Bucket) GetResponseWithHeaders(path string, headers map[string][]string) (resp *http.Response, err error) { - req := &request{ - bucket: b.Name, - path: path, - headers: headers, - } - err = b.S3.prepare(req) - if err != nil { - return nil, err - } - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - resp, err := b.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return nil, err - } - return resp, nil - } - panic("unreachable") -} - -// Exists checks whether or not an object exists on an S3 bucket using a HEAD request. -func (b *Bucket) Exists(path string) (exists bool, err error) { - req := &request{ - method: "HEAD", - bucket: b.Name, - path: path, - } - err = b.S3.prepare(req) - if err != nil { - return - } - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - resp, err := b.S3.run(req, nil) - - if shouldRetry(err) && attempt.HasNext() { - continue - } - - if err != nil { - // We can treat a 403 or 404 as non existance - if e, ok := err.(*Error); ok && (e.StatusCode == 403 || e.StatusCode == 404) { - return false, nil - } - return false, err - } - - if resp.StatusCode/100 == 2 { - exists = true - } - return exists, err - } - return false, fmt.Errorf("S3 Currently Unreachable") -} - -// Head HEADs an object in the S3 bucket, returns the response with -// no body see http://bit.ly/17K1ylI -func (b *Bucket) Head(path string, headers map[string][]string) (*http.Response, error) { - req := &request{ - method: "HEAD", - bucket: b.Name, - path: path, - headers: headers, - } - err := b.S3.prepare(req) - if err != nil { - return nil, err - } - - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - resp, err := b.S3.run(req, nil) - if shouldRetry(err) && attempt.HasNext() { - continue - } - if err != nil { - return nil, err - } - return resp, err - } - return nil, fmt.Errorf("S3 Currently Unreachable") -} - -// Put inserts an object into the S3 bucket. -// -// See http://goo.gl/FEBPD for details. -func (b *Bucket) Put(path string, data []byte, contType string, perm ACL, options Options) error { - body := bytes.NewBuffer(data) - return b.PutReader(path, body, int64(len(data)), contType, perm, options) -} - -// PutCopy puts a copy of an object given by the key path into bucket b using b.Path as the target key -func (b *Bucket) PutCopy(path string, perm ACL, options CopyOptions, source string) (result *CopyObjectResult, err error) { - headers := map[string][]string{ - "x-amz-acl": {string(perm)}, - "x-amz-copy-source": {source}, - } - options.addHeaders(headers) - req := &request{ - method: "PUT", - bucket: b.Name, - path: path, - headers: headers, - } - result = &CopyObjectResult{} - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - err = b.S3.query(req, result) - if !shouldRetry(err) { - break - } - } - if err != nil { - return nil, err - } - return result, nil -} - -/* -PutHeader - like Put, inserts an object into the S3 bucket. -Instead of Content-Type string, pass in custom headers to override defaults. -*/ -func (b *Bucket) PutHeader(path string, data []byte, customHeaders map[string][]string, perm ACL) error { - body := bytes.NewBuffer(data) - return b.PutReaderHeader(path, body, int64(len(data)), customHeaders, perm) -} - -// PutReader inserts an object into the S3 bucket by consuming data -// from r until EOF. -func (b *Bucket) PutReader(path string, r io.Reader, length int64, contType string, perm ACL, options Options) error { - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(length, 10)}, - "Content-Type": {contType}, - "x-amz-acl": {string(perm)}, - } - options.addHeaders(headers) - req := &request{ - method: "PUT", - bucket: b.Name, - path: path, - headers: headers, - payload: r, - } - return b.S3.query(req, nil) -} - -/* -PutReaderHeader - like PutReader, inserts an object into S3 from a reader. -Instead of Content-Type string, pass in custom headers to override defaults. -*/ -func (b *Bucket) PutReaderHeader(path string, r io.Reader, length int64, customHeaders map[string][]string, perm ACL) error { - // Default headers - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(length, 10)}, - "Content-Type": {"application/text"}, - "x-amz-acl": {string(perm)}, - } - - // Override with custom headers - for key, value := range customHeaders { - headers[key] = value - } - - req := &request{ - method: "PUT", - bucket: b.Name, - path: path, - headers: headers, - payload: r, - } - return b.S3.query(req, nil) -} - -// addHeaders adds o's specified fields to headers -func (o Options) addHeaders(headers map[string][]string) { - if o.SSE { - headers["x-amz-server-side-encryption"] = []string{"AES256"} - } - if len(o.ContentEncoding) != 0 { - headers["Content-Encoding"] = []string{o.ContentEncoding} - } - if len(o.CacheControl) != 0 { - headers["Cache-Control"] = []string{o.CacheControl} - } - if len(o.ContentMD5) != 0 { - headers["Content-MD5"] = []string{o.ContentMD5} - } - if len(o.RedirectLocation) != 0 { - headers["x-amz-website-redirect-location"] = []string{o.RedirectLocation} - } - for k, v := range o.Meta { - headers["x-amz-meta-"+k] = v - } -} - -// addHeaders adds o's specified fields to headers -func (o CopyOptions) addHeaders(headers map[string][]string) { - o.Options.addHeaders(headers) - if len(o.MetadataDirective) != 0 { - headers["x-amz-metadata-directive"] = []string{o.MetadataDirective} - } - if len(o.ContentType) != 0 { - headers["Content-Type"] = []string{o.ContentType} - } -} - -func makeXmlBuffer(doc []byte) *bytes.Buffer { - buf := new(bytes.Buffer) - buf.WriteString(xml.Header) - buf.Write(doc) - return buf -} - -type RoutingRule struct { - ConditionKeyPrefixEquals string `xml:"Condition>KeyPrefixEquals"` - RedirectReplaceKeyPrefixWith string `xml:"Redirect>ReplaceKeyPrefixWith,omitempty"` - RedirectReplaceKeyWith string `xml:"Redirect>ReplaceKeyWith,omitempty"` -} - -type WebsiteConfiguration struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ WebsiteConfiguration"` - IndexDocumentSuffix string `xml:"IndexDocument>Suffix"` - ErrorDocumentKey string `xml:"ErrorDocument>Key"` - RoutingRules *[]RoutingRule `xml:"RoutingRules>RoutingRule,omitempty"` -} - -func (b *Bucket) PutBucketWebsite(configuration WebsiteConfiguration) error { - - doc, err := xml.Marshal(configuration) - if err != nil { - return err - } - - buf := makeXmlBuffer(doc) - - return b.PutBucketSubresource("website", buf, int64(buf.Len())) -} - -func (b *Bucket) PutBucketSubresource(subresource string, r io.Reader, length int64) error { - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(length, 10)}, - } - req := &request{ - path: "/", - method: "PUT", - bucket: b.Name, - headers: headers, - payload: r, - params: url.Values{subresource: {""}}, - } - - return b.S3.query(req, nil) -} - -// Del removes an object from the S3 bucket. -// -// See http://goo.gl/APeTt for details. -func (b *Bucket) Del(path string) error { - req := &request{ - method: "DELETE", - bucket: b.Name, - path: path, - } - return b.S3.query(req, nil) -} - -type Delete struct { - Quiet bool `xml:"Quiet,omitempty"` - Objects []Object `xml:"Object"` -} - -type Object struct { - Key string `xml:"Key"` - VersionId string `xml:"VersionId,omitempty"` -} - -// DelMulti removes up to 1000 objects from the S3 bucket. -// -// See http://goo.gl/jx6cWK for details. -func (b *Bucket) DelMulti(objects Delete) error { - doc, err := xml.Marshal(objects) - if err != nil { - return err - } - - buf := makeXmlBuffer(doc) - digest := md5.New() - size, err := digest.Write(buf.Bytes()) - if err != nil { - return err - } - - headers := map[string][]string{ - "Content-Length": {strconv.FormatInt(int64(size), 10)}, - "Content-MD5": {base64.StdEncoding.EncodeToString(digest.Sum(nil))}, - "Content-Type": {"text/xml"}, - } - req := &request{ - path: "/", - method: "POST", - params: url.Values{"delete": {""}}, - bucket: b.Name, - headers: headers, - payload: buf, - } - - return b.S3.query(req, nil) -} - -// The ListResp type holds the results of a List bucket operation. -type ListResp struct { - Name string - Prefix string - Delimiter string - Marker string - NextMarker string - MaxKeys int - - // IsTruncated is true if the results have been truncated because - // there are more keys and prefixes than can fit in MaxKeys. - // N.B. this is the opposite sense to that documented (incorrectly) in - // http://goo.gl/YjQTc - IsTruncated bool - Contents []Key - CommonPrefixes []string `xml:">Prefix"` -} - -// The Key type represents an item stored in an S3 bucket. -type Key struct { - Key string - LastModified string - Size int64 - // ETag gives the hex-encoded MD5 sum of the contents, - // surrounded with double-quotes. - ETag string - StorageClass string - Owner Owner -} - -// List returns information about objects in an S3 bucket. -// -// The prefix parameter limits the response to keys that begin with the -// specified prefix. -// -// The delim parameter causes the response to group all of the keys that -// share a common prefix up to the next delimiter in a single entry within -// the CommonPrefixes field. You can use delimiters to separate a bucket -// into different groupings of keys, similar to how folders would work. -// -// The marker parameter specifies the key to start with when listing objects -// in a bucket. Amazon S3 lists objects in alphabetical order and -// will return keys alphabetically greater than the marker. -// -// The max parameter specifies how many keys + common prefixes to return in -// the response. The default is 1000. -// -// For example, given these keys in a bucket: -// -// index.html -// index2.html -// photos/2006/January/sample.jpg -// photos/2006/February/sample2.jpg -// photos/2006/February/sample3.jpg -// photos/2006/February/sample4.jpg -// -// Listing this bucket with delimiter set to "/" would yield the -// following result: -// -// &ListResp{ -// Name: "sample-bucket", -// MaxKeys: 1000, -// Delimiter: "/", -// Contents: []Key{ -// {Key: "index.html", "index2.html"}, -// }, -// CommonPrefixes: []string{ -// "photos/", -// }, -// } -// -// Listing the same bucket with delimiter set to "/" and prefix set to -// "photos/2006/" would yield the following result: -// -// &ListResp{ -// Name: "sample-bucket", -// MaxKeys: 1000, -// Delimiter: "/", -// Prefix: "photos/2006/", -// CommonPrefixes: []string{ -// "photos/2006/February/", -// "photos/2006/January/", -// }, -// } -// -// See http://goo.gl/YjQTc for details. -func (b *Bucket) List(prefix, delim, marker string, max int) (result *ListResp, err error) { - params := map[string][]string{ - "prefix": {prefix}, - "delimiter": {delim}, - "marker": {marker}, - } - if max != 0 { - params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} - } - req := &request{ - bucket: b.Name, - params: params, - } - result = &ListResp{} - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - err = b.S3.query(req, result) - if !shouldRetry(err) { - break - } - } - if err != nil { - return nil, err - } - return result, nil -} - -// The VersionsResp type holds the results of a list bucket Versions operation. -type VersionsResp struct { - Name string - Prefix string - KeyMarker string - VersionIdMarker string - MaxKeys int - Delimiter string - IsTruncated bool - Versions []Version - CommonPrefixes []string `xml:">Prefix"` -} - -// The Version type represents an object version stored in an S3 bucket. -type Version struct { - Key string - VersionId string - IsLatest bool - LastModified string - // ETag gives the hex-encoded MD5 sum of the contents, - // surrounded with double-quotes. - ETag string - Size int64 - Owner Owner - StorageClass string -} - -func (b *Bucket) Versions(prefix, delim, keyMarker string, versionIdMarker string, max int) (result *VersionsResp, err error) { - params := map[string][]string{ - "versions": {""}, - "prefix": {prefix}, - "delimiter": {delim}, - } - - if len(versionIdMarker) != 0 { - params["version-id-marker"] = []string{versionIdMarker} - } - if len(keyMarker) != 0 { - params["key-marker"] = []string{keyMarker} - } - - if max != 0 { - params["max-keys"] = []string{strconv.FormatInt(int64(max), 10)} - } - req := &request{ - bucket: b.Name, - params: params, - } - result = &VersionsResp{} - for attempt := b.S3.AttemptStrategy.Start(); attempt.Next(); { - err = b.S3.query(req, result) - if !shouldRetry(err) { - break - } - } - if err != nil { - return nil, err - } - return result, nil -} - -// Returns a mapping of all key names in this bucket to Key objects -func (b *Bucket) GetBucketContents() (*map[string]Key, error) { - bucket_contents := map[string]Key{} - prefix := "" - path_separator := "" - marker := "" - for { - contents, err := b.List(prefix, path_separator, marker, 1000) - if err != nil { - return &bucket_contents, err - } - for _, key := range contents.Contents { - bucket_contents[key.Key] = key - } - if contents.IsTruncated { - marker = contents.NextMarker - } else { - break - } - } - - return &bucket_contents, nil -} - -// URL returns a non-signed URL that allows retriving the -// object at path. It only works if the object is publicly -// readable (see SignedURL). -func (b *Bucket) URL(path string) string { - req := &request{ - bucket: b.Name, - path: path, - } - err := b.S3.prepare(req) - if err != nil { - panic(err) - } - u, err := req.url() - if err != nil { - panic(err) - } - u.RawQuery = "" - return u.String() -} - -// SignedURL returns a signed URL that allows anyone holding the URL -// to retrieve the object at path. The signature is valid until expires. -func (b *Bucket) SignedURL(path string, expires time.Time) string { - req := &request{ - bucket: b.Name, - path: path, - params: url.Values{"Expires": {strconv.FormatInt(expires.Unix(), 10)}}, - } - err := b.S3.prepare(req) - if err != nil { - panic(err) - } - u, err := req.url() - if err != nil { - panic(err) - } - if b.S3.Auth.Token() != "" { - return u.String() + "&x-amz-security-token=" + url.QueryEscape(req.headers["X-Amz-Security-Token"][0]) - } else { - return u.String() - } -} - -// UploadSignedURL returns a signed URL that allows anyone holding the URL -// to upload the object at path. The signature is valid until expires. -// contenttype is a string like image/png -// path is the resource name in s3 terminalogy like images/ali.png [obviously exclusing the bucket name itself] -func (b *Bucket) UploadSignedURL(path, method, content_type string, expires time.Time) string { - expire_date := expires.Unix() - if method != "POST" { - method = "PUT" - } - stringToSign := method + "\n\n" + content_type + "\n" + strconv.FormatInt(expire_date, 10) + "\n/" + b.Name + "/" + path - fmt.Println("String to sign:\n", stringToSign) - a := b.S3.Auth - secretKey := a.SecretKey - accessId := a.AccessKey - mac := hmac.New(sha1.New, []byte(secretKey)) - mac.Write([]byte(stringToSign)) - macsum := mac.Sum(nil) - signature := base64.StdEncoding.EncodeToString([]byte(macsum)) - signature = strings.TrimSpace(signature) - - signedurl, err := url.Parse("https://" + b.Name + ".s3.amazonaws.com/") - if err != nil { - log.Println("ERROR sining url for S3 upload", err) - return "" - } - signedurl.Path += path - params := url.Values{} - params.Add("AWSAccessKeyId", accessId) - params.Add("Expires", strconv.FormatInt(expire_date, 10)) - params.Add("Signature", signature) - if a.Token() != "" { - params.Add("token", a.Token()) - } - - signedurl.RawQuery = params.Encode() - return signedurl.String() -} - -// PostFormArgs returns the action and input fields needed to allow anonymous -// uploads to a bucket within the expiration limit -func (b *Bucket) PostFormArgs(path string, expires time.Time, redirect string) (action string, fields map[string]string) { - conditions := make([]string, 0) - fields = map[string]string{ - "AWSAccessKeyId": b.Auth.AccessKey, - "key": path, - } - - conditions = append(conditions, fmt.Sprintf("{\"key\": \"%s\"}", path)) - conditions = append(conditions, fmt.Sprintf("{\"bucket\": \"%s\"}", b.Name)) - if redirect != "" { - conditions = append(conditions, fmt.Sprintf("{\"success_action_redirect\": \"%s\"}", redirect)) - fields["success_action_redirect"] = redirect - } - - vExpiration := expires.Format("2006-01-02T15:04:05Z") - vConditions := strings.Join(conditions, ",") - policy := fmt.Sprintf("{\"expiration\": \"%s\", \"conditions\": [%s]}", vExpiration, vConditions) - policy64 := base64.StdEncoding.EncodeToString([]byte(policy)) - fields["policy"] = policy64 - - signer := hmac.New(sha1.New, []byte(b.Auth.SecretKey)) - signer.Write([]byte(policy64)) - fields["signature"] = base64.StdEncoding.EncodeToString(signer.Sum(nil)) - - action = fmt.Sprintf("%s/%s/", b.S3.Region.S3Endpoint, b.Name) - return -} - -type request struct { - method string - bucket string - path string - signpath string - params url.Values - headers http.Header - baseurl string - payload io.Reader - prepared bool -} - -func (req *request) url() (*url.URL, error) { - u, err := url.Parse(req.baseurl) - if err != nil { - return nil, fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) - } - u.RawQuery = req.params.Encode() - u.Path = req.path - return u, nil -} - -// query prepares and runs the req request. -// If resp is not nil, the XML data contained in the response -// body will be unmarshalled on it. -func (s3 *S3) query(req *request, resp interface{}) error { - err := s3.prepare(req) - if err == nil { - var httpResponse *http.Response - httpResponse, err = s3.run(req, resp) - if resp == nil && httpResponse != nil { - httpResponse.Body.Close() - } - } - return err -} - -// prepare sets up req to be delivered to S3. -func (s3 *S3) prepare(req *request) error { - var signpath = req.path - - if !req.prepared { - req.prepared = true - if req.method == "" { - req.method = "GET" - } - // Copy so they can be mutated without affecting on retries. - params := make(url.Values) - headers := make(http.Header) - for k, v := range req.params { - params[k] = v - } - for k, v := range req.headers { - headers[k] = v - } - req.params = params - req.headers = headers - if !strings.HasPrefix(req.path, "/") { - req.path = "/" + req.path - } - signpath = req.path - if req.bucket != "" { - req.baseurl = s3.Region.S3BucketEndpoint - if req.baseurl == "" { - // Use the path method to address the bucket. - req.baseurl = s3.Region.S3Endpoint - req.path = "/" + req.bucket + req.path - } else { - // Just in case, prevent injection. - if strings.IndexAny(req.bucket, "/:@") >= 0 { - return fmt.Errorf("bad S3 bucket: %q", req.bucket) - } - req.baseurl = strings.Replace(req.baseurl, "${bucket}", req.bucket, -1) - } - signpath = "/" + req.bucket + signpath - } - } - - // Always sign again as it's not clear how far the - // server has handled a previous attempt. - u, err := url.Parse(req.baseurl) - if err != nil { - return fmt.Errorf("bad S3 endpoint URL %q: %v", req.baseurl, err) - } - reqSignpathSpaceFix := (&url.URL{Path: signpath}).String() - req.headers["Host"] = []string{u.Host} - req.headers["Date"] = []string{time.Now().In(time.UTC).Format(time.RFC1123)} - if s3.Auth.Token() != "" { - req.headers["X-Amz-Security-Token"] = []string{s3.Auth.Token()} - } - sign(s3.Auth, req.method, reqSignpathSpaceFix, req.params, req.headers) - return nil -} - -// run sends req and returns the http response from the server. -// If resp is not nil, the XML data contained in the response -// body will be unmarshalled on it. -func (s3 *S3) run(req *request, resp interface{}) (*http.Response, error) { - if debug { - log.Printf("Running S3 request: %#v", req) - } - - u, err := req.url() - if err != nil { - return nil, err - } - - hreq := http.Request{ - URL: u, - Method: req.method, - ProtoMajor: 1, - ProtoMinor: 1, - Close: true, - Header: req.headers, - } - - if v, ok := req.headers["Content-Length"]; ok { - hreq.ContentLength, _ = strconv.ParseInt(v[0], 10, 64) - delete(req.headers, "Content-Length") - if hreq.ContentLength == 0 { - req.payload = nil - } - } - if req.payload != nil { - hreq.Body = ioutil.NopCloser(req.payload) - } - - if s3.client == nil { - s3.client = &http.Client{ - Transport: &http.Transport{ - Dial: func(netw, addr string) (c net.Conn, err error) { - c, err = net.DialTimeout(netw, addr, s3.ConnectTimeout) - if err != nil { - return - } - - var deadline time.Time - if s3.RequestTimeout > 0 { - deadline = time.Now().Add(s3.RequestTimeout) - c.SetDeadline(deadline) - } - - if s3.ReadTimeout > 0 || s3.WriteTimeout > 0 { - c = &ioTimeoutConn{ - TCPConn: c.(*net.TCPConn), - readTimeout: s3.ReadTimeout, - writeTimeout: s3.WriteTimeout, - requestDeadline: deadline, - } - } - return - }, - }, - } - } - - hresp, err := s3.client.Do(&hreq) - if err != nil { - return nil, err - } - if debug { - dump, _ := httputil.DumpResponse(hresp, true) - log.Printf("} -> %s\n", dump) - } - if hresp.StatusCode != 200 && hresp.StatusCode != 204 && hresp.StatusCode != 206 { - defer hresp.Body.Close() - return nil, buildError(hresp) - } - if resp != nil { - err = xml.NewDecoder(hresp.Body).Decode(resp) - hresp.Body.Close() - if debug { - log.Printf("goamz.s3> decoded xml into %#v", resp) - } - } - return hresp, err -} - -// Error represents an error in an operation with S3. -type Error struct { - StatusCode int // HTTP status code (200, 403, ...) - Code string // EC2 error code ("UnsupportedOperation", ...) - Message string // The human-oriented error message - BucketName string - RequestId string - HostId string -} - -func (e *Error) Error() string { - return e.Message -} - -func buildError(r *http.Response) error { - if debug { - log.Printf("got error (status code %v)", r.StatusCode) - data, err := ioutil.ReadAll(r.Body) - if err != nil { - log.Printf("\tread error: %v", err) - } else { - log.Printf("\tdata:\n%s\n\n", data) - } - r.Body = ioutil.NopCloser(bytes.NewBuffer(data)) - } - - err := Error{} - // TODO return error if Unmarshal fails? - xml.NewDecoder(r.Body).Decode(&err) - r.Body.Close() - err.StatusCode = r.StatusCode - if err.Message == "" { - err.Message = r.Status - } - if debug { - log.Printf("err: %#v\n", err) - } - return &err -} - -func shouldRetry(err error) bool { - if err == nil { - return false - } - if e, ok := err.(*url.Error); ok { - // Transport returns this string if it detects a write on a connection which - // has already had an error - if e.Err.Error() == "http: can't write HTTP request on broken connection" { - return true - } - err = e.Err - } - - switch err { - case io.ErrUnexpectedEOF, io.EOF: - return true - } - switch e := err.(type) { - case *net.DNSError: - return true - case *net.OpError: - switch e.Op { - case "read", "write", "WSARecv", "WSASend", "ConnectEx": - return true - } - case *Error: - switch e.Code { - case "InternalError", "NoSuchUpload", "NoSuchBucket", "RequestTimeout": - return true - } - // let's handle tls handshake timeout issues and similar temporary errors - case net.Error: - return e.Temporary() - } - - return false -} - -func hasCode(err error, code string) bool { - s3err, ok := err.(*Error) - return ok && s3err.Code == code -} - -// ioTimeoutConn is a net.Conn which sets a deadline for each Read or Write operation -type ioTimeoutConn struct { - *net.TCPConn - readTimeout time.Duration - writeTimeout time.Duration - requestDeadline time.Time -} - -func (c *ioTimeoutConn) deadline(timeout time.Duration) time.Time { - dl := time.Now().Add(timeout) - if c.requestDeadline.IsZero() || dl.Before(c.requestDeadline) { - return dl - } - - return c.requestDeadline -} - -func (c *ioTimeoutConn) Read(b []byte) (int, error) { - if c.readTimeout > 0 { - err := c.TCPConn.SetReadDeadline(c.deadline(c.readTimeout)) - if err != nil { - return 0, err - } - } - return c.TCPConn.Read(b) -} - -func (c *ioTimeoutConn) Write(b []byte) (int, error) { - if c.writeTimeout > 0 { - err := c.TCPConn.SetWriteDeadline(c.deadline(c.writeTimeout)) - if err != nil { - return 0, err - } - } - return c.TCPConn.Write(b) -} |