diff options
Diffstat (limited to 'vendor/github.com/minio/minio-go/pkg/s3utils')
| -rw-r--r-- | vendor/github.com/minio/minio-go/pkg/s3utils/utils.go | 195 | ||||
| -rw-r--r-- | vendor/github.com/minio/minio-go/pkg/s3utils/utils_test.go | 284 |
2 files changed, 479 insertions, 0 deletions
diff --git a/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go b/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go new file mode 100644 index 000000000..ae1cea337 --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/s3utils/utils.go @@ -0,0 +1,195 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3utils + +import ( + "bytes" + "encoding/hex" + "net" + "net/url" + "regexp" + "sort" + "strings" + "unicode/utf8" +) + +// Sentinel URL is the default url value which is invalid. +var sentinelURL = url.URL{} + +// IsValidDomain validates if input string is a valid domain name. +func IsValidDomain(host string) bool { + // See RFC 1035, RFC 3696. + host = strings.TrimSpace(host) + if len(host) == 0 || len(host) > 255 { + return false + } + // host cannot start or end with "-" + if host[len(host)-1:] == "-" || host[:1] == "-" { + return false + } + // host cannot start or end with "_" + if host[len(host)-1:] == "_" || host[:1] == "_" { + return false + } + // host cannot start or end with a "." + if host[len(host)-1:] == "." || host[:1] == "." { + return false + } + // All non alphanumeric characters are invalid. + if strings.ContainsAny(host, "`~!@#$%^&*()+={}[]|\\\"';:><?/") { + return false + } + // No need to regexp match, since the list is non-exhaustive. + // We let it valid and fail later. + return true +} + +// IsValidIP parses input string for ip address validity. +func IsValidIP(ip string) bool { + return net.ParseIP(ip) != nil +} + +// IsVirtualHostSupported - verifies if bucketName can be part of +// virtual host. Currently only Amazon S3 and Google Cloud Storage +// would support this. +func IsVirtualHostSupported(endpointURL url.URL, bucketName string) bool { + if endpointURL == sentinelURL { + return false + } + // bucketName can be valid but '.' in the hostname will fail SSL + // certificate validation. So do not use host-style for such buckets. + if endpointURL.Scheme == "https" && strings.Contains(bucketName, ".") { + return false + } + // Return true for all other cases + return IsAmazonEndpoint(endpointURL) || IsGoogleEndpoint(endpointURL) +} + +// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint. +func IsAmazonEndpoint(endpointURL url.URL) bool { + if IsAmazonChinaEndpoint(endpointURL) { + return true + } + + if IsAmazonS3AccelerateEndpoint(endpointURL) { + return true + } + + return endpointURL.Host == "s3.amazonaws.com" +} + +// IsAmazonChinaEndpoint - Match if it is exactly Amazon S3 China endpoint. +// Customers who wish to use the new Beijing Region are required +// to sign up for a separate set of account credentials unique to +// the China (Beijing) Region. Customers with existing AWS credentials +// will not be able to access resources in the new Region, and vice versa. +// For more info https://aws.amazon.com/about-aws/whats-new/2013/12/18/announcing-the-aws-china-beijing-region/ +func IsAmazonChinaEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return endpointURL.Host == "s3.cn-north-1.amazonaws.com.cn" +} + +// IsAmazonS3AccelerateEndpoint - Match if it is an Amazon S3 Accelerate +func IsAmazonS3AccelerateEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return endpointURL.Host == "s3-accelerate.amazonaws.com" +} + +// IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint. +func IsGoogleEndpoint(endpointURL url.URL) bool { + if endpointURL == sentinelURL { + return false + } + return endpointURL.Host == "storage.googleapis.com" +} + +// Expects ascii encoded strings - from output of urlEncodePath +func percentEncodeSlash(s string) string { + return strings.Replace(s, "/", "%2F", -1) +} + +// QueryEncode - encodes query values in their URL encoded form. In +// addition to the percent encoding performed by urlEncodePath() used +// here, it also percent encodes '/' (forward slash) +func QueryEncode(v url.Values) string { + if v == nil { + return "" + } + var buf bytes.Buffer + keys := make([]string, 0, len(v)) + for k := range v { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + vs := v[k] + prefix := percentEncodeSlash(EncodePath(k)) + "=" + for _, v := range vs { + if buf.Len() > 0 { + buf.WriteByte('&') + } + buf.WriteString(prefix) + buf.WriteString(percentEncodeSlash(EncodePath(v))) + } + } + return buf.String() +} + +// if object matches reserved string, no need to encode them +var reservedObjectNames = regexp.MustCompile("^[a-zA-Z0-9-_.~/]+$") + +// EncodePath encode the strings from UTF-8 byte representations to HTML hex escape sequences +// +// This is necessary since regular url.Parse() and url.Encode() functions do not support UTF-8 +// non english characters cannot be parsed due to the nature in which url.Encode() is written +// +// This function on the other hand is a direct replacement for url.Encode() technique to support +// pretty much every UTF-8 character. +func EncodePath(pathName string) string { + if reservedObjectNames.MatchString(pathName) { + return pathName + } + var encodedPathname string + for _, s := range pathName { + if 'A' <= s && s <= 'Z' || 'a' <= s && s <= 'z' || '0' <= s && s <= '9' { // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + } + switch s { + case '-', '_', '.', '~', '/': // §2.3 Unreserved characters (mark) + encodedPathname = encodedPathname + string(s) + continue + default: + len := utf8.RuneLen(s) + if len < 0 { + // if utf8 cannot convert return the same string as is + return pathName + } + u := make([]byte, len) + utf8.EncodeRune(u, s) + for _, r := range u { + hex := hex.EncodeToString([]byte{r}) + encodedPathname = encodedPathname + "%" + strings.ToUpper(hex) + } + } + } + return encodedPathname +} diff --git a/vendor/github.com/minio/minio-go/pkg/s3utils/utils_test.go b/vendor/github.com/minio/minio-go/pkg/s3utils/utils_test.go new file mode 100644 index 000000000..f790861cd --- /dev/null +++ b/vendor/github.com/minio/minio-go/pkg/s3utils/utils_test.go @@ -0,0 +1,284 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package s3utils + +import ( + "net/url" + "testing" +) + +// Tests for 'isValidDomain(host string) bool'. +func TestIsValidDomain(t *testing.T) { + testCases := []struct { + // Input. + host string + // Expected result. + result bool + }{ + {"s3.amazonaws.com", true}, + {"s3.cn-north-1.amazonaws.com.cn", true}, + {"s3.amazonaws.com_", false}, + {"%$$$", false}, + {"s3.amz.test.com", true}, + {"s3.%%", false}, + {"localhost", true}, + {"-localhost", false}, + {"", false}, + {"\n \t", false}, + {" ", false}, + } + + for i, testCase := range testCases { + result := IsValidDomain(testCase.host) + if testCase.result != result { + t.Errorf("Test %d: Expected isValidDomain test to be '%v', but found '%v' instead", i+1, testCase.result, result) + } + } +} + +// Tests validate IP address validator. +func TestIsValidIP(t *testing.T) { + testCases := []struct { + // Input. + ip string + // Expected result. + result bool + }{ + {"192.168.1.1", true}, + {"192.168.1", false}, + {"192.168.1.1.1", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + } + + for i, testCase := range testCases { + result := IsValidIP(testCase.ip) + if testCase.result != result { + t.Errorf("Test %d: Expected isValidIP to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.ip, result) + } + } + +} + +// Tests validate virtual host validator. +func TestIsVirtualHostSupported(t *testing.T) { + testCases := []struct { + url string + bucket string + // Expeceted result. + result bool + }{ + {"https://s3.amazonaws.com", "my-bucket", true}, + {"https://s3.cn-north-1.amazonaws.com.cn", "my-bucket", true}, + {"https://s3.amazonaws.com", "my-bucket.", false}, + {"https://amazons3.amazonaws.com", "my-bucket.", false}, + {"https://storage.googleapis.com/", "my-bucket", true}, + {"https://mystorage.googleapis.com/", "my-bucket", false}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err) + } + result := IsVirtualHostSupported(*u, testCase.bucket) + if testCase.result != result { + t.Errorf("Test %d: Expected isVirtualHostSupported to be '%v' for input url \"%s\" and bucket \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, testCase.bucket, result) + } + } +} + +// Tests validate Amazon endpoint validator. +func TestIsAmazonEndpoint(t *testing.T) { + testCases := []struct { + url string + // Expected result. + result bool + }{ + {"https://192.168.1.1", false}, + {"192.168.1.1", false}, + {"http://storage.googleapis.com", false}, + {"https://storage.googleapis.com", false}, + {"storage.googleapis.com", false}, + {"s3.amazonaws.com", false}, + {"https://amazons3.amazonaws.com", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + // valid inputs. + {"https://s3.amazonaws.com", true}, + {"https://s3.cn-north-1.amazonaws.com.cn", true}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err) + } + result := IsAmazonEndpoint(*u) + if testCase.result != result { + t.Errorf("Test %d: Expected isAmazonEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) + } + } + +} + +// Tests validate Amazon S3 China endpoint validator. +func TestIsAmazonChinaEndpoint(t *testing.T) { + testCases := []struct { + url string + // Expected result. + result bool + }{ + {"https://192.168.1.1", false}, + {"192.168.1.1", false}, + {"http://storage.googleapis.com", false}, + {"https://storage.googleapis.com", false}, + {"storage.googleapis.com", false}, + {"s3.amazonaws.com", false}, + {"https://amazons3.amazonaws.com", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + // s3.amazonaws.com is not a valid Amazon S3 China end point. + {"https://s3.amazonaws.com", false}, + // valid input. + {"https://s3.cn-north-1.amazonaws.com.cn", true}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err) + } + result := IsAmazonChinaEndpoint(*u) + if testCase.result != result { + t.Errorf("Test %d: Expected isAmazonEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) + } + } + +} + +// Tests validate Google Cloud end point validator. +func TestIsGoogleEndpoint(t *testing.T) { + testCases := []struct { + url string + // Expected result. + result bool + }{ + {"192.168.1.1", false}, + {"https://192.168.1.1", false}, + {"s3.amazonaws.com", false}, + {"http://s3.amazonaws.com", false}, + {"https://s3.amazonaws.com", false}, + {"https://s3.cn-north-1.amazonaws.com.cn", false}, + {"-192.168.1.1", false}, + {"260.192.1.1", false}, + // valid inputs. + {"http://storage.googleapis.com", true}, + {"https://storage.googleapis.com", true}, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.url) + if err != nil { + t.Errorf("Test %d: Expected to pass, but failed with: <ERROR> %s", i+1, err) + } + result := IsGoogleEndpoint(*u) + if testCase.result != result { + t.Errorf("Test %d: Expected isGoogleEndpoint to be '%v' for input \"%s\", but found it to be '%v' instead", i+1, testCase.result, testCase.url, result) + } + } + +} + +func TestPercentEncodeSlash(t *testing.T) { + testCases := []struct { + input string + output string + }{ + {"test123", "test123"}, + {"abc,+_1", "abc,+_1"}, + {"%40prefix=test%40123", "%40prefix=test%40123"}, + {"key1=val1/val2", "key1=val1%2Fval2"}, + {"%40prefix=test%40123/", "%40prefix=test%40123%2F"}, + } + + for i, testCase := range testCases { + receivedOutput := percentEncodeSlash(testCase.input) + if testCase.output != receivedOutput { + t.Errorf( + "Test %d: Input: \"%s\" --> Expected percentEncodeSlash to return \"%s\", but it returned \"%s\" instead!", + i+1, testCase.input, testCase.output, + receivedOutput, + ) + + } + } +} + +// Tests validate the query encoder. +func TestQueryEncode(t *testing.T) { + testCases := []struct { + queryKey string + valueToEncode []string + // Expected result. + result string + }{ + {"prefix", []string{"test@123", "test@456"}, "prefix=test%40123&prefix=test%40456"}, + {"@prefix", []string{"test@123"}, "%40prefix=test%40123"}, + {"@prefix", []string{"a/b/c/"}, "%40prefix=a%2Fb%2Fc%2F"}, + {"prefix", []string{"test#123"}, "prefix=test%23123"}, + {"prefix#", []string{"test#123"}, "prefix%23=test%23123"}, + {"prefix", []string{"test123"}, "prefix=test123"}, + {"prefix", []string{"test本語123", "test123"}, "prefix=test%E6%9C%AC%E8%AA%9E123&prefix=test123"}, + } + + for i, testCase := range testCases { + urlValues := make(url.Values) + for _, valueToEncode := range testCase.valueToEncode { + urlValues.Add(testCase.queryKey, valueToEncode) + } + result := QueryEncode(urlValues) + if testCase.result != result { + t.Errorf("Test %d: Expected queryEncode result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) + } + } +} + +// Tests validate the URL path encoder. +func TestEncodePath(t *testing.T) { + testCases := []struct { + // Input. + inputStr string + // Expected result. + result string + }{ + {"thisisthe%url", "thisisthe%25url"}, + {"本語", "%E6%9C%AC%E8%AA%9E"}, + {"本語.1", "%E6%9C%AC%E8%AA%9E.1"}, + {">123", "%3E123"}, + {"myurl#link", "myurl%23link"}, + {"space in url", "space%20in%20url"}, + {"url+path", "url%2Bpath"}, + } + + for i, testCase := range testCases { + result := EncodePath(testCase.inputStr) + if testCase.result != result { + t.Errorf("Test %d: Expected queryEncode result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) + } + } +} |