diff options
Diffstat (limited to 'vendor/github.com/minio/minio-go/api_functional_v4_test.go')
-rw-r--r-- | vendor/github.com/minio/minio-go/api_functional_v4_test.go | 510 |
1 files changed, 447 insertions, 63 deletions
diff --git a/vendor/github.com/minio/minio-go/api_functional_v4_test.go b/vendor/github.com/minio/minio-go/api_functional_v4_test.go index 426d2ddcc..a553ea2cd 100644 --- a/vendor/github.com/minio/minio-go/api_functional_v4_test.go +++ b/vendor/github.com/minio/minio-go/api_functional_v4_test.go @@ -18,7 +18,7 @@ package minio import ( "bytes" - crand "crypto/rand" + "encoding/hex" "errors" "fmt" "io" @@ -28,9 +28,11 @@ import ( "net/url" "os" "strconv" + "strings" "testing" "time" + "github.com/minio/minio-go/pkg/encrypt" "github.com/minio/minio-go/pkg/policy" ) @@ -64,6 +66,9 @@ func TestMakeBucketError(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for short runs") } + if os.Getenv("S3_ADDRESS") != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -110,6 +115,9 @@ func TestMakeBucketRegions(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for short runs") } + if os.Getenv("S3_ADDRESS") != "s3.amazonaws.com" { + t.Skip("skipping region functional tests for non s3 runs") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -192,14 +200,10 @@ func TestPutObjectReadAt(t *testing.T) { } // Generate data using 4 parts so that all 3 'workers' are utilized and a part is leftover. - buf := make([]byte, minPartSize*4) - // Use crand.Reader for multipart tests to ensure part order at the end. - size, err := io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) + // Use different data for each part for multipart tests to ensure part order at the end. + var buf []byte + for i := 0; i < 4; i++ { + buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } // Save the data @@ -286,14 +290,10 @@ func TestPutObjectWithMetadata(t *testing.T) { } // Generate data using 2 parts - buf := make([]byte, minPartSize*2) - // Use crand.Reader for multipart tests to ensure part order at the end. - size, err := io.ReadFull(crand.Reader, buf) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*2 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*2, size) + // Use different data in each part for multipart tests to ensure part order at the end. + var buf []byte + for i := 0; i < 2; i++ { + buf = append(buf, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } // Save the data @@ -302,7 +302,7 @@ func TestPutObjectWithMetadata(t *testing.T) { // Object custom metadata customContentType := "custom/contenttype" - n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{"Content-Type": []string{customContentType}}, nil) + n, err := c.PutObjectWithMetadata(bucketName, objectName, bytes.NewReader(buf), map[string][]string{"Content-Type": {customContentType}}, nil) if err != nil { t.Fatal("Error:", err, bucketName, objectName) } @@ -346,6 +346,70 @@ func TestPutObjectWithMetadata(t *testing.T) { } } +// Test put object with streaming signature. +func TestPutObjectStreaming(t *testing.T) { + if testing.Short() { + t.Skip("skipping function tests for short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), + "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Upload an object. + sizes := []int64{0, 64*1024 - 1, 64 * 1024} + objectName := "test-object" + for i, size := range sizes { + data := bytes.Repeat([]byte("a"), int(size)) + n, err := c.PutObjectStreaming(bucketName, objectName, bytes.NewReader(data)) + if err != nil { + t.Fatalf("Test %d Error: %v %s %s", i+1, err, bucketName, objectName) + } + + if n != size { + t.Errorf("Test %d Expected upload object size %d but got %d", i+1, size, n) + } + } + + // Remove the object. + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + // Remove the bucket. + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + // Test listing partially uploaded objects. func TestListPartiallyUploaded(t *testing.T) { if testing.Short() { @@ -387,17 +451,14 @@ func TestListPartiallyUploaded(t *testing.T) { go func() { i := 0 for i < 25 { - _, err = io.CopyN(writer, r, (minPartSize*2)/25) - if err != nil { - t.Fatal("Error:", err, bucketName) + _, cerr := io.CopyN(writer, r, (minPartSize*2)/25) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) } i++ r.Seek(0, 0) } - err := writer.CloseWithError(errors.New("Proactively closed to be verified later.")) - if err != nil { - t.Fatal("Error:", err) - } + writer.CloseWithError(errors.New("proactively closed to be verified later")) }() objectName := bucketName + "-resumable" @@ -405,7 +466,7 @@ func TestListPartiallyUploaded(t *testing.T) { if err == nil { t.Fatal("Error: PutObject should fail.") } - if err.Error() != "Proactively closed to be verified later." { + if err.Error() != "proactively closed to be verified later" { t.Fatal("Error:", err) } @@ -650,7 +711,8 @@ func TestRemoveMultipleObjects(t *testing.T) { objectName := "sample" + strconv.Itoa(i) + ".txt" _, err = c.PutObject(bucketName, objectName, r, "application/octet-stream") if err != nil { - t.Fatal("Error: PutObject shouldn't fail.") + t.Error("Error: PutObject shouldn't fail.", err) + continue } objectsCh <- objectName } @@ -715,17 +777,14 @@ func TestRemovePartiallyUploaded(t *testing.T) { go func() { i := 0 for i < 25 { - _, err = io.CopyN(writer, r, 128*1024) - if err != nil { - t.Fatal("Error:", err, bucketName) + _, cerr := io.CopyN(writer, r, 128*1024) + if cerr != nil { + t.Fatal("Error:", cerr, bucketName) } i++ r.Seek(0, 0) } - err := writer.CloseWithError(errors.New("Proactively closed to be verified later.")) - if err != nil { - t.Fatal("Error:", err) - } + writer.CloseWithError(errors.New("proactively closed to be verified later")) }() objectName := bucketName + "-resumable" @@ -733,7 +792,7 @@ func TestRemovePartiallyUploaded(t *testing.T) { if err == nil { t.Fatal("Error: PutObject should fail.") } - if err.Error() != "Proactively closed to be verified later." { + if err.Error() != "proactively closed to be verified later" { t.Fatal("Error:", err) } err = c.RemoveIncompleteUpload(bucketName, objectName) @@ -788,7 +847,6 @@ func TestResumablePutObject(t *testing.T) { t.Fatal("Error:", err) } r := bytes.NewReader(bytes.Repeat([]byte("b"), minPartSize*2)) - // Copy 11MiB worth of random data. n, err := io.CopyN(file, r, minPartSize*2) if err != nil { t.Fatal("Error:", err) @@ -904,16 +962,13 @@ func TestResumableFPutObject(t *testing.T) { } // Upload 4 parts to use all 3 multipart 'workers' and have an extra part. - buffer := make([]byte, minPartSize*4) - // Use crand.Reader for multipart tests to ensure parts are uploaded in correct order. - size, err := io.ReadFull(crand.Reader, buffer) - if err != nil { - t.Fatal("Error:", err) + // Use different data in each part for multipart tests to ensure parts are uploaded in correct order. + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) - } - size, err = file.Write(buffer) + + size, err := file.Write(buffer) if err != nil { t.Fatal("Error:", err) } @@ -995,16 +1050,12 @@ func TestFPutObjectMultipart(t *testing.T) { } // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. - buffer := make([]byte, minPartSize*4) - - size, err := io.ReadFull(crand.Reader, buffer) - if err != nil { - t.Fatal("Error:", err) + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) - } - size, err = file.Write(buffer) + + size, err := file.Write(buffer) if err != nil { t.Fatal("Error:", err) } @@ -1100,18 +1151,14 @@ func TestFPutObject(t *testing.T) { } // Upload 4 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. - buffer := make([]byte, minPartSize*4) - // Use random data for multipart tests to check parts are uploaded in correct order. - size, err := io.ReadFull(crand.Reader, buffer) - if err != nil { - t.Fatal("Error:", err) - } - if size != minPartSize*4 { - t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", minPartSize*4, size) + // Use different data in part for multipart tests to check parts are uploaded in correct order. + var buffer []byte + for i := 0; i < 4; i++ { + buffer = append(buffer, bytes.Repeat([]byte(string('a'+i)), minPartSize)...) } // Write the data to the file. - size, err = file.Write(buffer) + size, err := file.Write(buffer) if err != nil { t.Fatal("Error:", err) } @@ -1784,10 +1831,178 @@ func TestCopyObject(t *testing.T) { } } +// TestEncryptionPutGet tests client side encryption +func TestEncryptionPutGet(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := New( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate a symmetric key + symKey := encrypt.NewSymmetricKey([]byte("my-secret-key-00")) + + // Generate an assymmetric key from predefine public and private certificates + privateKey, err := hex.DecodeString( + "30820277020100300d06092a864886f70d0101010500048202613082025d" + + "0201000281810087b42ea73243a3576dc4c0b6fa245d339582dfdbddc20c" + + "bb8ab666385034d997210c54ba79275c51162a1221c3fb1a4c7c61131ca6" + + "5563b319d83474ef5e803fbfa7e52b889e1893b02586b724250de7ac6351" + + "cc0b7c638c980acec0a07020a78eed7eaa471eca4b92071394e061346c06" + + "15ccce2f465dee2080a89e43f29b5702030100010281801dd5770c3af8b3" + + "c85cd18cacad81a11bde1acfac3eac92b00866e142301fee565365aa9af4" + + "57baebf8bb7711054d071319a51dd6869aef3848ce477a0dc5f0dbc0c336" + + "5814b24c820491ae2bb3c707229a654427e03307fec683e6b27856688f08" + + "bdaa88054c5eeeb773793ff7543ee0fb0e2ad716856f2777f809ef7e6fa4" + + "41024100ca6b1edf89e8a8f93cce4b98c76c6990a09eb0d32ad9d3d04fbf" + + "0b026fa935c44f0a1c05dd96df192143b7bda8b110ec8ace28927181fd8c" + + "d2f17330b9b63535024100aba0260afb41489451baaeba423bee39bcbd1e" + + "f63dd44ee2d466d2453e683bf46d019a8baead3a2c7fca987988eb4d565e" + + "27d6be34605953f5034e4faeec9bdb0241009db2cb00b8be8c36710aff96" + + "6d77a6dec86419baca9d9e09a2b761ea69f7d82db2ae5b9aae4246599bb2" + + "d849684d5ab40e8802cfe4a2b358ad56f2b939561d2902404e0ead9ecafd" + + "bb33f22414fa13cbcc22a86bdf9c212ce1a01af894e3f76952f36d6c904c" + + "bd6a7e0de52550c9ddf31f1e8bfe5495f79e66a25fca5c20b3af5b870241" + + "0083456232aa58a8c45e5b110494599bda8dbe6a094683a0539ddd24e19d" + + "47684263bbe285ad953d725942d670b8f290d50c0bca3d1dc9688569f1d5" + + "9945cb5c7d") + + if err != nil { + t.Fatal(err) + } + + publicKey, err := hex.DecodeString("30819f300d06092a864886f70d010101050003818d003081890281810087" + + "b42ea73243a3576dc4c0b6fa245d339582dfdbddc20cbb8ab666385034d9" + + "97210c54ba79275c51162a1221c3fb1a4c7c61131ca65563b319d83474ef" + + "5e803fbfa7e52b889e1893b02586b724250de7ac6351cc0b7c638c980ace" + + "c0a07020a78eed7eaa471eca4b92071394e061346c0615ccce2f465dee20" + + "80a89e43f29b570203010001") + if err != nil { + t.Fatal(err) + } + + // Generate an asymmetric key + asymKey, err := encrypt.NewAsymmetricKey(privateKey, publicKey) + if err != nil { + t.Fatal(err) + } + + // Object custom metadata + customContentType := "custom/contenttype" + + testCases := []struct { + buf []byte + encKey encrypt.Key + }{ + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 15)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 17)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 31)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 33)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*2)}, + {encKey: symKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 0)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 16)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 32)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024)}, + {encKey: asymKey, buf: bytes.Repeat([]byte("F"), 1024*1024)}, + } + + for i, testCase := range testCases { + // Generate a random object name + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + + // Secured object + cbcMaterials, err := encrypt.NewCBCSecureMaterials(testCase.encKey) + if err != nil { + t.Fatal(err) + } + + // Put encrypted data + _, err = c.PutEncryptedObject(bucketName, objectName, bytes.NewReader(testCase.buf), cbcMaterials, map[string][]string{"Content-Type": {customContentType}}, nil) + if err != nil { + t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + + // Read the data back + r, err := c.GetEncryptedObject(bucketName, objectName, cbcMaterials) + if err != nil { + t.Fatalf("Test %d, error: %v %v %v", i+1, err, bucketName, objectName) + } + + // Compare the sent object with the received one + recvBuffer := bytes.NewBuffer([]byte{}) + if _, err = io.Copy(recvBuffer, r); err != nil { + t.Fatalf("Test %d, error: %v", i+1, err) + } + if recvBuffer.Len() != len(testCase.buf) { + t.Fatalf("Test %d, error: number of bytes of received object does not match, want %v, got %v\n", + i+1, len(testCase.buf), recvBuffer.Len()) + } + if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { + t.Fatalf("Test %d, error: Encrypted sent is not equal to decrypted, want `%x`, go `%x`", i+1, testCase.buf, recvBuffer.Bytes()) + } + + // Remove test object + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatalf("Test %d, error: %v", i+1, err) + } + + } + + // Remove test bucket + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } + +} + func TestBucketNotification(t *testing.T) { if testing.Short() { t.Skip("skipping functional tests for the short runs") } + if os.Getenv("NOTIFY_BUCKET") == "" || + os.Getenv("NOTIFY_SERVICE") == "" || + os.Getenv("NOTIFY_REGION") == "" || + os.Getenv("NOTIFY_ACCOUNTID") == "" || + os.Getenv("NOTIFY_RESOURCE") == "" { + t.Skip("skipping notification test if not configured") + } // Seed random based on current time. rand.Seed(time.Now().Unix()) @@ -2187,3 +2402,172 @@ func TestFunctional(t *testing.T) { t.Fatal("Error: ", err) } } + +// Test for validating GetObject Reader* methods functioning when the +// object is modified in the object store. +func TestGetObjectObjectModified(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + // Upload an object. + objectName := "myobject" + content := "helloworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(content), "application/text") + if err != nil { + t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + defer c.RemoveObject(bucketName, objectName) + + reader, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatalf("Failed to get object %s/%s: %v", bucketName, objectName, err) + } + defer reader.Close() + + // Read a few bytes of the object. + b := make([]byte, 5) + n, err := reader.ReadAt(b, 0) + if err != nil { + t.Fatalf("Failed to read object %s/%s at an offset: %v", bucketName, objectName, err) + } + + // Upload different contents to the same object while object is being read. + newContent := "goodbyeworld" + _, err = c.PutObject(bucketName, objectName, strings.NewReader(newContent), "application/text") + if err != nil { + t.Fatalf("Failed to upload %s/%s: %v", bucketName, objectName, err) + } + + // Confirm that a Stat() call in between doesn't change the Object's cached etag. + _, err = reader.Stat() + if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { + t.Errorf("Expected Stat to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) + } + + // Read again only to find object contents have been modified since last read. + _, err = reader.ReadAt(b, int64(n)) + if err.Error() != s3ErrorResponseMap["PreconditionFailed"] { + t.Errorf("Expected ReadAt to fail with error %s but received %s", s3ErrorResponseMap["PreconditionFailed"], err.Error()) + } +} + +// Test validates putObject to upload a file seeked at a given offset. +func TestPutObjectUploadSeekedObject(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Instantiate new minio client object. + c, err := NewV4( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Make a new bucket. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + defer c.RemoveBucket(bucketName) + + tempfile, err := ioutil.TempFile("", "minio-go-upload-test-") + if err != nil { + t.Fatal("Error:", err) + } + + var length = 120000 + data := bytes.Repeat([]byte("1"), length) + + if _, err = tempfile.Write(data); err != nil { + t.Fatal("Error:", err) + } + + objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) + + offset := length / 2 + if _, err := tempfile.Seek(int64(offset), 0); err != nil { + t.Fatal("Error:", err) + } + + n, err := c.PutObject(bucketName, objectName, tempfile, "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(length-offset) { + t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + tempfile.Close() + if err = os.Remove(tempfile.Name()); err != nil { + t.Fatal("Error:", err) + } + + length = int(n) + + obj, err := c.GetObject(bucketName, objectName) + if err != nil { + t.Fatal("Error:", err) + } + + n, err = obj.Seek(int64(offset), 0) + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(offset) { + t.Fatalf("Invalid offset returned, want %v, got %v", int64(offset), n) + } + + n, err = c.PutObject(bucketName, objectName+"getobject", obj, "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err) + } + if n != int64(length-offset) { + t.Fatalf("Invalid length returned, want %v, got %v", int64(length-offset), n) + } + + if err = c.RemoveObject(bucketName, objectName); err != nil { + t.Fatal("Error:", err) + } + + if err = c.RemoveObject(bucketName, objectName+"getobject"); err != nil { + t.Fatal("Error:", err) + } +} |