//go:build mint /* * MinIO Go Library for Amazon S3 Compatible Cloud Storage * Copyright 2015-2020 MinIO, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package main import ( "archive/zip" "bytes" "context" "crypto/sha256" "encoding/base64" "errors" "fmt" "hash" "hash/crc32" "io" "iter" "log/slog" "math/rand" "mime/multipart" "net/http" "net/url" "os" "path" "path/filepath" "reflect" "runtime" "sort" "strconv" "strings" "sync" "time" "github.com/dustin/go-humanize" "github.com/google/uuid" "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/cors" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/encrypt" "github.com/minio/minio-go/v7/pkg/notification" "github.com/minio/minio-go/v7/pkg/tags" ) const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" const ( letterIdxBits = 6 // 6 bits to represent a letter index letterIdxMask = 1<= len(buf) { err = nil } else if n > 0 && err == io.EOF { err = io.ErrUnexpectedEOF } return n, err } func baseLogger(testName, function string, args map[string]interface{}, startTime time.Time) *slog.Logger { // calculate the test case duration duration := time.Since(startTime) // log with the fields as per mint l := slog.With( "name", "minio-go: "+testName, "duration", duration.Nanoseconds()/1000000, ) if function != "" { l = l.With("function", function) } if len(args) > 0 { l = l.With("args", args) } return l } // log successful test runs func logSuccess(testName, function string, args map[string]interface{}, startTime time.Time) { baseLogger(testName, function, args, startTime). With("status", "PASS"). Info("") } // As few of the features are not available in Gateway(s) currently, Check if err value is NotImplemented, // and log as NA in that case and continue execution. Otherwise log as failure and return func logError(testName, function string, args map[string]interface{}, startTime time.Time, alert, message string, err error) { // If server returns NotImplemented we assume it is gateway mode and hence log it as info and move on to next tests // Special case for ComposeObject API as it is implemented on client side and adds specific error details like `Error in upload-part-copy` in // addition to NotImplemented error returned from server if isErrNotImplemented(err) { logIgnored(testName, function, args, startTime, message) } else { logFailure(testName, function, args, startTime, alert, message, err) if !isRunOnFail() { panic(fmt.Sprintf("Test failed with message: %s, err: %v", message, err)) } } } // Log failed test runs, do not call this directly, use logError instead, as that correctly stops the test run func logFailure(testName, function string, args map[string]interface{}, startTime time.Time, alert, message string, err error) { l := baseLogger(testName, function, args, startTime).With( "status", "FAIL", "alert", alert, "message", message, ) if err != nil { l = l.With("error", err) } l.Error("") } // log not applicable test runs func logIgnored(testName, function string, args map[string]interface{}, startTime time.Time, alert string) { baseLogger(testName, function, args, startTime). With( "status", "NA", "alert", strings.Split(alert, " ")[0]+" is NotImplemented", ).Info("") } // Delete objects in given bucket, recursively func cleanupBucket(bucketName string, c *minio.Client) error { // Create a done channel to control 'ListObjectsV2' go routine. doneCh := make(chan struct{}) // Exit cleanly upon return. defer close(doneCh) // Iterate over all objects in the bucket via listObjectsV2 and delete for objCh := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Recursive: true}) { if objCh.Err != nil { return objCh.Err } if objCh.Key != "" { err := c.RemoveObject(context.Background(), bucketName, objCh.Key, minio.RemoveObjectOptions{}) if err != nil { return err } } } for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) { if objPartInfo.Err != nil { return objPartInfo.Err } if objPartInfo.Key != "" { err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key) if err != nil { return err } } } // objects are already deleted, clear the buckets now return c.RemoveBucket(context.Background(), bucketName) } func cleanupVersionedBucket(bucketName string, c *minio.Client) error { doneCh := make(chan struct{}) defer close(doneCh) for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) { if obj.Err != nil { return obj.Err } if obj.Key != "" { err := c.RemoveObject(context.Background(), bucketName, obj.Key, minio.RemoveObjectOptions{VersionID: obj.VersionID, GovernanceBypass: true}) if err != nil { return err } } } for objPartInfo := range c.ListIncompleteUploads(context.Background(), bucketName, "", true) { if objPartInfo.Err != nil { return objPartInfo.Err } if objPartInfo.Key != "" { err := c.RemoveIncompleteUpload(context.Background(), bucketName, objPartInfo.Key) if err != nil { return err } } } // objects are already deleted, clear the buckets now err := c.RemoveBucket(context.Background(), bucketName) if err != nil { for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) { slog.Info("found object", "key", obj.Key, "version", obj.VersionID) } } return err } func isErrNotImplemented(err error) bool { return minio.ToErrorResponse(err).Code == minio.NotImplemented } func isRunOnFail() bool { return os.Getenv("RUN_ON_FAIL") == "1" } func init() { // If server endpoint is not set, all tests default to // using https://play.min.io if os.Getenv(serverEndpoint) == "" { os.Setenv(serverEndpoint, "play.min.io") os.Setenv(accessKey, "Q3AM3UQ867SPQQA43P2F") os.Setenv(secretKey, "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") os.Setenv(enableHTTPS, "1") } } var mintDataDir = os.Getenv("MINT_DATA_DIR") func getMintDataDirFilePath(filename string) (fp string) { if mintDataDir == "" { return fp } return filepath.Join(mintDataDir, filename) } func newRandomReader(seed, size int64) io.Reader { return io.LimitReader(rand.New(rand.NewSource(seed)), size) } func mustCrcReader(r io.Reader) uint32 { crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { panic(err) } return crc.Sum32() } func crcMatches(r io.Reader, want uint32) error { crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { panic(err) } got := crc.Sum32() if got != want { return fmt.Errorf("crc mismatch, want %x, got %x", want, got) } return nil } func crcMatchesName(r io.Reader, name string) error { want := dataFileCRC32[name] crc := crc32.NewIEEE() _, err := io.Copy(crc, r) if err != nil { panic(err) } got := crc.Sum32() if got != want { return fmt.Errorf("crc mismatch, want %x, got %x", want, got) } return nil } // read data from file if it exists or optionally create a buffer of particular size func getDataReader(fileName string) io.ReadCloser { if mintDataDir == "" { size := int64(dataFileMap[fileName]) if _, ok := dataFileCRC32[fileName]; !ok { dataFileCRC32[fileName] = mustCrcReader(newRandomReader(size, size)) } return io.NopCloser(newRandomReader(size, size)) } reader, _ := os.Open(getMintDataDirFilePath(fileName)) if _, ok := dataFileCRC32[fileName]; !ok { dataFileCRC32[fileName] = mustCrcReader(reader) reader.Close() reader, _ = os.Open(getMintDataDirFilePath(fileName)) } return reader } // randString generates random names and prepends them with a known prefix. func randString(n int, src rand.Source, prefix string) string { b := make([]byte, n) // A rand.Int63() generates 63 random bits, enough for letterIdxMax letters! for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { if remain == 0 { cache, remain = src.Int63(), letterIdxMax } if idx := int(cache & letterIdxMask); idx < len(letterBytes) { b[i] = letterBytes[idx] i-- } cache >>= letterIdxBits remain-- } return prefix + string(b[0:30-len(prefix)]) } var dataFileMap = map[string]int{ "datafile-0-b": 0, "datafile-1-b": 1, "datafile-1-kB": 1 * humanize.KiByte, "datafile-10-kB": 10 * humanize.KiByte, "datafile-33-kB": 33 * humanize.KiByte, "datafile-100-kB": 100 * humanize.KiByte, "datafile-1.03-MB": 1056 * humanize.KiByte, "datafile-1-MB": 1 * humanize.MiByte, "datafile-5-MB": 5 * humanize.MiByte, "datafile-6-MB": 6 * humanize.MiByte, "datafile-11-MB": 11 * humanize.MiByte, "datafile-65-MB": 65 * humanize.MiByte, "datafile-129-MB": 129 * humanize.MiByte, } var dataFileCRC32 = map[string]uint32{} func isFullMode() bool { return os.Getenv("MINT_MODE") == "full" } func getFuncName() string { return getFuncNameLoc(2) } func getFuncNameLoc(caller int) string { pc, _, _, _ := runtime.Caller(caller) return strings.TrimPrefix(runtime.FuncForPC(pc).Name(), "main.") } type ClientConfig struct { // MinIO client configuration TraceOn bool // Turn on tracing of HTTP requests and responses to stderr CredsV2 bool // Use V2 credentials if true, otherwise use v4 TrailingHeaders bool // Send trailing headers in requests } func NewClient(config ClientConfig) (*minio.Client, error) { // Instantiate new MinIO client var creds *credentials.Credentials if config.CredsV2 { creds = credentials.NewStaticV2(os.Getenv(accessKey), os.Getenv(secretKey), "") } else { creds = credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), "") } opts := &minio.Options{ Creds: creds, Transport: createHTTPTransport(), Secure: mustParseBool(os.Getenv(enableHTTPS)), TrailingHeaders: config.TrailingHeaders, } client, err := minio.New(os.Getenv(serverEndpoint), opts) if err != nil { return nil, err } if config.TraceOn { client.TraceOn(os.Stderr) } // Set user agent. client.SetAppInfo("MinIO-go-FunctionalTest", appVersion) return client, nil } // Tests bucket re-create errors. func testMakeBucketError() { region := "eu-central-1" // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" // initialize logging params args := map[string]interface{}{ "bucketName": "", "region": region, } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket Failed", err) return } defer cleanupBucket(bucketName, c) if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil { logError(testName, function, args, startTime, "", "Bucket already exists", err) return } // Verify valid error response from server. if minio.ToErrorResponse(err).Code != minio.BucketAlreadyExists && minio.ToErrorResponse(err).Code != minio.BucketAlreadyOwnedByYou { logError(testName, function, args, startTime, "", "Invalid error returned by server", err) return } logSuccess(testName, function, args, startTime) } func testMetadataSizeLimit() { startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, objectSize, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts.UserMetadata": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) const HeaderSizeLimit = 8 * 1024 const UserMetadataLimit = 2 * 1024 // Meta-data greater than the 2 KB limit of AWS - PUT calls with this meta-data should fail metadata := make(map[string]string) metadata["X-Amz-Meta-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+UserMetadataLimit-len("X-Amz-Meta-Mint-Test"))) args["metadata"] = fmt.Sprint(metadata) _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) if err == nil { logError(testName, function, args, startTime, "", "Created object with user-defined metadata exceeding metadata size limits", nil) return } // Meta-data (headers) greater than the 8 KB limit of AWS - PUT calls with this meta-data should fail metadata = make(map[string]string) metadata["X-Amz-Mint-Test"] = string(bytes.Repeat([]byte("m"), 1+HeaderSizeLimit-len("X-Amz-Mint-Test"))) args["metadata"] = fmt.Sprint(metadata) _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(nil), 0, minio.PutObjectOptions{UserMetadata: metadata}) if err == nil { logError(testName, function, args, startTime, "", "Created object with headers exceeding header size limits", nil) return } logSuccess(testName, function, args, startTime) } // Tests various bucket supported formats. func testMakeBucketRegions() { region := "eu-central-1" // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" // initialize logging params args := map[string]interface{}{ "bucketName": "", "region": region, } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } // Make a new bucket with '.' in its name, in 'us-west-2'. This // request is internally staged into a path style instead of // virtual host style. region = "us-west-2" args["region"] = region if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName+".withperiod", c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test PutObject using a large data to trigger multipart readat func testPutObjectReadAt() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "objectContentType", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Object content type objectContentType := "binary/octet-stream" args["objectContentType"] = objectContentType _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Get Object failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat Object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", fmt.Sprintf("Number of bytes in stat does not match, expected %d got %d", bufSize, st.Size), err) return } if st.ContentType != objectContentType && st.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content types don't match", err) return } if err := crcMatchesName(r, "datafile-129-MB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object is already closed, didn't return error on Close", err) return } logSuccess(testName, function, args, startTime) } func testListObjectVersions() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ListObjectVersions(bucketName, prefix, recursive)" args := map[string]interface{}{ "bucketName": "", "prefix": "", "recursive": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName bufSize := dataFileMap["datafile-10-kB"] reader := getDataReader("datafile-10-kB") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() bufSize = dataFileMap["datafile-1-b"] reader = getDataReader("datafile-1-b") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Unexpected object deletion", err) return } var deleteMarkers, versions int objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } if info.Key != objectName { logError(testName, function, args, startTime, "", "Unexpected object name in listing objects", nil) return } if info.VersionID == "" { logError(testName, function, args, startTime, "", "Unexpected version id in listing objects", nil) return } if info.IsDeleteMarker { deleteMarkers++ if !info.IsLatest { logError(testName, function, args, startTime, "", "Unexpected IsLatest field in listing objects", nil) return } } else { versions++ } } if deleteMarkers != 1 { logError(testName, function, args, startTime, "", "Unexpected number of DeleteMarker elements in listing objects", nil) return } if versions != 2 { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testStatObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "StatObject" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName bufSize := dataFileMap["datafile-10-kB"] reader := getDataReader("datafile-10-kB") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() bufSize = dataFileMap["datafile-1-b"] reader = getDataReader("datafile-1-b") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } reader.Close() objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } results = append(results, info) } if len(results) != 2 { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } for i := 0; i < len(results); i++ { opts := minio.StatObjectOptions{VersionID: results[i].VersionID} statInfo, err := c.StatObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during HEAD object", err) return } if statInfo.VersionID == "" || statInfo.VersionID != results[i].VersionID { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected version id", err) return } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return } if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) return } if statInfo.Size != results[i].Size { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) return } } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testGetObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Save the contents of datafiles to check with GetObject() reader output later var buffers [][]byte testFiles := []string{"datafile-1-b", "datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } buffers = append(buffers, buf) } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } results = append(results, info) } if len(results) != 2 { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } sort.SliceStable(results, func(i, j int) bool { return results[i].Size < results[j].Size }) sort.SliceStable(buffers, func(i, j int) bool { return len(buffers[i]) < len(buffers[j]) }) for i := 0; i < len(results); i++ { opts := minio.GetObjectOptions{VersionID: results[i].VersionID} reader, err := c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during GET object", err) return } statInfo, err := reader.Stat() if err != nil { logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) return } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return } if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) return } if statInfo.Size != results[i].Size { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) return } tmpBuffer := bytes.NewBuffer([]byte{}) _, err = io.Copy(tmpBuffer, reader) if err != nil { logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) return } if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) return } } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testPutObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName const n = 10 // Read input... // Save the data concurrently. var wg sync.WaitGroup wg.Add(n) buffers := make([][]byte, n) var errs [n]error for i := 0; i < n; i++ { r := newRandomReader(int64((1<<20)*i+i), int64(i)) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } buffers[i] = buf go func(i int) { defer wg.Done() _, errs[i] = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{PartSize: 5 << 20}) }(i) } wg.Wait() for _, err := range errs { if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", info.Err) return } results = append(results, info) } if len(results) != n { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } sort.Slice(results, func(i, j int) bool { return results[i].Size < results[j].Size }) sort.Slice(buffers, func(i, j int) bool { return len(buffers[i]) < len(buffers[j]) }) for i := 0; i < len(results); i++ { opts := minio.GetObjectOptions{VersionID: results[i].VersionID} reader, err := c.GetObject(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "error during GET object", err) return } statInfo, err := reader.Stat() if err != nil { logError(testName, function, args, startTime, "", "error during calling reader.Stat()", err) return } if statInfo.ETag != results[i].ETag { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected ETag", err) return } if statInfo.LastModified.Unix() != results[i].LastModified.Unix() { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Last-Modified", err) return } if statInfo.Size != results[i].Size { logError(testName, function, args, startTime, "", "error during HEAD object, unexpected Content-Length", err) return } tmpBuffer := bytes.NewBuffer([]byte{}) _, err = io.Copy(tmpBuffer, reader) if err != nil { logError(testName, function, args, startTime, "", "unexpected io.Copy()", err) return } if !bytes.Equal(tmpBuffer.Bytes(), buffers[i]) { logError(testName, function, args, startTime, "", "unexpected content of GetObject()", err) return } } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testListMultipartUpload() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } core := minio.Core{Client: c} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. ctx := context.Background() err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer func() { if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) } }() objName := "prefix/objectName" want := minio.ListMultipartUploadsResult{ Bucket: bucketName, KeyMarker: "", UploadIDMarker: "", NextKeyMarker: "", NextUploadIDMarker: "", EncodingType: "url", MaxUploads: 1000, IsTruncated: false, Prefix: "prefix/objectName", Delimiter: "/", CommonPrefixes: nil, } for i := 0; i < 5; i++ { uid, err := core.NewMultipartUpload(ctx, bucketName, objName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload failed", err) return } want.Uploads = append(want.Uploads, minio.ObjectMultipartInfo{ Initiated: time.Time{}, StorageClass: "", Key: objName, Size: 0, UploadID: uid, Err: nil, }) for j := 0; j < 5; j++ { cmpGot := func(call string, got minio.ListMultipartUploadsResult) bool { for i := range got.Uploads { got.Uploads[i].Initiated = time.Time{} } if !reflect.DeepEqual(want, got) { err := fmt.Errorf("want: %#v\ngot : %#v", want, got) logError(testName, function, args, startTime, "", call+" failed", err) } return true } got, err := core.ListMultipartUploads(ctx, bucketName, objName, "", "", "/", 1000) if err != nil { logError(testName, function, args, startTime, "", "ListMultipartUploads failed", err) return } if !cmpGot("ListMultipartUploads-prefix", got) { return } got, err = core.ListMultipartUploads(ctx, bucketName, objName, objName, "", "/", 1000) got.KeyMarker = "" if err != nil { logError(testName, function, args, startTime, "", "ListMultipartUploads failed", err) return } if !cmpGot("ListMultipartUploads-marker", got) { return } } if i > 2 { err = core.AbortMultipartUpload(ctx, bucketName, objName, uid) if err != nil { logError(testName, function, args, startTime, "", "AbortMultipartUpload failed", err) return } want.Uploads = want.Uploads[:len(want.Uploads)-1] } } for _, up := range want.Uploads { err = core.AbortMultipartUpload(ctx, bucketName, objName, up.UploadID) if err != nil { logError(testName, function, args, startTime, "", "AbortMultipartUpload failed", err) return } } logSuccess(testName, function, args, startTime) } func testCopyObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName testFiles := []string{"datafile-1-b", "datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var infos []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } infos = append(infos, info) } sort.Slice(infos, func(i, j int) bool { return infos[i].Size < infos[j].Size }) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) return } oldestContent, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) return } // Copy Source srcOpts := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: infos[0].VersionID, } args["src"] = srcOpts dstOpts := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName + "-copy", } args["dst"] = dstOpts // Perform the Copy if _, err = c.CopyObject(context.Background(), dstOpts, srcOpts); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer readerCopy.Close() newestContent, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) return } if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { logError(testName, function, args, startTime, "", "Unexpected destination object content", err) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testConcurrentCopyObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName testFiles := []string{"datafile-10-kB"} for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var infos []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } infos = append(infos, info) } sort.Slice(infos, func(i, j int) bool { return infos[i].Size < infos[j].Size }) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{VersionID: infos[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of the oldest version content failed", err) return } oldestContent, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Reading the oldest object version failed", err) return } // Copy Source srcOpts := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: infos[0].VersionID, } args["src"] = srcOpts dstOpts := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName + "-copy", } args["dst"] = dstOpts // Perform the Copy concurrently const n = 10 var wg sync.WaitGroup wg.Add(n) var errs [n]error for i := 0; i < n; i++ { go func(i int) { defer wg.Done() _, errs[i] = c.CopyObject(context.Background(), dstOpts, srcOpts) }(i) } wg.Wait() for _, err := range errs { if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } } objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: false, Prefix: dstOpts.Object}) infos = []minio.ObjectInfo{} for info := range objectsInfo { // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{VersionID: info.VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer readerCopy.Close() newestContent, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from GetObject reader failed", err) return } if len(newestContent) == 0 || !bytes.Equal(oldestContent, newestContent) { logError(testName, function, args, startTime, "", "Unexpected destination object content", err) return } infos = append(infos, info) } if len(infos) != n { logError(testName, function, args, startTime, "", "Unexpected number of Version elements in listing objects", nil) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testComposeObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // var testFiles = []string{"datafile-5-MB", "datafile-10-kB"} testFiles := []string{"datafile-5-MB", "datafile-10-kB"} var testFilesBytes [][]byte for _, testFile := range testFiles { r := getDataReader(testFile) buf, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "unexpected failure", err) return } r.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } testFilesBytes = append(testFilesBytes, buf) } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var results []minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } results = append(results, info) } sort.SliceStable(results, func(i, j int) bool { return results[i].Size > results[j].Size }) // Source objects to concatenate. We also specify decryption // key for each src1 := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: results[0].VersionID, } src2 := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, VersionID: results[1].VersionID, } dst := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName + "-copy", } _, err = c.ComposeObject(context.Background(), dst, src1, src2) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName, objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of the copy object failed", err) return } defer readerCopy.Close() copyContentBytes, err := io.ReadAll(readerCopy) if err != nil { logError(testName, function, args, startTime, "", "Reading from the copy object reader failed", err) return } var expectedContent []byte for _, fileBytes := range testFilesBytes { expectedContent = append(expectedContent, fileBytes...) } if len(copyContentBytes) == 0 || !bytes.Equal(copyContentBytes, expectedContent) { logError(testName, function, args, startTime, "", "Unexpected destination object content", err) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testRemoveObjectWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "DeleteObject()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } objectsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var version minio.ObjectInfo for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } version = info break } err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{VersionID: version.VersionID}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObject failed", err) return } objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for range objectsInfo { logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) return } // test delete marker version id is non-null _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // create delete marker err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObject failed", err) return } objectsInfo = c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) idx := 0 for info := range objectsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } if idx == 0 { if !info.IsDeleteMarker { logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to have been created", err) return } if info.VersionID == "" { logError(testName, function, args, startTime, "", "Unexpected error - expected delete marker to be versioned", err) return } } idx++ } defer cleanupBucket(bucketName, c) logSuccess(testName, function, args, startTime) } func testRemoveObjectsWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "DeleteObjects()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader("datafile-10-kB"), int64(dataFileMap["datafile-10-kB"]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } objectsVersions := make(chan minio.ObjectInfo) go func() { objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for info := range objectsVersionsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } objectsVersions <- info } close(objectsVersions) }() removeErrors := c.RemoveObjects(context.Background(), bucketName, objectsVersions, minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "DeleteObjects call failed", err) return } for e := range removeErrors { if e.Err != nil { logError(testName, function, args, startTime, "", "Single delete operation failed", err) return } } objectsVersionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) for range objectsVersionsInfo { logError(testName, function, args, startTime, "", "Unexpected versioning info, should not have any one ", err) return } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testObjectTaggingWithVersioning() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "{Get,Set,Remove}ObjectTagging()" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "Enable versioning failed", err) return } objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName for _, file := range []string{"datafile-1-b", "datafile-10-kB"} { _, err = c.PutObject(context.Background(), bucketName, objectName, getDataReader(file), int64(dataFileMap[file]), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } versionsInfo := c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{WithVersions: true, Recursive: true}) var versions []minio.ObjectInfo for info := range versionsInfo { if info.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error during listing objects", err) return } versions = append(versions, info) } sort.SliceStable(versions, func(i, j int) bool { return versions[i].Size < versions[j].Size }) tagsV1 := map[string]string{"key1": "val1"} t1, err := tags.MapToObjectTags(tagsV1) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } err = c.PutObjectTagging(context.Background(), bucketName, objectName, t1, minio.PutObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } tagsV2 := map[string]string{"key2": "val2"} t2, err := tags.MapToObjectTags(tagsV2) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (1) failed", err) return } err = c.PutObjectTagging(context.Background(), bucketName, objectName, t2, minio.PutObjectTaggingOptions{VersionID: versions[1].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return } tagsEqual := func(tags1, tags2 map[string]string) bool { for k1, v1 := range tags1 { v2, found := tags2[k1] if found { if v1 != v2 { return false } } } return true } gotTagsV1, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectTagging failed", err) return } if !tagsEqual(t1.ToMap(), gotTagsV1.ToMap()) { logError(testName, function, args, startTime, "", "Unexpected tags content (1)", err) return } gotTagsV2, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectTaggingContext failed", err) return } if !tagsEqual(t2.ToMap(), gotTagsV2.ToMap()) { logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } err = c.RemoveObjectTagging(context.Background(), bucketName, objectName, minio.RemoveObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectTagging (2) failed", err) return } emptyTags, err := c.GetObjectTagging(context.Background(), bucketName, objectName, minio.GetObjectTaggingOptions{VersionID: versions[0].VersionID}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectTagging failed", err) return } if len(emptyTags.ToMap()) != 0 { logError(testName, function, args, startTime, "", "Unexpected tags content (2)", err) return } // Delete all objects and their versions as long as the bucket itself if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } func testPutObjectWithAutoChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) const testfile = "datafile-1.03-MB" bufSize := dataFileMap[testfile] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader(testfile) b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := minio.ChecksumCRC64NVME.Hasher() h.Reset() h.Write(b) // Upload the data without explicit checksum. resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, DisableContentSha256: false, UserMetadata: meta, AutoChecksum: minio.ChecksumNone, Checksum: minio.ChecksumNone, }) _ = resp if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the metadata back gopts := minio.GetObjectOptions{Checksum: true} st, err := c.StatObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } if st.ChecksumCRC64NVME != "" { meta[minio.ChecksumCRC64NVME.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) if st.ChecksumMode != minio.ChecksumFullObjectMode.String() { logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String())) } } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } logSuccess(testName, function, args, startTime) } // Test PutObject with custom checksums. func testPutObjectWithChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { cs minio.ChecksumType }{ {cs: minio.ChecksumMD5}, {cs: minio.ChecksumSHA512}, {cs: minio.ChecksumXXHash64}, {cs: minio.ChecksumXXHash3}, {cs: minio.ChecksumXXHash128}, {cs: minio.ChecksumCRC32C}, {cs: minio.ChecksumCRC32}, {cs: minio.ChecksumSHA1}, {cs: minio.ChecksumSHA256}, {cs: minio.ChecksumCRC64NVME}, } for _, test := range tests { if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() { continue } bufSize := dataFileMap["datafile-10-kB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader("datafile-10-kB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := test.cs.Hasher() h.Reset() if test.cs.IsSet() { meta["x-amz-checksum-algorithm"] = test.cs.String() } // Test with a bad CRC - we haven't called h.Write(b), so this is a checksum of empty data meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) args["metadata"] = meta args["range"] = "false" args["checksum"] = test.cs.String() resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: meta, }) if err == nil { logError(testName, function, args, startTime, "", "PutObject did not fail on wrong CRC", err) return } // Set correct CRC. h.Write(b) meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) reader.Close() resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, DisableContentSha256: true, UserMetadata: meta, }) if err != nil { if ignore2026ChecksumError(test.cs, err) { logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String()) continue } logError(testName, function, args, startTime, "", "PutObject failed", err) return } cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"]) cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"]) cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"]) cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"]) cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"]) if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() { logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String())) } // Read the data back gopts := minio.GetObjectOptions{Checksum: true} r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"]) cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"]) cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"]) cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"]) cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"]) if st.ChecksumMode != minio.ChecksumFullObjectMode.String() { logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String())) } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) return } args["range"] = "true" err = gopts.SetRange(100, 1000) if err != nil { logError(testName, function, args, startTime, "", "SetRange failed", err) return } r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } b, err = io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } st, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Range requests should return empty checksums... cmpChecksum(st.ChecksumSHA256, "") cmpChecksum(st.ChecksumSHA1, "") cmpChecksum(st.ChecksumCRC32, "") cmpChecksum(st.ChecksumCRC32C, "") cmpChecksum(st.ChecksumCRC64NVME, "") cmpChecksum(st.ChecksumMD5, "") cmpChecksum(st.ChecksumSHA512, "") cmpChecksum(st.ChecksumXXHash64, "") cmpChecksum(st.ChecksumXXHash3, "") cmpChecksum(st.ChecksumXXHash128, "") delete(args, "range") delete(args, "metadata") logSuccess(testName, function, args, startTime) } } // Test PutObject with custom checksums. func testPutObjectWithTrailingChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { cs minio.ChecksumType }{ {cs: minio.ChecksumMD5}, {cs: minio.ChecksumSHA512}, {cs: minio.ChecksumXXHash64}, {cs: minio.ChecksumXXHash3}, {cs: minio.ChecksumXXHash128}, {cs: minio.ChecksumCRC64NVME}, {cs: minio.ChecksumCRC32C}, {cs: minio.ChecksumCRC32}, {cs: minio.ChecksumSHA1}, {cs: minio.ChecksumSHA256}, } for _, test := range tests { if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() { continue } function := "PutObject(bucketName, objectName, reader,size, opts)" bufSize := dataFileMap["datafile-10-kB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader("datafile-10-kB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := test.cs.Hasher() h.Reset() // Test with Wrong CRC. args["metadata"] = meta args["range"] = "false" args["checksum"] = test.cs.String() resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, DisableContentSha256: true, UserMetadata: meta, Checksum: test.cs, }) if err != nil { if ignore2026ChecksumError(test.cs, err) { logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String()) continue } logError(testName, function, args, startTime, "", "PutObject failed", err) return } h.Write(b) meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"]) cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"]) cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"]) cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"]) cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"]) // Read the data back gopts := minio.GetObjectOptions{Checksum: true} function = "GetObject(...)" r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"]) cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"]) cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"]) cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"]) cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"]) if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) return } function = "GetObject( Range...)" args["range"] = "true" err = gopts.SetRange(100, 1000) if err != nil { logError(testName, function, args, startTime, "", "SetRange failed", err) return } r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } b, err = io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } st, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Range requests should return empty checksums... cmpChecksum(st.ChecksumSHA256, "") cmpChecksum(st.ChecksumSHA1, "") cmpChecksum(st.ChecksumCRC32, "") cmpChecksum(st.ChecksumCRC32C, "") cmpChecksum(st.ChecksumCRC64NVME, "") cmpChecksum(st.ChecksumMD5, "") cmpChecksum(st.ChecksumSHA512, "") cmpChecksum(st.ChecksumXXHash64, "") cmpChecksum(st.ChecksumXXHash3, "") cmpChecksum(st.ChecksumXXHash128, "") function = "GetObjectAttributes(...)" s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) return } cmpChecksum(s.Checksum.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) cmpChecksum(s.Checksum.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"]) cmpChecksum(s.Checksum.ChecksumMD5, meta["x-amz-checksum-md5"]) cmpChecksum(s.Checksum.ChecksumSHA512, meta["x-amz-checksum-sha512"]) cmpChecksum(s.Checksum.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"]) cmpChecksum(s.Checksum.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"]) cmpChecksum(s.Checksum.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"]) if s.Checksum.ChecksumType != "" && s.Checksum.ChecksumType != minio.ChecksumFullObjectMode.String() { logError(testName, function, args, startTime, "", "ChecksumType mismatch in GetObjectAttributes", fmt.Errorf("want %s, got %s", minio.ChecksumFullObjectMode.String(), s.Checksum.ChecksumType)) return } delete(args, "range") delete(args, "metadata") logSuccess(testName, function, args, startTime) } } // Test PutObject with custom checksums. func testPutMultipartObjectWithChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Trailing: true}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } hashMultiPart := func(b []byte, partSize int, cs minio.ChecksumType) string { r := bytes.NewReader(b) hasher := cs.Hasher() if cs.FullObjectRequested() { partSize = len(b) } tmp := make([]byte, partSize) parts := 0 var all []byte for { n, err := io.ReadFull(r, tmp) if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF { logError(testName, function, args, startTime, "", "Calc crc failed", err) } if n == 0 { break } parts++ hasher.Reset() hasher.Write(tmp[:n]) all = append(all, hasher.Sum(nil)...) if err != nil { break } } if parts == 1 { return base64.StdEncoding.EncodeToString(hasher.Sum(nil)) } hasher.Reset() hasher.Write(all) return fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(hasher.Sum(nil)), parts) } defer cleanupBucket(bucketName, c) tests := []struct { cs minio.ChecksumType }{ {cs: minio.ChecksumMD5}, {cs: minio.ChecksumSHA512}, {cs: minio.ChecksumXXHash64}, {cs: minio.ChecksumXXHash3}, {cs: minio.ChecksumXXHash128}, {cs: minio.ChecksumFullObjectCRC32}, {cs: minio.ChecksumFullObjectCRC32C}, {cs: minio.ChecksumCRC64NVME}, {cs: minio.ChecksumCRC32C}, {cs: minio.ChecksumCRC32}, {cs: minio.ChecksumSHA1}, {cs: minio.ChecksumSHA256}, } for _, test := range tests { if os.Getenv("MINT_NO_FULL_OBJECT") != "" && test.cs.FullObjectRequested() { continue } args["section"] = "prep" bufSize := dataFileMap["datafile-129-MB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName args["checksum"] = test.cs.String() cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) // fmt.Printf("want %s, got %s\n", want, got) return } } const partSize = 10 << 20 reader := getDataReader("datafile-129-MB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } reader.Close() h := test.cs.Hasher() h.Reset() // wantChksm might be the full object checksum or the multipart checksum, depending on the test.cs type. wantChksm := hashMultiPart(b, partSize, test.cs) // wantFullObjectChksm is always the full object checksum that is returned after CopyObject. wantFullObjectChksm := hashMultiPart(b, len(b), test.cs) rd := bytes.NewReader(b) cs := test.cs // Set correct CRC. args["section"] = "PutObject" resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: partSize, Checksum: cs, }) if err != nil { if ignore2026ChecksumError(test.cs, err) { logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String()) continue } logError(testName, function, args, startTime, "", "PutObject failed", err) return } switch test.cs.Base() { case minio.ChecksumCRC32C: cmpChecksum(resp.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(resp.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(resp.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(resp.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: cmpChecksum(resp.ChecksumCRC64NVME, wantChksm) case minio.ChecksumMD5: cmpChecksum(resp.ChecksumMD5, wantChksm) case minio.ChecksumSHA512: cmpChecksum(resp.ChecksumSHA512, wantChksm) case minio.ChecksumXXHash64: cmpChecksum(resp.ChecksumXXHash64, wantChksm) case minio.ChecksumXXHash3: cmpChecksum(resp.ChecksumXXHash3, wantChksm) case minio.ChecksumXXHash128: cmpChecksum(resp.ChecksumXXHash128, wantChksm) } args["section"] = "HeadObject" st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } switch test.cs.Base() { case minio.ChecksumCRC32C: cmpChecksum(st.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(st.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(st.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(st.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: cmpChecksum(st.ChecksumCRC64NVME, wantChksm) case minio.ChecksumMD5: cmpChecksum(st.ChecksumMD5, wantChksm) case minio.ChecksumSHA512: cmpChecksum(st.ChecksumSHA512, wantChksm) case minio.ChecksumXXHash64: cmpChecksum(st.ChecksumXXHash64, wantChksm) case minio.ChecksumXXHash3: cmpChecksum(st.ChecksumXXHash3, wantChksm) case minio.ChecksumXXHash128: cmpChecksum(st.ChecksumXXHash128, wantChksm) } // Use the CopyObject API to make a copy, in the case it was a composite checksum, // it will change because the copy is no longer a multipart object. S3 returns the checksum // of the full object when HeadObject is called on the copy. args["section"] = "CopyObject" objectCopyName := objectName + "-copy" _, err = c.CopyObject(context.Background(), minio.CopyDestOptions{ Bucket: bucketName, Object: objectCopyName, }, minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, }) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } args["section"] = "HeadObject-Copy" st, err = c.StatObject(context.Background(), bucketName, objectCopyName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } switch test.cs.Base() { case minio.ChecksumCRC32C: cmpChecksum(st.ChecksumCRC32C, wantFullObjectChksm) case minio.ChecksumCRC32: cmpChecksum(st.ChecksumCRC32, wantFullObjectChksm) case minio.ChecksumSHA1: cmpChecksum(st.ChecksumSHA1, wantFullObjectChksm) case minio.ChecksumSHA256: cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm) case minio.ChecksumCRC64NVME: cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm) case minio.ChecksumMD5: cmpChecksum(st.ChecksumMD5, wantFullObjectChksm) case minio.ChecksumSHA512: cmpChecksum(st.ChecksumSHA512, wantFullObjectChksm) case minio.ChecksumXXHash64: cmpChecksum(st.ChecksumXXHash64, wantFullObjectChksm) case minio.ChecksumXXHash3: cmpChecksum(st.ChecksumXXHash3, wantFullObjectChksm) case minio.ChecksumXXHash128: cmpChecksum(st.ChecksumXXHash128, wantFullObjectChksm) } args["section"] = "GetObjectAttributes" s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) return } if strings.ContainsRune(wantChksm, '-') { wantChksm = wantChksm[:strings.IndexByte(wantChksm, '-')] } switch test.cs { // Full Object CRC does not return anything with GetObjectAttributes case minio.ChecksumCRC32C: cmpChecksum(s.Checksum.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(s.Checksum.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: cmpChecksum(s.Checksum.ChecksumCRC64NVME, wantChksm) case minio.ChecksumMD5: cmpChecksum(s.Checksum.ChecksumMD5, wantChksm) case minio.ChecksumSHA512: cmpChecksum(s.Checksum.ChecksumSHA512, wantChksm) case minio.ChecksumXXHash64: cmpChecksum(s.Checksum.ChecksumXXHash64, wantChksm) case minio.ChecksumXXHash3: cmpChecksum(s.Checksum.ChecksumXXHash3, wantChksm) case minio.ChecksumXXHash128: cmpChecksum(s.Checksum.ChecksumXXHash128, wantChksm) } if s.Checksum.ChecksumType != "" { var wantType string if test.cs.FullObjectRequested() { wantType = minio.ChecksumFullObjectMode.String() } else { wantType = minio.ChecksumCompositeMode.String() } cmpChecksum(s.Checksum.ChecksumType, wantType) } for _, part := range s.ObjectParts.Parts { if test.cs == minio.ChecksumCRC64NVME && part.ChecksumCRC64NVME == "" { logError(testName, function, args, startTime, "", "Part missing CRC64NVME checksum in GetObjectAttributes", fmt.Errorf("part %d", part.PartNumber)) return } } // Read the data back gopts := minio.GetObjectOptions{Checksum: true} gopts.PartNumber = 2 // We cannot use StatObject, since it ignores partnumber. args["section"] = "GetObject-Part" r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } io.Copy(io.Discard, r) st, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Test part 2 checksum... h.Reset() h.Write(b[partSize : 2*partSize]) wantChksm = base64.StdEncoding.EncodeToString(h.Sum(nil)) switch test.cs { // Full Object CRC does not return any part CRC for whatever reason. case minio.ChecksumCRC32C: cmpChecksum(st.ChecksumCRC32C, wantChksm) case minio.ChecksumCRC32: cmpChecksum(st.ChecksumCRC32, wantChksm) case minio.ChecksumSHA1: cmpChecksum(st.ChecksumSHA1, wantChksm) case minio.ChecksumSHA256: cmpChecksum(st.ChecksumSHA256, wantChksm) case minio.ChecksumCRC64NVME: // AWS doesn't return part checksum, but may in the future. if st.ChecksumCRC64NVME != "" { cmpChecksum(st.ChecksumCRC64NVME, wantChksm) } case minio.ChecksumMD5: cmpChecksum(st.ChecksumMD5, wantChksm) case minio.ChecksumSHA512: cmpChecksum(st.ChecksumSHA512, wantChksm) case minio.ChecksumXXHash64: cmpChecksum(st.ChecksumXXHash64, wantChksm) case minio.ChecksumXXHash3: cmpChecksum(st.ChecksumXXHash3, wantChksm) case minio.ChecksumXXHash128: cmpChecksum(st.ChecksumXXHash128, wantChksm) } delete(args, "metadata") delete(args, "section") logSuccess(testName, function, args, startTime) } } // Test PutObject with trailing checksums. func testTrailingChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } hashMultiPart := func(b []byte, partSize int, hasher hash.Hash) (oparts []minio.ObjectPart) { r := bytes.NewReader(b) tmp := make([]byte, partSize) parts := 0 for { n, err := io.ReadFull(r, tmp) if err != nil && err != io.ErrUnexpectedEOF { logError(testName, function, args, startTime, "", "Calc crc failed", err) } if n == 0 { break } parts++ hasher.Reset() hasher.Write(tmp[:n]) oparts = append(oparts, minio.ObjectPart{ PartNumber: parts, Size: int64(n), ChecksumCRC32C: base64.StdEncoding.EncodeToString(hasher.Sum(nil)), }) if err != nil { break } } return oparts } defer cleanupBucket(bucketName, c) tests := []struct { header string hasher hash.Hash // Checksum values ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string PO minio.PutObjectOptions }{ // Currently there is no way to override the checksum type. { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: 5 << 20, Checksum: minio.ChecksumFullObjectCRC32C, }, }, { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: 6_645_654, // Rather arbitrary size Checksum: minio.ChecksumFullObjectCRC32C, }, }, { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: false, DisableMultipart: false, UserMetadata: nil, PartSize: 5 << 20, Checksum: minio.ChecksumFullObjectCRC32C, }, }, { header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli)), ChecksumCRC32C: "set", PO: minio.PutObjectOptions{ DisableContentSha256: false, DisableMultipart: false, UserMetadata: nil, PartSize: 6_645_654, // Rather arbitrary size Checksum: minio.ChecksumFullObjectCRC32C, }, }, } for _, test := range tests { bufSize := dataFileMap["datafile-11-MB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %q, got %q", want, got)) return } } reader := getDataReader("datafile-11-MB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } reader.Close() h := test.hasher h.Reset() parts := hashMultiPart(b, int(test.PO.PartSize), test.hasher) cksum, err := minio.ChecksumFullObjectCRC32C.FullObjectChecksum(parts) if err != nil { logError(testName, function, args, startTime, "", "checksum calculation failed", err) return } test.ChecksumCRC32C = cksum.Encoded() // Set correct CRC. resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), test.PO) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // c.TraceOff() cmpChecksum(resp.ChecksumSHA256, test.ChecksumSHA256) cmpChecksum(resp.ChecksumSHA1, test.ChecksumSHA1) cmpChecksum(resp.ChecksumCRC32, test.ChecksumCRC32) cmpChecksum(resp.ChecksumCRC32C, test.ChecksumCRC32C) // Read the data back gopts := minio.GetObjectOptions{Checksum: true} gopts.PartNumber = 2 // We cannot use StatObject, since it ignores partnumber. r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } io.Copy(io.Discard, r) st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Test part 2 checksum... h.Reset() p2 := b[test.PO.PartSize:] if len(p2) > int(test.PO.PartSize) { p2 = p2[:test.PO.PartSize] } h.Write(p2) got := base64.StdEncoding.EncodeToString(h.Sum(nil)) if test.ChecksumSHA256 != "" { cmpChecksum(st.ChecksumSHA256, got) } if test.ChecksumSHA1 != "" { cmpChecksum(st.ChecksumSHA1, got) } if test.ChecksumCRC32 != "" { cmpChecksum(st.ChecksumCRC32, got) } if test.ChecksumCRC32C != "" { cmpChecksum(st.ChecksumCRC32C, got) } delete(args, "metadata") logSuccess(testName, function, args, startTime) } } // Test PutObject with custom checksums. func testPutObjectWithAutomaticChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { header string hasher hash.Hash // Checksum values ChecksumCRC32 string ChecksumCRC32C string ChecksumSHA1 string ChecksumSHA256 string }{ // Built-in will only add crc32c, when no MD5 nor SHA256. {header: "x-amz-checksum-crc32c", hasher: crc32.New(crc32.MakeTable(crc32.Castagnoli))}, } // defer c.TraceOff() for i, test := range tests { bufSize := dataFileMap["datafile-10-kB"] // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName cmpChecksum := func(got, want string) { if want != got { logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) return } } meta := map[string]string{} reader := getDataReader("datafile-10-kB") b, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "Read failed", err) return } h := test.hasher h.Reset() h.Write(b) meta[test.header] = base64.StdEncoding.EncodeToString(h.Sum(nil)) args["metadata"] = meta resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: nil, DisableContentSha256: true, SendContentMd5: false, }) if err == nil { if i == 0 && resp.ChecksumCRC32C == "" { logIgnored(testName, function, args, startTime, "Checksums does not appear to be supported by backend") return } } else { logError(testName, function, args, startTime, "", "PutObject failed", err) return } cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) // Usually this will be the same as above, since we skip automatic checksum when SHA256 content is sent. // When/if we add a checksum control to PutObjectOptions this will make more sense. resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: nil, DisableContentSha256: false, SendContentMd5: false, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // The checksum will not be enabled on HTTP, since it uses SHA256 blocks. if mustParseBool(os.Getenv(enableHTTPS)) { cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) } // Set SHA256 header manually sh256 := sha256.Sum256(b) meta = map[string]string{"x-amz-checksum-sha256": base64.StdEncoding.EncodeToString(sh256[:])} args["metadata"] = meta resp, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ DisableMultipart: true, UserMetadata: meta, DisableContentSha256: true, SendContentMd5: false, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) delete(args, "metadata") } logSuccess(testName, function, args, startTime) } func testGetObjectAttributes() { startTime := time.Now() testName := getFuncName() function := "GetObjectAttributes(ctx, bucketName, objectName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.ObjectAttributesOptions{}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket( context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } bucketNameV := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-versioned-") args["bucketName"] = bucketNameV err = c.MakeBucket( context.Background(), bucketNameV, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketNameV) if err != nil { logError(testName, function, args, startTime, "", "Unable to enable versioning", err) return } defer cleanupBucket(bucketName, c) defer cleanupVersionedBucket(bucketNameV, c) testFiles := make(map[string]*objectAttributesNewObject) testFiles["file1"] = &objectAttributesNewObject{ Object: "file1", ObjectReaderType: "datafile-1.03-MB", Bucket: bucketNameV, ContentType: "custom/contenttype", SendContentMd5: false, } testFiles["file2"] = &objectAttributesNewObject{ Object: "file2", ObjectReaderType: "datafile-129-MB", Bucket: bucketName, ContentType: "custom/contenttype", SendContentMd5: false, } for i, v := range testFiles { bufSize := dataFileMap[v.ObjectReaderType] reader := getDataReader(v.ObjectReaderType) args["objectName"] = v.Object testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: v.ContentType, SendContentMd5: v.SendContentMd5, Checksum: minio.ChecksumCRC32C, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } } testTable := make(map[string]objectAttributesTableTest) testTable["none-versioned"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{}, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["0-to-0-marker"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 0, MaxParts: 0, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["0-marker-to-max"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 0, MaxParts: 10000, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["0-to-1-marker"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 0, MaxParts: 1, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["7-to-6-marker"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{ PartNumberMarker: 7, MaxParts: 6, }, test: objectAttributesTestOptions{ TestFileName: "file2", StorageClass: "STANDARD", HasFullChecksum: true, HasPartChecksums: true, HasParts: true, }, } testTable["versioned"] = objectAttributesTableTest{ opts: minio.ObjectAttributesOptions{}, test: objectAttributesTestOptions{ TestFileName: "file1", StorageClass: "STANDARD", HasFullChecksum: true, }, } for i, v := range testTable { tf, ok := testFiles[v.test.TestFileName] if !ok { continue } args["objectName"] = tf.Object args["bucketName"] = tf.Bucket if tf.UploadInfo.VersionID != "" { v.opts.VersionID = tf.UploadInfo.VersionID } s, err := c.GetObjectAttributes(context.Background(), tf.Bucket, tf.Object, v.opts) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) return } v.test.NumberOfParts = s.ObjectParts.PartsCount v.test.ETag = tf.UploadInfo.ETag v.test.ObjectSize = int(tf.UploadInfo.Size) err = validateObjectAttributeRequest(s, &v.opts, &v.test) if err != nil { logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed, table test: "+i, err) return } } logSuccess(testName, function, args, startTime) } func testGetObjectAttributesSSECEncryption() { startTime := time.Now() testName := getFuncName() function := "GetObjectAttributes(ctx, bucketName, objectName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.ObjectAttributesOptions{}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket( context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := "encrypted-object" args["objectName"] = objectName bufSize := dataFileMap["datafile-11-MB"] reader := getDataReader("datafile-11-MB") sse := encrypt.DefaultPBKDF([]byte("word1 word2 word3 word4"), []byte(bucketName+objectName)) info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: "content/custom", SendContentMd5: false, ServerSideEncryption: sse, PartSize: uint64(bufSize) / 2, Checksum: minio.ChecksumCRC32C, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } opts := minio.ObjectAttributesOptions{ ServerSideEncryption: sse, } attr, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty bucket name should have failed", nil) return } err = validateObjectAttributeRequest(attr, &opts, &objectAttributesTestOptions{ TestFileName: info.Key, ETag: info.ETag, NumberOfParts: 2, ObjectSize: int(info.Size), HasFullChecksum: true, HasParts: true, HasPartChecksums: true, }) if err != nil { logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err) return } logSuccess(testName, function, args, startTime) } func testGetObjectAttributesErrorCases() { startTime := time.Now() testName := getFuncName() function := "GetObjectAttributes(ctx, bucketName, objectName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.ObjectAttributesOptions{}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } unknownBucket := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-bucket-") unknownObject := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-object-") _, err = c.GetObjectAttributes(context.Background(), unknownBucket, unknownObject, minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", nil) return } errorResponse := err.(minio.ErrorResponse) if errorResponse.Code != minio.NoSuchBucket { logError(testName, function, args, startTime, "", "Invalid error code, expected NoSuchBucket but got "+errorResponse.Code, nil) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket( context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } bucketNameV := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-versioned-") args["bucketName"] = bucketNameV err = c.MakeBucket( context.Background(), bucketNameV, minio.MakeBucketOptions{Region: "us-east-1"}, ) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } err = c.EnableVersioning(context.Background(), bucketNameV) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) defer cleanupVersionedBucket(bucketNameV, c) _, err = c.GetObjectAttributes(context.Background(), bucketName, unknownObject, minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes failed", nil) return } errorResponse = err.(minio.ErrorResponse) if errorResponse.Code != minio.NoSuchKey { logError(testName, function, args, startTime, "", "Invalid error code, expected "+minio.NoSuchKey+" but got "+errorResponse.Code, nil) return } _, err = c.GetObjectAttributes(context.Background(), bucketName, "", minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty object name should have failed", nil) return } _, err = c.GetObjectAttributes(context.Background(), "", unknownObject, minio.ObjectAttributesOptions{}) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty bucket name should have failed", nil) return } _, err = c.GetObjectAttributes(context.Background(), bucketNameV, unknownObject, minio.ObjectAttributesOptions{ VersionID: uuid.NewString(), }) if err == nil { logError(testName, function, args, startTime, "", "GetObjectAttributes with empty bucket name should have failed", nil) return } errorResponse = err.(minio.ErrorResponse) if errorResponse.Code != minio.NoSuchVersion { logError(testName, function, args, startTime, "", "Invalid error code, expected "+minio.NoSuchVersion+" but got "+errorResponse.Code, nil) return } logSuccess(testName, function, args, startTime) } type objectAttributesNewObject struct { Object string ObjectReaderType string Bucket string ContentType string SendContentMd5 bool UploadInfo minio.UploadInfo } type objectAttributesTableTest struct { opts minio.ObjectAttributesOptions test objectAttributesTestOptions } type objectAttributesTestOptions struct { TestFileName string ETag string NumberOfParts int StorageClass string ObjectSize int HasPartChecksums bool HasFullChecksum bool HasParts bool } func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.ObjectAttributesOptions, test *objectAttributesTestOptions) (err error) { if opts.VersionID != "" { if OA.VersionID != opts.VersionID { err = fmt.Errorf("Expected versionId %s but got versionId %s", opts.VersionID, OA.VersionID) return err } } partsMissingChecksum := false foundPartChecksum := false for _, v := range OA.ObjectParts.Parts { checksumFound := false if v.ChecksumSHA256 != "" { checksumFound = true } else if v.ChecksumSHA1 != "" { checksumFound = true } else if v.ChecksumCRC32 != "" { checksumFound = true } else if v.ChecksumCRC32C != "" { checksumFound = true } else if v.ChecksumCRC64NVME != "" { checksumFound = true } if !checksumFound { partsMissingChecksum = true } else { foundPartChecksum = true } } if test.HasPartChecksums { if partsMissingChecksum { err = fmt.Errorf("One or all parts were missing a checksum") return err } } else { if foundPartChecksum { err = fmt.Errorf("Did not expect ObjectParts to have checksums but found one") return err } } hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" || OA.Checksum.ChecksumCRC32C != "" || OA.Checksum.ChecksumCRC64NVME != "" || OA.Checksum.ChecksumSHA1 != "" || OA.Checksum.ChecksumSHA256 != "") if test.HasFullChecksum { if !hasFullObjectChecksum { err = fmt.Errorf("Full object checksum not found") return err } } else { if hasFullObjectChecksum { err = fmt.Errorf("Did not expect a full object checksum but we got one") return err } } if OA.ETag != test.ETag { err = fmt.Errorf("Etags do not match, got %s but expected %s", OA.ETag, test.ETag) return err } if test.HasParts { if len(OA.ObjectParts.Parts) < 1 { err = fmt.Errorf("Was expecting ObjectParts but none were present") return err } } if OA.StorageClass == "" { err = fmt.Errorf("Was expecting a StorageClass but got none") return err } if OA.ObjectSize != test.ObjectSize { err = fmt.Errorf("Was expecting a ObjectSize but got none") return err } if test.HasParts { if opts.MaxParts == 0 { if len(OA.ObjectParts.Parts) != OA.ObjectParts.PartsCount { err = fmt.Errorf("expected %s parts but got %d", OA.ObjectParts.PartsCount, len(OA.ObjectParts.Parts)) return err } } else if (opts.MaxParts + opts.PartNumberMarker) > OA.ObjectParts.PartsCount { if len(OA.ObjectParts.Parts) != (OA.ObjectParts.PartsCount - opts.PartNumberMarker) { err = fmt.Errorf("expected %d parts but got %d", (OA.ObjectParts.PartsCount - opts.PartNumberMarker), len(OA.ObjectParts.Parts)) return err } } else if opts.MaxParts != 0 { if opts.MaxParts != len(OA.ObjectParts.Parts) { err = fmt.Errorf("expected %d parts but got %d", opts.MaxParts, len(OA.ObjectParts.Parts)) return err } } } if OA.ObjectParts.NextPartNumberMarker == OA.ObjectParts.PartsCount { if OA.ObjectParts.IsTruncated { err = fmt.Errorf("Expected ObjectParts to NOT be truncated, but it was") return err } } if OA.ObjectParts.NextPartNumberMarker != OA.ObjectParts.PartsCount { if !OA.ObjectParts.IsTruncated { err = fmt.Errorf("Expected ObjectParts to be truncated, but it was NOT") return err } } return err } // Test PutObject using a large data to trigger multipart readat func testPutObjectWithMetadata() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", } if !isFullMode() { logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") return } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "Make bucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Object custom metadata customContentType := "custom/contenttype" args["metadata"] = map[string][]string{ "Content-Type": {customContentType}, "X-Amz-Meta-CustomKey": {"extra spaces in value"}, } _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: customContentType, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) return } if st.ContentType != customContentType && st.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected "+customContentType+" got "+st.ContentType, err) return } if err := crcMatchesName(r, "datafile-129-MB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) return } logSuccess(testName, function, args, startTime) } func testPutObjectWithContentLanguage() { // initialize logging params objectName := "test-object" startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": objectName, "size": -1, "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) data := []byte{} _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(data), int64(0), minio.PutObjectOptions{ ContentLanguage: "en", }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if objInfo.Metadata.Get("Content-Language") != "en" { logError(testName, function, args, startTime, "", "Expected content-language 'en' doesn't match with StatObject return value", err) return } logSuccess(testName, function, args, startTime) } // Test put object with streaming signature. func testPutObjectStreaming() { // initialize logging params objectName := "test-object" startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size,opts)" args := map[string]interface{}{ "bucketName": "", "objectName": objectName, "size": -1, "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload an object. sizes := []int64{0, 64*1024 - 1, 64 * 1024} for _, size := range sizes { data := newRandomReader(size, size) ui, err := c.PutObject(context.Background(), bucketName, objectName, data, int64(size), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return } if ui.Size != size { logError(testName, function, args, startTime, "", "PutObjectStreaming result has unexpected size", nil) return } objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if objInfo.Size != size { logError(testName, function, args, startTime, "", "Unexpected size", err) return } } logSuccess(testName, function, args, startTime) } // Test PutObject with preconditions on non-existent objects func testPutObjectPreconditionOnNonExistent() { startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts) with preconditions" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{SetMatchETag/SetMatchETagExcept}", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Test 1: PutObject with SetMatchETag on non-existent object should fail objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object-") args["objectName"] = objectName data := bytes.NewReader([]byte("test data")) opts := minio.PutObjectOptions{} opts.SetMatchETag("some-etag") _, err = c.PutObject(context.Background(), bucketName, objectName, data, int64(data.Len()), opts) if err == nil { logError(testName, function, args, startTime, "", "PutObject with SetMatchETag on non-existent object should have failed", nil) return } errResp := minio.ToErrorResponse(err) if errResp.Code != "NoSuchKey" { logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects), got %s", errResp.Code), err) return } // Test 2: PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should succeed objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object2-") args["objectName"] = objectName2 data2 := bytes.NewReader([]byte("test data 2")) opts2 := minio.PutObjectOptions{} opts2.SetMatchETagExcept("some-etag") _, err = c.PutObject(context.Background(), bucketName, objectName2, data2, int64(data2.Len()), opts2) if err != nil { logError(testName, function, args, startTime, "", "PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should have succeeded", err) return } // Test 3: CompleteMultipartUpload with preconditions on non-existent object should fail objectName3 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-multipart-") args["objectName"] = objectName3 data3 := bytes.Repeat([]byte("a"), 5*1024*1024+1) reader3 := bytes.NewReader(data3) opts3 := minio.PutObjectOptions{} opts3.SetMatchETag("non-existent-etag") _, err = c.PutObject(context.Background(), bucketName, objectName3, reader3, int64(len(data3)), opts3) if err == nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload with SetMatchETag on non-existent object should have failed", nil) return } errResp = minio.ToErrorResponse(err) if errResp.Code != "NoSuchKey" { logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects) for multipart, got %s", errResp.Code), err) return } logSuccess(testName, function, args, startTime) } // Test get object seeker from the end, using whence set to '2'. func testGetObjectSeekEnd() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes read does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } pos, err := r.Seek(-100, 2) if err != nil { logError(testName, function, args, startTime, "", "Object Seek failed", err) return } if pos != st.Size-100 { logError(testName, function, args, startTime, "", "Incorrect position", err) return } buf2 := make([]byte, 100) m, err := readFull(r, buf2) if err != nil { logError(testName, function, args, startTime, "", "Error reading through readFull", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "Number of bytes dont match, expected "+string(len(buf2))+" got "+string(m), err) return } hexBuf1 := fmt.Sprintf("%02x", buf[len(buf)-100:]) hexBuf2 := fmt.Sprintf("%02x", buf2[:m]) if hexBuf1 != hexBuf2 { logError(testName, function, args, startTime, "", "Values at same index dont match", err) return } pos, err = r.Seek(-100, 2) if err != nil { logError(testName, function, args, startTime, "", "Object Seek failed", err) return } if pos != st.Size-100 { logError(testName, function, args, startTime, "", "Incorrect position", err) return } if err = r.Close(); err != nil { logError(testName, function, args, startTime, "", "ObjectClose failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object reader to not throw error on being closed twice. func testGetObjectClosedTwice() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } if err := crcMatchesName(r, "datafile-33-kB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Object Close failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Already closed object. No error returned", err) return } logSuccess(testName, function, args, startTime) } // Test RemoveObjects request where context cancels after timeout func testRemoveObjectsContext() { // Initialize logging params. startTime := time.Now() testName := getFuncName() function := "RemoveObjects(ctx, bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } // Instantiate new minio client. c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate put data. r := bytes.NewReader(bytes.Repeat([]byte("a"), 8)) // Multi remove of 20 objects. nrObjects := 20 objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, r, 8, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } objectsCh <- minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, } } }() // Set context to cancel in 1 nanosecond. ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() // Call RemoveObjects API with short timeout. errorCh := c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{}) // Check for error. select { case r := <-errorCh: if r.Err == nil { logError(testName, function, args, startTime, "", "RemoveObjects should fail on short timeout", err) return } } // Set context with longer timeout. ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() // Perform RemoveObjects with the longer timeout. Expect the removals to succeed. errorCh = c.RemoveObjects(ctx, bucketName, objectsCh, minio.RemoveObjectsOptions{}) select { case r, more := <-errorCh: if more || r.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error", r.Err) return } } logSuccess(testName, function, args, startTime) } // Test removing multiple objects with Remove API func testRemoveMultipleObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) r := bytes.NewReader(bytes.Repeat([]byte("a"), 1)) // Multi remove of 1100 objects nrObjects := 1100 objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, r, 1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } objectsCh <- minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, } } }() // Call RemoveObjects API errorCh := c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) // Check if errorCh doesn't receive any error select { case r, more := <-errorCh: if more { logError(testName, function, args, startTime, "", "Unexpected error", r.Err) return } } logSuccess(testName, function, args, startTime) } // Test removing multiple objects with Remove API as iterator func testRemoveMultipleObjectsIter() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) buf := []byte("a") // Multi remove of 1100 objects nrObjects := 1100 objectsIter := func() iter.Seq[minio.ObjectInfo] { return func(yield func(minio.ObjectInfo) bool) { // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), 1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) continue } if !yield(minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, }) { return } } } } // Call RemoveObjects API results, err := c.RemoveObjectsWithIter(context.Background(), bucketName, objectsIter(), minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Unexpected error", err) return } for result := range results { if result.Err != nil { logError(testName, function, args, startTime, "", "Unexpected error", result.Err) return } } logSuccess(testName, function, args, startTime) } // Test removing multiple objects and check for results func testRemoveMultipleObjectsWithResult() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupVersionedBucket(bucketName, c) buf := []byte("a") nrObjects := 10 nrLockedObjects := 5 objectsCh := make(chan minio.ObjectInfo) go func() { defer close(objectsCh) // Upload objects and send them to objectsCh for i := 0; i < nrObjects; i++ { objectName := "sample" + strconv.Itoa(i) + ".txt" info, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), 1, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } if i < nrLockedObjects { // t := time.Date(2130, time.April, 25, 14, 0, 0, 0, time.UTC) t := time.Now().Add(5 * time.Minute) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: false, RetainUntilDate: &t, Mode: &m, VersionID: info.VersionID, } err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "Error setting retention", err) return } } objectsCh <- minio.ObjectInfo{ Key: info.Key, VersionID: info.VersionID, } } }() // Call RemoveObjects API resultCh := c.RemoveObjectsWithResult(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) var foundNil, foundErr int for { // Check if errorCh doesn't receive any error select { case deleteRes, ok := <-resultCh: if !ok { goto out } if deleteRes.ObjectName == "" { logError(testName, function, args, startTime, "", "Unexpected object name", nil) return } if deleteRes.ObjectVersionID == "" { logError(testName, function, args, startTime, "", "Unexpected object version ID", nil) return } if deleteRes.Err == nil { foundNil++ } else { foundErr++ } } } out: if foundNil+foundErr != nrObjects { logError(testName, function, args, startTime, "", "Unexpected number of results", nil) return } if foundNil != nrObjects-nrLockedObjects { logError(testName, function, args, startTime, "", "Unexpected number of nil errors", nil) return } if foundErr != nrLockedObjects { logError(testName, function, args, startTime, "", "Unexpected number of errors", nil) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject of a big file to trigger multipart func testFPutObjectMultipart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 4 parts to utilize all 3 'workers' in multipart and still have a part to upload. fileName := getMintDataDirFilePath("datafile-129-MB") if fileName == "" { // Make a temp file with minPartSize bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } // Upload 2 parts to utilize all 3 'workers' in multipart and still have a part to upload. if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } fileName = file.Name() args["fileName"] = fileName } totalSize := dataFileMap["datafile-129-MB"] // Set base object name objectName := bucketName + "FPutObject" + "-standard" args["objectName"] = objectName objectContentType := "testapplication/octet-stream" args["objectContentType"] = objectContentType // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ContentType: objectContentType}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Unexpected error", err) return } if objInfo.Size != int64(totalSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(totalSize))+" got "+string(objInfo.Size), err) return } if objInfo.ContentType != objectContentType && objInfo.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType doesn't match", err) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject with null contentType (default = application/octet-stream) func testFPutObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") location := "us-east-1" // Make a new bucket. args["bucketName"] = bucketName args["location"] = location function = "MakeBucket(bucketName, location)" err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 3 parts worth of data to use all 3 of multiparts 'workers' and have an extra part. // Use different data in part for multipart tests to check parts are uploaded in correct order. fName := getMintDataDirFilePath("datafile-129-MB") if fName == "" { // Make a temp file with minPartSize bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } // Upload 3 parts to utilize all 3 'workers' in multipart and still have a part to upload. if _, err = io.Copy(file, getDataReader("datafile-129-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } // Close the file pro-actively for windows. if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } defer os.Remove(file.Name()) fName = file.Name() } // Set base object name function = "FPutObject(bucketName, objectName, fileName, opts)" objectName := bucketName + "FPutObject" args["objectName"] = objectName + "-standard" args["fileName"] = fName args["opts"] = minio.PutObjectOptions{ContentType: "application/octet-stream"} // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) ui, err := c.FPutObject(context.Background(), bucketName, objectName+"-standard", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } if ui.Size != int64(dataFileMap["datafile-129-MB"]) { logError(testName, function, args, startTime, "", "FPutObject returned an unexpected upload size", err) return } // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } srcFile, err := os.Open(fName) if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } defer srcFile.Close() // Add extension to temp file name tmpFile, err := os.Create(fName + ".gtar") if err != nil { logError(testName, function, args, startTime, "", "File create failed", err) return } _, err = io.Copy(tmpFile, srcFile) if err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } tmpFile.Close() // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-GTar" args["opts"] = minio.PutObjectOptions{} _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Check headers function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-standard" rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rStandard.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rStandard.ContentType, err) return } function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-Octet" rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rOctet.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "ContentType does not match, expected application/octet-stream, got "+rOctet.ContentType, err) return } function = "StatObject(bucketName, objectName, opts)" args["objectName"] = objectName + "-GTar" rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" && rGTar.ContentType != "application/x-tar" { logError(testName, function, args, startTime, "", "ContentType does not match, expected application/x-tar or application/octet-stream, got "+rGTar.ContentType, err) return } os.Remove(fName + ".gtar") logSuccess(testName, function, args, startTime) } // Tests FPutObject request when context cancels after timeout func testFPutObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 1 parts worth of data to use multipart upload. // Use different data in part for multipart tests to check parts are uploaded in correct order. fName := getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } // Upload 1 parts to trigger multipart upload if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } // Close the file pro-actively for windows. if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } defer os.Remove(file.Name()) fName = file.Name() } // Set base object name objectName := bucketName + "FPutObjectContext" args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() // Perform FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err == nil { logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Perform FPutObject with a long timeout. Expect the put object to succeed _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on long timeout", err) return } _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject request when context cancels after timeout func testFPutObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObjectContext(ctx, bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "opts": "minio.PutObjectOptions{ContentType:objectContentType}", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload 1 parts worth of data to use multipart upload. // Use different data in part for multipart tests to check parts are uploaded in correct order. fName := getMintDataDirFilePath("datafile-1-MB") if fName == "" { // Make a temp file with 1 MiB bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectContextTest") if err != nil { logError(testName, function, args, startTime, "", "Temp file creation failed", err) return } // Upload 1 parts to trigger multipart upload if _, err = io.Copy(file, getDataReader("datafile-1-MB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } // Close the file pro-actively for windows. if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } defer os.Remove(file.Name()) fName = file.Name() } // Set base object name objectName := bucketName + "FPutObjectContext" args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() // Perform FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(ctx, bucketName, objectName+"-Shorttimeout", fName, minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err == nil { logError(testName, function, args, startTime, "", "FPutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Perform FPutObject with a long timeout. Expect the put object to succeed _, err = c.FPutObject(ctx, bucketName, objectName+"-Longtimeout", fName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject shouldn't fail on longer timeout", err) return } _, err = c.StatObject(context.Background(), bucketName, objectName+"-Longtimeout", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } logSuccess(testName, function, args, startTime) } // Test validates putObject with context to see if request cancellation is honored. func testPutObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(ctx, bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "opts": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) cancel() args["ctx"] = ctx args["opts"] = minio.PutObjectOptions{ContentType: "binary/octet-stream"} _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err == nil { logError(testName, function, args, startTime, "", "PutObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() reader = getDataReader("datafile-33-kB") defer reader.Close() _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err) return } logSuccess(testName, function, args, startTime) } // Tests get object with s3zip extensions. func testGetObjectS3Zip() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{"x-minio-extract": true} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + ".zip" args["objectName"] = objectName var zipFile bytes.Buffer zw := zip.NewWriter(&zipFile) rng := rand.New(rand.NewSource(0xc0cac01a)) const nFiles = 500 for i := 0; i <= nFiles; i++ { if i == nFiles { // Make one large, compressible file. i = 1000000 } b := make([]byte, i) if i < nFiles { rng.Read(b) } wc, err := zw.Create(fmt.Sprintf("test/small/file-%d.bin", i)) if err != nil { logError(testName, function, args, startTime, "", "zw.Create failed", err) return } wc.Write(b) } err = zw.Close() if err != nil { logError(testName, function, args, startTime, "", "zw.Close failed", err) return } buf := zipFile.Bytes() // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(len(buf))+", got "+string(st.Size), err) return } r.Close() zr, err := zip.NewReader(bytes.NewReader(buf), int64(len(buf))) if err != nil { logError(testName, function, args, startTime, "", "zip.NewReader failed", err) return } lOpts := minio.ListObjectsOptions{} lOpts.Set("x-minio-extract", "true") lOpts.Prefix = objectName + "/" lOpts.Recursive = true list := c.ListObjects(context.Background(), bucketName, lOpts) listed := map[string]minio.ObjectInfo{} for item := range list { if item.Err != nil { break } listed[item.Key] = item } if len(listed) == 0 { // Assume we are running against non-minio. args["SKIPPED"] = true logIgnored(testName, function, args, startTime, "s3zip does not appear to be present") return } for _, file := range zr.File { if file.FileInfo().IsDir() { continue } args["zipfile"] = file.Name zfr, err := file.Open() if err != nil { logError(testName, function, args, startTime, "", "file.Open failed", err) return } want, err := io.ReadAll(zfr) if err != nil { logError(testName, function, args, startTime, "", "fzip file read failed", err) return } opts := minio.GetObjectOptions{} opts.Set("x-minio-extract", "true") key := path.Join(objectName, file.Name) r, err = c.GetObject(context.Background(), bucketName, key, opts) if err != nil { terr := minio.ToErrorResponse(err) if terr.StatusCode != http.StatusNotFound { logError(testName, function, args, startTime, "", "GetObject failed", err) } return } got, err := io.ReadAll(r) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } r.Close() if !bytes.Equal(want, got) { logError(testName, function, args, startTime, "", "Content mismatch", err) return } oi, ok := listed[key] if !ok { logError(testName, function, args, startTime, "", "Object Missing", fmt.Errorf("%s not present in listing", key)) return } if int(oi.Size) != len(got) { logError(testName, function, args, startTime, "", "Object Size Incorrect", fmt.Errorf("listing %d, read %d", oi.Size, len(got))) return } delete(listed, key) } delete(args, "zipfile") if len(listed) > 0 { logError(testName, function, args, startTime, "", "Extra listed objects", fmt.Errorf("left over: %v", listed)) return } logSuccess(testName, function, args, startTime) } // Tests get object ReaderSeeker interface methods. func testGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } // This following function helps us to compare data from the reader after seek // with the data from the original buffer cmpData := func(r io.Reader, start, end int) { if end-start == 0 { return } buffer := bytes.NewBuffer([]byte{}) if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "CopyN failed", err) return } } if !bytes.Equal(buf[start:end], buffer.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } } // Generic seek error for errors other than io.EOF seekErr := errors.New("seek error") testCases := []struct { offset int64 whence int pos int64 err error shouldCmp bool start int end int }{ // Start from offset 0, fetch data and compare {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, // Start from offset larger than possible {int64(bufSize) + 1024, 0, 0, seekErr, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, // Move larger than possible {int64(bufSize), 1, 0, seekErr, false, 0, 0}, // Provide negative offset with CUR_SEEK {int64(-1), 1, 0, seekErr, false, 0, 0}, // Test with whence SEEK_END and with positive offset {1024, 2, int64(bufSize) - 1024, io.EOF, true, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, seekErr, true, 0, 0}, } for i, testCase := range testCases { // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) // We expect an error if testCase.err == seekErr && err == nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) return } // We expect a specific error if testCase.err != seekErr && testCase.err != err { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", unexpected err value: expected: "+testCase.err.Error()+", found: "+err.Error(), err) return } // If we expect an error go to the next loop if testCase.err != nil { continue } // Check the returned seek pos if n != testCase.pos { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", number of bytes seeked does not match, expected "+string(testCase.pos)+", got "+string(n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } } logSuccess(testName, function, args, startTime) } // Tests get object ReaderAt interface methods. func testGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf1) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 readOffset := 0 bufRead := make([]byte, 512) // Read (again) using the regular read function. // Should not have been affected by ReadAt. m, err = io.ReadFull(r, bufRead) if err != nil { logError(testName, function, args, startTime, "", "ReadFull failed", err) return } if m != len(bufRead) { logError(testName, function, args, startTime, "", "ReadFull read shorter bytes before reaching EOF, expected "+string(len(bufRead))+", got "+string(m), err) return } if !bytes.Equal(bufRead, buf[readOffset:readOffset+len(bufRead)]) { logError(testName, function, args, startTime, "", "Incorrect Read from offset", err) return } readOffset += len(bufRead) st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } // Read (again) using the regular read function. // Should not have been affected by ReadAt. m, err = io.ReadFull(r, bufRead) if err != nil { logError(testName, function, args, startTime, "", "ReadFull (2) failed", err) return } if m != len(bufRead) { logError(testName, function, args, startTime, "", "ReadFull read shorter bytes before reaching EOF", err) return } if !bytes.Equal(bufRead, buf[readOffset:readOffset+len(bufRead)]) { logError(testName, function, args, startTime, "", "Incorrect Read from offset", err) return } readOffset += len(bufRead) offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // Reproduces issue https://github.com/minio/minio-go/issues/1137 func testGetObjectReadAtWhenEOFWasReached() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read directly buf1 := make([]byte, len(buf)) buf2 := make([]byte, 512) m, err := r.Read(buf1) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "Read failed", err) return } } if m != len(buf1) { logError(testName, function, args, startTime, "", "Read read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf) { logError(testName, function, args, startTime, "", "Incorrect count of Read data", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, 512) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[512:1024]) { logError(testName, function, args, startTime, "", "Incorrect count of ReadAt data", err) return } logSuccess(testName, function, args, startTime) } // Test Presigned Post Policy func testPresignedPostPolicy() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PresignedPostPolicy(policy)" args := map[string]interface{}{ "policy": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Azure requires the key to not start with a number metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } policy := minio.NewPostPolicy() policy.SetBucket(bucketName) policy.SetKey(objectName) policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days policy.SetContentType("binary/octet-stream") policy.SetContentLengthRange(10, 1024*1024) policy.SetUserMetadata(metadataKey, metadataValue) policy.SetContentEncoding("gzip") // Add CRC32C checksum := minio.ChecksumCRC32C.ChecksumBytes(buf) err = policy.SetChecksum(checksum) if err != nil { logError(testName, function, args, startTime, "", "SetChecksum failed", err) return } args["policy"] = policy.String() presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return } var formBuf bytes.Buffer writer := multipart.NewWriter(&formBuf) for k, v := range formData { writer.WriteField(k, v) } // Get a 33KB file to upload and test if set post policy works filePath := getMintDataDirFilePath("datafile-33-kB") if filePath == "" { // Make a temp file with 33 KB data. file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } filePath = file.Name() } // add file to post request f, err := os.Open(filePath) defer f.Close() if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } w, err := writer.CreateFormFile("file", filePath) if err != nil { logError(testName, function, args, startTime, "", "CreateFormFile failed", err) return } _, err = io.Copy(w, f) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } writer.Close() httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } req.Header.Set("Content-Type", writer.FormDataContentType()) // make post request with correct form data res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } defer res.Body.Close() if res.StatusCode != http.StatusNoContent { logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) return } // expected path should be absolute path of the object var scheme string if mustParseBool(os.Getenv(enableHTTPS)) { scheme = "https://" } else { scheme = "http://" } expectedLocation := scheme + os.Getenv(serverEndpoint) + "/" + bucketName + "/" + objectName expectedLocationBucketDNS := scheme + bucketName + "." + os.Getenv(serverEndpoint) + "/" + objectName if !strings.Contains(expectedLocation, ".amazonaws.com/") { // Test when not against AWS S3. if val, ok := res.Header["Location"]; ok { if val[0] != expectedLocation && val[0] != expectedLocationBucketDNS { logError(testName, function, args, startTime, "", fmt.Sprintf("Location in header response is incorrect. Want %q or %q, got %q", expectedLocation, expectedLocationBucketDNS, val[0]), err) return } } else { logError(testName, function, args, startTime, "", "Location not found in header response", err) return } } wantChecksumCrc32c := checksum.Encoded() if got := res.Header.Get("X-Amz-Checksum-Crc32c"); got != wantChecksumCrc32c { logError(testName, function, args, startTime, "", fmt.Sprintf("Want checksum %q, got %q", wantChecksumCrc32c, got), nil) return } // Ensure that when we subsequently GetObject, the checksum is returned gopts := minio.GetObjectOptions{Checksum: true} r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.ChecksumCRC32C != wantChecksumCrc32c { logError(testName, function, args, startTime, "", fmt.Sprintf("Want checksum %s, got %s", wantChecksumCrc32c, st.ChecksumCRC32C), nil) return } logSuccess(testName, function, args, startTime) } // testPresignedPostPolicyWrongFile tests that when we have a policy with a checksum, we cannot POST the wrong file func testPresignedPostPolicyWrongFile() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PresignedPostPolicy(policy)" args := map[string]interface{}{ "policy": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Azure requires the key to not start with a number metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") policy := minio.NewPostPolicy() policy.SetBucket(bucketName) policy.SetKey(objectName) policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days policy.SetContentType("binary/octet-stream") policy.SetContentLengthRange(10, 1024*1024) policy.SetUserMetadata(metadataKey, metadataValue) // Add CRC32C of some data that the policy will explicitly allow. checksum := minio.ChecksumCRC32C.ChecksumBytes([]byte{0x01, 0x02, 0x03}) err = policy.SetChecksum(checksum) if err != nil { logError(testName, function, args, startTime, "", "SetChecksum failed", err) return } args["policy"] = policy.String() presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return } // At this stage, we have a policy that allows us to upload for a specific checksum. // Test that uploading datafile-10-kB, with a different checksum, fails as expected filePath := getMintDataDirFilePath("datafile-10-kB") if filePath == "" { // Make a temp file with 10 KB data. file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } if _, err = io.Copy(file, getDataReader("datafile-10-kB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } filePath = file.Name() } fileReader := getDataReader("datafile-10-kB") defer fileReader.Close() buf10k, err := io.ReadAll(fileReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } otherChecksum := minio.ChecksumCRC32C.ChecksumBytes(buf10k) var formBuf bytes.Buffer writer := multipart.NewWriter(&formBuf) for k, v := range formData { if k == "x-amz-checksum-crc32c" { v = otherChecksum.Encoded() } writer.WriteField(k, v) } // Add file to post request f, err := os.Open(filePath) defer f.Close() if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } w, err := writer.CreateFormFile("file", filePath) if err != nil { logError(testName, function, args, startTime, "", "CreateFormFile failed", err) return } _, err = io.Copy(w, f) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } writer.Close() httpClient := &http.Client{ Timeout: 30 * time.Second, Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) if err != nil { logError(testName, function, args, startTime, "", "HTTP request failed", err) return } req.Header.Set("Content-Type", writer.FormDataContentType()) // Make the POST request with the form data. res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request failed", err) return } defer res.Body.Close() if res.StatusCode != http.StatusForbidden { logError(testName, function, args, startTime, "", "HTTP request unexpected status", errors.New(res.Status)) return } // Read the response body, ensure it has checksum failure message resBody, err := io.ReadAll(res.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Normalize the response body, because S3 uses quotes around the policy condition components // in the error message, MinIO does not. resBodyStr := strings.ReplaceAll(string(resBody), `"`, "") if !strings.Contains(resBodyStr, "Policy Condition failed: [eq, $x-amz-checksum-crc32c, 8TDyHg=") { logError(testName, function, args, startTime, "", "Unexpected response body", errors.New(resBodyStr)) return } logSuccess(testName, function, args, startTime) } // testPresignedPostPolicyEmptyFileName tests that an empty file name in the presigned post policy func testPresignedPostPolicyEmptyFileName() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PresignedPostPolicy(policy)" args := map[string]interface{}{ "policy": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") // Azure requires the key to not start with a number metadataKey := randString(60, rand.NewSource(time.Now().UnixNano()), "user") metadataValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } policy := minio.NewPostPolicy() policy.SetBucket(bucketName) policy.SetKey(objectName) policy.SetExpires(time.Now().UTC().AddDate(0, 0, 10)) // expires in 10 days policy.SetContentType("binary/octet-stream") policy.SetContentLengthRange(10, 1024*1024) policy.SetUserMetadata(metadataKey, metadataValue) policy.SetContentEncoding("gzip") // Add CRC32C checksum := minio.ChecksumCRC32C.ChecksumBytes(buf) err = policy.SetChecksum(checksum) if err != nil { logError(testName, function, args, startTime, "", "SetChecksum failed", err) return } args["policy"] = policy.String() presignedPostPolicyURL, formData, err := c.PresignedPostPolicy(context.Background(), policy) if err != nil { logError(testName, function, args, startTime, "", "PresignedPostPolicy failed", err) return } var formBuf bytes.Buffer writer := multipart.NewWriter(&formBuf) for k, v := range formData { writer.WriteField(k, v) } // Get a 33KB file to upload and test if set post policy works filePath := getMintDataDirFilePath("datafile-33-kB") if filePath == "" { // Make a temp file with 33 KB data. file, err := os.CreateTemp(os.TempDir(), "PresignedPostPolicyTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } if _, err = io.Copy(file, getDataReader("datafile-33-kB")); err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if err = file.Close(); err != nil { logError(testName, function, args, startTime, "", "File Close failed", err) return } filePath = file.Name() } // add file to post request f, err := os.Open(filePath) defer f.Close() if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } w, err := writer.CreateFormFile("", filePath) if err != nil { logError(testName, function, args, startTime, "", "CreateFormFile failed", err) return } _, err = io.Copy(w, f) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } writer.Close() httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() req, err := http.NewRequest(http.MethodPost, presignedPostPolicyURL.String(), bytes.NewReader(formBuf.Bytes())) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } req.Header.Set("Content-Type", writer.FormDataContentType()) // make post request with correct form data res, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "Http request failed", err) return } defer res.Body.Close() if res.StatusCode != http.StatusBadRequest { logError(testName, function, args, startTime, "", "Http request failed", errors.New(res.Status)) return } body, err := io.ReadAll(res.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !strings.Contains(string(body), "MalformedPOSTRequest") { logError(testName, function, args, startTime, "", "Invalid error from server", errors.New(string(body))) } logSuccess(testName, function, args, startTime) } // Tests copy object func testCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(dst, src)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a new bucket in 'us-east-1' (destination bucket). err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName+"-copy", c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } // Copy Source src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, // Set copy conditions. MatchETag: objInfo.ETag, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), } args["src"] = src dst := minio.CopyDestOptions{ Bucket: bucketName + "-copy", Object: objectName + "-copy", } // Perform the Copy if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Source object r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } objInfoCopy, err := readerCopy.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if objInfo.Size != objInfoCopy.Size { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+", got "+string(objInfo.Size), err) return } if err := crcMatchesName(r, "datafile-33-kB"); err != nil { logError(testName, function, args, startTime, "", "data CRC check failed", err) return } if err := crcMatchesName(readerCopy, "datafile-33-kB"); err != nil { logError(testName, function, args, startTime, "", "copy data CRC check failed", err) return } // Close all the get readers before proceeding with CopyObject operations. r.Close() readerCopy.Close() // CopyObject again but with wrong conditions src = minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), NoMatchETag: objInfo.ETag, } // Perform the Copy which should fail _, err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return } src = minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, } dst = minio.CopyDestOptions{ Bucket: bucketName, Object: objectName, ReplaceMetadata: true, UserMetadata: map[string]string{ "Copy": "should be same", }, } args["dst"] = dst args["src"] = src _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject shouldn't fail", err) return } oi, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } stOpts := minio.StatObjectOptions{} stOpts.SetMatchETag(oi.ETag) objInfo, err = c.StatObject(context.Background(), bucketName, objectName, stOpts) if err != nil { logError(testName, function, args, startTime, "", "CopyObject ETag should match and not fail", err) return } if objInfo.Metadata.Get("x-amz-meta-copy") != "should be same" { logError(testName, function, args, startTime, "", "CopyObject modified metadata should match", err) return } logSuccess(testName, function, args, startTime) } // Tests SSE-C get object ReaderSeeker interface methods. func testSSECEncryptedGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } // This following function helps us to compare data from the reader after seek // with the data from the original buffer cmpData := func(r io.Reader, start, end int) { if end-start == 0 { return } buffer := bytes.NewBuffer([]byte{}) if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "CopyN failed", err) return } } if !bytes.Equal(buf[start:end], buffer.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } } testCases := []struct { offset int64 whence int pos int64 err error shouldCmp bool start int end int }{ // Start from offset 0, fetch data and compare {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, // Start from offset larger than possible {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, // Move larger than possible {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, // Provide negative offset with CUR_SEEK {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, // Test with whence SEEK_END and with positive offset {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, // Test with invalid whence {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, } for i, testCase := range testCases { // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) if err != nil && testCase.err == nil { // We expected success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err == nil && testCase.err != nil { // We expected failure, but got success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err != nil && testCase.err != nil { if err.Error() != testCase.err.Error() { // We expect a specific error logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } } // Check the returned seek pos if n != testCase.pos { logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } } logSuccess(testName, function, args, startTime) } // Tests SSE-S3 get object ReaderSeeker interface methods. func testSSES3EncryptedGetObjectReadSeekFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer func() { // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } }() // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat object failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } // This following function helps us to compare data from the reader after seek // with the data from the original buffer cmpData := func(r io.Reader, start, end int) { if end-start == 0 { return } buffer := bytes.NewBuffer([]byte{}) if _, err := io.CopyN(buffer, r, int64(bufSize)); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "CopyN failed", err) return } } if !bytes.Equal(buf[start:end], buffer.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } } testCases := []struct { offset int64 whence int pos int64 err error shouldCmp bool start int end int }{ // Start from offset 0, fetch data and compare {0, 0, 0, nil, true, 0, 0}, // Start from offset 2048, fetch data and compare {2048, 0, 2048, nil, true, 2048, bufSize}, // Start from offset larger than possible {int64(bufSize) + 1024, 0, 0, io.EOF, false, 0, 0}, // Move to offset 0 without comparing {0, 0, 0, nil, false, 0, 0}, // Move one step forward and compare {1, 1, 1, nil, true, 1, bufSize}, // Move larger than possible {int64(bufSize), 1, 0, io.EOF, false, 0, 0}, // Provide negative offset with CUR_SEEK {int64(-1), 1, 0, fmt.Errorf("Negative position not allowed for 1"), false, 0, 0}, // Test with whence SEEK_END and with positive offset {1024, 2, 0, io.EOF, false, 0, 0}, // Test with whence SEEK_END and with negative offset {-1024, 2, int64(bufSize) - 1024, nil, true, bufSize - 1024, bufSize}, // Test with whence SEEK_END and with large negative offset {-int64(bufSize) * 2, 2, 0, fmt.Errorf("Seeking at negative offset not allowed for 2"), false, 0, 0}, // Test with invalid whence {0, 3, 0, fmt.Errorf("Invalid whence 3"), false, 0, 0}, } for i, testCase := range testCases { // Perform seek operation n, err := r.Seek(testCase.offset, testCase.whence) if err != nil && testCase.err == nil { // We expected success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err == nil && testCase.err != nil { // We expected failure, but got success. logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } if err != nil && testCase.err != nil { if err.Error() != testCase.err.Error() { // We expect a specific error logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, unexpected err value: expected: %s, found: %s", i+1, testCase.err, err), err) return } } // Check the returned seek pos if n != testCase.pos { logError(testName, function, args, startTime, "", fmt.Sprintf("Test %d, number of bytes seeked does not match, expected %d, got %d", i+1, testCase.pos, n), err) return } // Compare only if shouldCmp is activated if testCase.shouldCmp { cmpData(r, testCase.start, testCase.end) } } logSuccess(testName, function, args, startTime) } // Tests SSE-C get object ReaderAt interface methods. func testSSECEncryptedGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ ServerSideEncryption: encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } defer r.Close() offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf1) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // Tests SSE-S3 get object ReaderAt interface methods. func testSSES3EncryptedGetObjectReadAtFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 129MiB of data. bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ContentType: "binary/octet-stream", ServerSideEncryption: encrypt.NewSSE(), }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } defer r.Close() offset := int64(2048) // read directly buf1 := make([]byte, 512) buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) // Test readAt before stat is called such that objectInfo doesn't change. m, err := r.ReadAt(buf1, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf1) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf1))+", got "+string(m), err) return } if !bytes.Equal(buf1, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+", got "+string(st.Size), err) return } m, err = r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+", got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+", got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+", got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, len(buf)) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+", got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, len(buf)+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // testSSECEncryptionPutGet tests encryption with customer provided encryption keys func testSSECEncryptionPutGet() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutEncryptedObject(bucketName, objectName, reader, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } const password = "correct horse battery staple" // https://xkcd.com/936/ for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) args["sse"] = sse // Put encrypted data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } logSuccess(testName, function, args, startTime) } logSuccess(testName, function, args, startTime) } // TestEncryptionFPut tests encryption with customer specified encryption keys func testSSECEncryptionFPut() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "filePath": "", "contentType": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Object custom metadata customContentType := "custom/contenttype" args["metadata"] = customContentType testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 0)}, {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } const password = "correct horse battery staple" // https://xkcd.com/936/ for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) args["sse"] = sse // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "file create failed", err) return } _, err = file.Write(testCase.buf) if err != nil { logError(testName, function, args, startTime, "", "file write failed", err) return } file.Close() // Put encrypted data if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } os.Remove(fileName) } logSuccess(testName, function, args, startTime) } // testSSES3EncryptionPutGet tests SSE-S3 encryption func testSSES3EncryptionPutGet() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutEncryptedObject(bucketName, objectName, reader, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.NewSSE() args["sse"] = sse // Put encrypted data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(testCase.buf), int64(len(testCase.buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutEncryptedObject failed", err) return } // Read the data back without any encryption headers r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } logSuccess(testName, function, args, startTime) } logSuccess(testName, function, args, startTime) } // TestSSES3EncryptionFPut tests server side encryption func testSSES3EncryptionFPut() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutEncryptedObject(bucketName, objectName, filePath, contentType, sse)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "filePath": "", "contentType": "", "sse": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Object custom metadata customContentType := "custom/contenttype" args["metadata"] = customContentType testCases := []struct { buf []byte }{ {buf: bytes.Repeat([]byte("F"), 0)}, {buf: bytes.Repeat([]byte("F"), 1)}, {buf: bytes.Repeat([]byte("F"), 15)}, {buf: bytes.Repeat([]byte("F"), 16)}, {buf: bytes.Repeat([]byte("F"), 17)}, {buf: bytes.Repeat([]byte("F"), 31)}, {buf: bytes.Repeat([]byte("F"), 32)}, {buf: bytes.Repeat([]byte("F"), 33)}, {buf: bytes.Repeat([]byte("F"), 1024)}, {buf: bytes.Repeat([]byte("F"), 1024*2)}, {buf: bytes.Repeat([]byte("F"), 1024*1024)}, } for i, testCase := range testCases { // Generate a random object name objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Secured object sse := encrypt.NewSSE() args["sse"] = sse // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "file create failed", err) return } _, err = file.Write(testCase.buf) if err != nil { logError(testName, function, args, startTime, "", "file write failed", err) return } file.Close() // Put encrypted data if _, err = c.FPutObject(context.Background(), bucketName, objectName, fileName, minio.PutObjectOptions{ServerSideEncryption: sse}); err != nil { logError(testName, function, args, startTime, "", "FPutEncryptedObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetEncryptedObject failed", err) return } defer r.Close() // Compare the sent object with the received one recvBuffer := bytes.NewBuffer([]byte{}) if _, err = io.Copy(recvBuffer, r); err != nil { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", error: "+err.Error(), err) return } if recvBuffer.Len() != len(testCase.buf) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Number of bytes of received object does not match, expected "+string(len(testCase.buf))+", got "+string(recvBuffer.Len()), err) return } if !bytes.Equal(testCase.buf, recvBuffer.Bytes()) { logError(testName, function, args, startTime, "", "Test "+string(i+1)+", Encrypted sent is not equal to decrypted, expected "+string(testCase.buf)+", got "+string(recvBuffer.Bytes()), err) return } os.Remove(fileName) } logSuccess(testName, function, args, startTime) } func testBucketNotification() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "SetBucketNotification(bucketName)" args := map[string]interface{}{ "bucketName": "", } if os.Getenv("NOTIFY_BUCKET") == "" || os.Getenv("NOTIFY_SERVICE") == "" || os.Getenv("NOTIFY_REGION") == "" || os.Getenv("NOTIFY_ACCOUNTID") == "" || os.Getenv("NOTIFY_RESOURCE") == "" { logIgnored(testName, function, args, startTime, "Skipped notification test as it is not configured") return } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } bucketName := os.Getenv("NOTIFY_BUCKET") args["bucketName"] = bucketName topicArn := notification.NewArn("aws", os.Getenv("NOTIFY_SERVICE"), os.Getenv("NOTIFY_REGION"), os.Getenv("NOTIFY_ACCOUNTID"), os.Getenv("NOTIFY_RESOURCE")) queueArn := notification.NewArn("aws", "dummy-service", "dummy-region", "dummy-accountid", "dummy-resource") topicConfig := notification.NewConfig(topicArn) topicConfig.AddEvents(notification.ObjectCreatedAll, notification.ObjectRemovedAll) topicConfig.AddFilterSuffix("jpg") queueConfig := notification.NewConfig(queueArn) queueConfig.AddEvents(notification.ObjectCreatedAll) queueConfig.AddFilterPrefix("photos/") config := notification.Configuration{} config.AddTopic(topicConfig) // Add the same topicConfig again, should have no effect // because it is duplicated config.AddTopic(topicConfig) if len(config.TopicConfigs) != 1 { logError(testName, function, args, startTime, "", "Duplicate entry added", err) return } // Add and remove a queue config config.AddQueue(queueConfig) config.RemoveQueueByArn(queueArn) err = c.SetBucketNotification(context.Background(), bucketName, config) if err != nil { logError(testName, function, args, startTime, "", "SetBucketNotification failed", err) return } config, err = c.GetBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketNotification failed", err) return } if len(config.TopicConfigs) != 1 { logError(testName, function, args, startTime, "", "Topic config is empty", err) return } if config.TopicConfigs[0].Filter.S3Key.FilterRules[0].Value != "jpg" { logError(testName, function, args, startTime, "", "Couldn't get the suffix", err) return } err = c.RemoveAllBucketNotification(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveAllBucketNotification failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Tests comprehensive list of all methods. func testFunctional() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "testFunctional()" functionAll := "" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, nil, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket. function = "MakeBucket(bucketName, region)" functionAll = "MakeBucket(bucketName, region)" args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) defer cleanupBucket(bucketName, c) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "File creation failed", err) return } for i := 0; i < 3; i++ { buf := make([]byte, rand.Intn(1<<19)) _, err = file.Write(buf) if err != nil { logError(testName, function, args, startTime, "", "File write failed", err) return } } file.Close() // Verify if bucket exits and you have access. var exists bool function = "BucketExists(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "BucketExists failed", err) return } if !exists { logError(testName, function, args, startTime, "", "Could not find the bucket", err) return } // Asserting the default bucket policy. function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } nilPolicy, err := c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } if nilPolicy != "" { logError(testName, function, args, startTime, "", "policy should be set to nil", err) return } // Set the bucket policy to 'public readonly'. function = "SetBucketPolicy(bucketName, readOnlyPolicy)" functionAll += ", " + function readOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": readOnlyPolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, readOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readonly`. function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } // Make the bucket 'public writeonly'. function = "SetBucketPolicy(bucketName, writeOnlyPolicy)" functionAll += ", " + function writeOnlyPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": writeOnlyPolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, writeOnlyPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `writeonly`. function = "GetBucketPolicy(ctx, bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } // Make the bucket 'public read/write'. function = "SetBucketPolicy(bucketName, readWritePolicy)" functionAll += ", " + function readWritePolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:ListBucket","s3:ListBucketMultipartUploads"],"Resource":["arn:aws:s3:::` + bucketName + `"]}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": readWritePolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // should return policy `readwrite`. function = "GetBucketPolicy(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } _, err = c.GetBucketPolicy(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketPolicy failed", err) return } // List all buckets. function = "ListBuckets()" functionAll += ", " + function args = nil buckets, err := c.ListBuckets(context.Background()) if len(buckets) == 0 { logError(testName, function, args, startTime, "", "Found bucket list to be empty", err) return } if err != nil { logError(testName, function, args, startTime, "", "ListBuckets failed", err) return } // Verify if previously created bucket is listed in list buckets. bucketFound := false for _, bucket := range buckets { if bucket.Name == bucketName { bucketFound = true } } // If bucket not found error out. if !bucketFound { logError(testName, function, args, startTime, "", "Bucket: "+bucketName+" not found", err) return } objectName := bucketName + "unique" // Generate data buf := bytes.Repeat([]byte("f"), 1<<19) function = "PutObject(bucketName, objectName, reader, contentType)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "contentType": "", } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-nolength", "contentType": "binary/octet-stream", } _, err = c.PutObject(context.Background(), bucketName, objectName+"-nolength", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Instantiate a done channel to close all listing. doneCh := make(chan struct{}) defer close(doneCh) objFound := false isRecursive := true // Recursive is true. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: true}) { if obj.Key == objectName { objFound = true break } } if !objFound { logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) return } objFound = false isRecursive = true // Recursive is true. function = "ListObjects()" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break } } if !objFound { logError(testName, function, args, startTime, "", "Object "+objectName+" not found", err) return } incompObjNotFound := true function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) { if objIncompl.Key != "" { incompObjNotFound = false break } } if !incompObjNotFound { logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) return } function = "GetObject(bucketName, objectName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, } newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } newReadBytes, err := io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "GetObject bytes mismatch", err) return } newReader.Close() function = "FGetObject(bucketName, objectName, fileName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "fileName": fileName + "-f", } err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject failed", err) return } function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", "expires": 3600 * time.Second, } if _, err = c.PresignedHeadObject(context.Background(), bucketName, "", 3600*time.Second, nil); err == nil { logError(testName, function, args, startTime, "", "PresignedHeadObject success", err) return } // Generate presigned HEAD object url. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return } transport := createHTTPTransport() if err != nil { logError(testName, function, args, startTime, "", "DefaultTransport failed", err) return } httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: transport, } req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject request was incorrect", err) return } // Verify if presigned url works. resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect, status "+string(resp.StatusCode), err) return } if resp.Header.Get("ETag") == "" { logError(testName, function, args, startTime, "", "PresignedHeadObject response incorrect", err) return } resp.Body.Close() function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", "expires": 3600 * time.Second, } _, err = c.PresignedGetObject(context.Background(), bucketName, "", 3600*time.Second, nil) if err == nil { logError(testName, function, args, startTime, "", "PresignedGetObject success", err) return } // Generate presigned GET object url. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) return } newPresignedBytes, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } resp.Body.Close() if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } // Set request parameters. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, "reqParams": reqParams, } presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect, status "+string(resp.StatusCode), err) return } newPresignedBytes, err = io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch for presigned GET URL", err) return } if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { logError(testName, function, args, startTime, "", "wrong Content-Disposition received "+string(resp.Header.Get("Content-Disposition")), err) return } function = "PresignedPutObject(bucketName, objectName, expires)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": "", "expires": 3600 * time.Second, } _, err = c.PresignedPutObject(context.Background(), bucketName, "", 3600*time.Second) if err == nil { logError(testName, function, args, startTime, "", "PresignedPutObject success", err) return } function = "PresignedPutObject(bucketName, objectName, expires)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", "expires": 3600 * time.Second, } presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return } buf = bytes.Repeat([]byte("g"), 1<<19) req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "Couldn't make HTTP request with PresignedPutObject URL", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return } newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject after PresignedPutObject failed", err) return } newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll after GetObject failed", err) return } if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)" functionAll += ", " + function presignExtraHeaders := map[string][]string{ "mysecret": {"abcxxx"}, } args = map[string]interface{}{ "method": "PUT", "bucketName": bucketName, "objectName": objectName + "-presign-custom", "expires": 3600 * time.Second, "extraHeaders": presignExtraHeaders, } presignedURL, err := c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders) if err != nil { logError(testName, function, args, startTime, "", "Presigned failed", err) return } // Generate data more than 32K buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) req, err = http.NewRequest(http.MethodPut, presignedURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err) return } req.Header.Add("mysecret", "abcxxx") resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to Presigned URL failed", err) return } // Download the uploaded object to verify args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presign-custom", } newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presign-custom", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of uploaded custom-presigned object failed", err) return } newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed during get on custom-presigned put object", err) return } newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch on custom-presigned object upload verification", err) return } function = "RemoveObject(bucketName, objectName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, } err = c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-f" err = c.RemoveObject(context.Background(), bucketName, objectName+"-f", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-nolength" err = c.RemoveObject(context.Background(), bucketName, objectName+"-nolength", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-presigned" err = c.RemoveObject(context.Background(), bucketName, objectName+"-presigned", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } args["objectName"] = objectName + "-presign-custom" err = c.RemoveObject(context.Background(), bucketName, objectName+"-presign-custom", minio.RemoveObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "RemoveObject failed", err) return } function = "RemoveBucket(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } err = c.RemoveBucket(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveBucket failed", err) return } err = c.RemoveBucket(context.Background(), bucketName) if err == nil { logError(testName, function, args, startTime, "", "RemoveBucket did not fail for invalid bucket name", err) return } if err.Error() != "The specified bucket does not exist" { logError(testName, function, args, startTime, "", "RemoveBucket failed", err) return } os.Remove(fileName) os.Remove(fileName + "-f") logSuccess(testName, functionAll, args, startTime) } // Test for validating GetObject Reader* methods functioning when the // object is modified in the object store. func testGetObjectModified() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload an object. objectName := "myobject" args["objectName"] = objectName content := "helloworld" _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(content), int64(len(content)), minio.PutObjectOptions{ContentType: "application/text"}) if err != nil { logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) return } defer c.RemoveObject(context.Background(), bucketName, objectName, minio.RemoveObjectOptions{}) reader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Failed to GetObject "+objectName+", from bucket "+bucketName, err) return } defer reader.Close() // Read a few bytes of the object. b := make([]byte, 5) n, err := reader.ReadAt(b, 0) if err != nil { logError(testName, function, args, startTime, "", "Failed to read object "+objectName+", from bucket "+bucketName+" at an offset", err) return } // Upload different contents to the same object while object is being read. newContent := "goodbyeworld" _, err = c.PutObject(context.Background(), bucketName, objectName, strings.NewReader(newContent), int64(len(newContent)), minio.PutObjectOptions{ContentType: "application/text"}) if err != nil { logError(testName, function, args, startTime, "", "Failed to upload "+objectName+", to bucket "+bucketName, err) return } // Confirm that a Stat() call in between doesn't change the Object's cached etag. _, err = reader.Stat() expectedError := "At least one of the pre-conditions you specified did not hold." if err.Error() != expectedError { logError(testName, function, args, startTime, "", "Expected Stat to fail with error "+expectedError+", but received "+err.Error(), err) return } // Read again only to find object contents have been modified since last read. _, err = reader.ReadAt(b, int64(n)) if err.Error() != expectedError { logError(testName, function, args, startTime, "", "Expected ReadAt to fail with error "+expectedError+", but received "+err.Error(), err) return } logSuccess(testName, function, args, startTime) } // Test validates putObject to upload a file seeked at a given offset. func testPutObjectUploadSeekedObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, fileToUpload, contentType)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileToUpload": "", "contentType": "binary/octet-stream", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) var tempfile *os.File if fileName := getMintDataDirFilePath("datafile-100-kB"); fileName != "" { tempfile, err = os.Open(fileName) if err != nil { logError(testName, function, args, startTime, "", "File open failed", err) return } args["fileToUpload"] = fileName } else { tempfile, err = os.CreateTemp("", "minio-go-upload-test-") if err != nil { logError(testName, function, args, startTime, "", "TempFile create failed", err) return } args["fileToUpload"] = tempfile.Name() // Generate 100kB data if _, err = io.Copy(tempfile, getDataReader("datafile-100-kB")); err != nil { logError(testName, function, args, startTime, "", "File copy failed", err) return } defer os.Remove(tempfile.Name()) // Seek back to the beginning of the file. tempfile.Seek(0, 0) } length := 100 * humanize.KiByte objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) args["objectName"] = objectName offset := length / 2 if _, err = tempfile.Seek(int64(offset), 0); err != nil { logError(testName, function, args, startTime, "", "TempFile seek failed", err) return } _, err = c.PutObject(context.Background(), bucketName, objectName, tempfile, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } tempfile.Close() obj, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer obj.Close() n, err := obj.Seek(int64(offset), 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != int64(offset) { logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(offset), n), err) return } _, err = c.PutObject(context.Background(), bucketName, objectName+"getobject", obj, int64(length-offset), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName+"getobject", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(length-offset) { logError(testName, function, args, startTime, "", fmt.Sprintf("Invalid offset returned, expected %d got %d", int64(length-offset), n), err) return } logSuccess(testName, function, args, startTime) } // Tests bucket re-create errors. func testMakeBucketErrorV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" args := map[string]interface{}{ "bucketName": "", "region": "eu-west-1", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") region := "eu-west-1" args["bucketName"] = bucketName args["region"] = region // Make a new bucket in 'eu-west-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: region}); err == nil { logError(testName, function, args, startTime, "", "MakeBucket did not fail for existing bucket name", err) return } // Verify valid error response from server. if minio.ToErrorResponse(err).Code != minio.BucketAlreadyExists && minio.ToErrorResponse(err).Code != minio.BucketAlreadyOwnedByYou { logError(testName, function, args, startTime, "", "Invalid error returned by server", err) return } logSuccess(testName, function, args, startTime) } // Test get object reader to not throw error on being closed twice. func testGetObjectClosedTwiceV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" args := map[string]interface{}{ "bucketName": "", "region": "eu-west-1", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if err := r.Close(); err == nil { logError(testName, function, args, startTime, "", "Object is already closed, should return error", err) return } logSuccess(testName, function, args, startTime) } // Tests FPutObject hidden contentType setting func testFPutObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FPutObject(bucketName, objectName, fileName, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "fileName": "", "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a temp file with 11*1024*1024 bytes of data. file, err := os.CreateTemp(os.TempDir(), "FPutObjectTest") if err != nil { logError(testName, function, args, startTime, "", "TempFile creation failed", err) return } r := bytes.NewReader(bytes.Repeat([]byte("b"), 11*1024*1024)) n, err := io.CopyN(file, r, 11*1024*1024) if err != nil { logError(testName, function, args, startTime, "", "Copy failed", err) return } if n != int64(11*1024*1024) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(int64(11*1024*1024))+" got "+string(n), err) return } // Close the file pro-actively for windows. err = file.Close() if err != nil { logError(testName, function, args, startTime, "", "File close failed", err) return } // Set base object name objectName := bucketName + "FPutObject" args["objectName"] = objectName args["fileName"] = file.Name() // Perform standard FPutObject with contentType provided (Expecting application/octet-stream) _, err = c.FPutObject(context.Background(), bucketName, objectName+"-standard", file.Name(), minio.PutObjectOptions{ContentType: "application/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Perform FPutObject with no contentType provided (Expecting application/octet-stream) args["objectName"] = objectName + "-Octet" args["contentType"] = "" _, err = c.FPutObject(context.Background(), bucketName, objectName+"-Octet", file.Name(), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Add extension to temp file name fileName := file.Name() err = os.Rename(fileName, fileName+".gtar") if err != nil { logError(testName, function, args, startTime, "", "Rename failed", err) return } // Perform FPutObject with no contentType provided (Expecting application/x-gtar) args["objectName"] = objectName + "-Octet" args["contentType"] = "" args["fileName"] = fileName + ".gtar" _, err = c.FPutObject(context.Background(), bucketName, objectName+"-GTar", fileName+".gtar", minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FPutObject failed", err) return } // Check headers and sizes rStandard, err := c.StatObject(context.Background(), bucketName, objectName+"-standard", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rStandard.Size != 11*1024*1024 { logError(testName, function, args, startTime, "", "Unexpected size", nil) return } if rStandard.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rStandard.ContentType, err) return } rOctet, err := c.StatObject(context.Background(), bucketName, objectName+"-Octet", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rOctet.ContentType != "application/octet-stream" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/octet-stream , got "+rOctet.ContentType, err) return } if rOctet.Size != 11*1024*1024 { logError(testName, function, args, startTime, "", "Unexpected size", nil) return } rGTar, err := c.StatObject(context.Background(), bucketName, objectName+"-GTar", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if rGTar.Size != 11*1024*1024 { logError(testName, function, args, startTime, "", "Unexpected size", nil) return } if rGTar.ContentType != "application/x-gtar" && rGTar.ContentType != "application/octet-stream" && rGTar.ContentType != "application/x-tar" { logError(testName, function, args, startTime, "", "Content-Type headers mismatched, expected: application/x-tar , got "+rGTar.ContentType, err) return } os.Remove(fileName + ".gtar") logSuccess(testName, function, args, startTime) } // Tests various bucket supported formats. func testMakeBucketRegionsV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "MakeBucket(bucketName, region)" args := map[string]interface{}{ "bucketName": "", "region": "eu-west-1", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'eu-central-1'. if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "eu-west-1"}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } if err = cleanupBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err) return } // Make a new bucket with '.' in its name, in 'us-west-2'. This // request is internally staged into a path style instead of // virtual host style. if err = c.MakeBucket(context.Background(), bucketName+".withperiod", minio.MakeBucketOptions{Region: "us-west-2"}); err != nil { args["bucketName"] = bucketName + ".withperiod" args["region"] = "us-west-2" logError(testName, function, args, startTime, "", "MakeBucket test with a bucket name with period, '.', failed", err) return } // Delete all objects and buckets if err = cleanupBucket(bucketName+".withperiod", c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed while removing bucket recursively", err) return } logSuccess(testName, function, args, startTime) } // Tests get object ReaderSeeker interface methods. func testGetObjectReadSeekFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(int64(bufSize))+" got "+string(st.Size), err) return } offset := int64(2048) n, err := r.Seek(offset, 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != offset { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) return } n, err = r.Seek(0, 1) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != offset { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset)+" got "+string(n), err) return } _, err = r.Seek(offset, 2) if err == nil { logError(testName, function, args, startTime, "", "Seek on positive offset for whence '2' should error out", err) return } n, err = r.Seek(-offset, 2) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != st.Size-offset { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(st.Size-offset)+" got "+string(n), err) return } var buffer1 bytes.Buffer if _, err = io.CopyN(&buffer1, r, st.Size); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "Copy failed", err) return } } if !bytes.Equal(buf[len(buf)-int(offset):], buffer1.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } // Seek again and read again. n, err = r.Seek(offset-1, 0) if err != nil { logError(testName, function, args, startTime, "", "Seek failed", err) return } if n != (offset - 1) { logError(testName, function, args, startTime, "", "Number of seeked bytes does not match, expected "+string(offset-1)+" got "+string(n), err) return } var buffer2 bytes.Buffer if _, err = io.CopyN(&buffer2, r, st.Size); err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "Copy failed", err) return } } // Verify now lesser bytes. if !bytes.Equal(buf[2047:], buffer2.Bytes()) { logError(testName, function, args, startTime, "", "Incorrect read bytes v/s original buffer", err) return } logSuccess(testName, function, args, startTime) } // Tests get object ReaderAt interface methods. func testGetObjectReadAtFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(bucketName, objectName)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName buf, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } // Save the data _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer r.Close() st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } offset := int64(2048) // Read directly buf2 := make([]byte, 512) buf3 := make([]byte, 512) buf4 := make([]byte, 512) m, err := r.ReadAt(buf2, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf2) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf2))+" got "+string(m), err) return } if !bytes.Equal(buf2, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf3, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf3) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf3))+" got "+string(m), err) return } if !bytes.Equal(buf3, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } offset += 512 m, err = r.ReadAt(buf4, offset) if err != nil { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } if m != len(buf4) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf4))+" got "+string(m), err) return } if !bytes.Equal(buf4, buf[offset:offset+512]) { logError(testName, function, args, startTime, "", "Incorrect read between two ReadAt from same offset", err) return } buf5 := make([]byte, bufSize) // Read the whole object. m, err = r.ReadAt(buf5, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } if m != len(buf5) { logError(testName, function, args, startTime, "", "ReadAt read shorter bytes before reaching EOF, expected "+string(len(buf5))+" got "+string(m), err) return } if !bytes.Equal(buf, buf5) { logError(testName, function, args, startTime, "", "Incorrect data read in GetObject, than what was previously uploaded", err) return } buf6 := make([]byte, bufSize+1) // Read the whole object and beyond. _, err = r.ReadAt(buf6, 0) if err != nil { if err != io.EOF { logError(testName, function, args, startTime, "", "ReadAt failed", err) return } } logSuccess(testName, function, args, startTime) } // Tests copy object func testCopyObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a new bucket in 'us-east-1' (destination bucket). err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName+"-copy", c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } r.Close() // Copy Source src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), MatchETag: objInfo.ETag, } args["source"] = src // Set copy conditions. dst := minio.CopyDestOptions{ Bucket: bucketName + "-copy", Object: objectName + "-copy", } args["destination"] = dst // Perform the Copy _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Source object r, err = c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Destination object readerCopy, err := c.GetObject(context.Background(), bucketName+"-copy", objectName+"-copy", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } // Check the various fields of source object against destination object. objInfo, err = r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } objInfoCopy, err := readerCopy.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } if objInfo.Size != objInfoCopy.Size { logError(testName, function, args, startTime, "", "Number of bytes does not match, expected "+string(objInfoCopy.Size)+" got "+string(objInfo.Size), err) return } // Close all the readers. r.Close() readerCopy.Close() // CopyObject again but with wrong conditions src = minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchUnmodifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), NoMatchETag: objInfo.ETag, } // Perform the Copy which should fail _, err = c.CopyObject(context.Background(), dst, src) if err == nil { logError(testName, function, args, startTime, "", "CopyObject did not fail for invalid conditions", err) return } logSuccess(testName, function, args, startTime) } // Tests copy object with various checksum scenarios, tries to not repeat CopyObjectV2 test and // instead just focus on Checksum. func testCopyObjectWithChecksums() { startTime := time.Now() testName := getFuncName() function := "CopyObjectWithChecksums(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Make a new bucket in 'us-east-1' (destination bucket). err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName+"-copy", c) // Generate 33K of data. bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // PutObject to upload the object to the bucket, this object will have a Crc64NVME checksum applied // by default since nothing was explicitly specified. objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // GetObject to obtain the eTag r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } r.Close() // Copy source options src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), MatchETag: objInfo.ETag, } tests := []struct { csType minio.ChecksumType cs wantChecksums }{ {csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, {csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}}, {csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}}, {csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}}, {csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}}, // In S3, all copied objects without checksums and specified destination checksum algorithms // automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case. {csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, } for _, test := range tests { args := map[string]interface{}{} args["srcOpts"] = src args["section"] = "setup" args["checksum"] = test.csType.String() // Copy destination options bucketCopyName := bucketName + "-copy" objectCopyName := objectName + "-copy-" + test.csType.String() dst := minio.CopyDestOptions{ Bucket: bucketCopyName, Object: objectCopyName, ReplaceMetadata: true, } if test.csType != minio.ChecksumNone { // Request the server-side checksum on the copy. // ChecksumNone is a flag to leave off the header dst.ChecksumType = test.csType } args["destOpts"] = dst // Perform the Copy args["section"] = "CopyObject" _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Checksum verification args["section"] = "HeadObject" st, err := c.StatObject(context.Background(), bucketCopyName, objectCopyName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.ChecksumMode != "FULL_OBJECT" { logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil) return } err = cmpChecksum(st, test.cs) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch", err) return } logSuccess(testName, function, args, startTime) } } // Tests replacing an object with CopyObject and a new Checksum type func testReplaceObjectWithChecksums() { startTime := time.Now() testName := getFuncName() function := "CopyObjectWithChecksums(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) tests := []struct { csType minio.ChecksumType cs wantChecksums }{ {csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, {csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}}, {csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}}, {csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}}, {csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}}, // In S3, all copied objects without checksums and specified destination checksum algorithms // automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case. {csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}}, } for _, test := range tests { args := map[string]interface{}{} args["section"] = "setup" args["destOpts"] = "" args["checksum"] = test.csType.String() bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // PutObject to upload the object to the bucket objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // GetObject to obtain the eTag r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } objInfo, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return } r.Close() // Copy source options src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC), MatchETag: objInfo.ETag, } // Copy destination options, overwrite the existing object dst := minio.CopyDestOptions{ Bucket: bucketName, Object: objectName, // S3 requires that we send some new metadata otherwise it complains that the // CopyObject is illegal. UserMetadata: map[string]string{ "TestMeta": objectName + "-meta-" + test.csType.String(), }, ReplaceMetadata: true, } if test.csType != minio.ChecksumNone { // Request the server-side checksum on the copy. // ChecksumNone is a flag to leave off the header dst.ChecksumType = test.csType } args["destOpts"] = dst // Perform the Copy args["section"] = "CopyObject" _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Checksum verification args["section"] = "HeadObject" st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.ChecksumMode != "FULL_OBJECT" { logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil) return } err = cmpChecksum(st, test.cs) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch", err) return } logSuccess(testName, function, args, startTime) } } func testComposeObjectErrorCasesWrapper(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Test that more than 10K source objects cannot be // concatenated. srcArr := [10001]minio.CopySrcOptions{} srcSlice := srcArr[:] dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "object", } args["destination"] = dst // Just explain about srcArr in args["sourceList"] // to stop having 10,001 null headers logged args["sourceList"] = "source array of 10,001 elements" if _, err := c.ComposeObject(context.Background(), dst, srcSlice...); err == nil { logError(testName, function, args, startTime, "", "Expected error in ComposeObject", err) return } else if err.Error() != "There must be as least one and up to 10000 source objects." { logError(testName, function, args, startTime, "", "Got unexpected error", err) return } // Create a source with invalid offset spec and check that // error is returned: // 1. Create the source object. const badSrcSize = 5 * 1024 * 1024 buf := bytes.Repeat([]byte("1"), badSrcSize) _, err = c.PutObject(context.Background(), bucketName, "badObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // 2. Set invalid range spec on the object (going beyond // object size) badSrc := minio.CopySrcOptions{ Bucket: bucketName, Object: "badObject", MatchRange: true, Start: 1, End: badSrcSize, } // 3. ComposeObject call should fail. if _, err := c.ComposeObject(context.Background(), dst, badSrc); err == nil { logError(testName, function, args, startTime, "", "ComposeObject expected to fail", err) return } else if !strings.Contains(err.Error(), "has invalid segment-to-copy") { logError(testName, function, args, startTime, "", "Got invalid error", err) return } logSuccess(testName, function, args, startTime) } // Test expected error cases func testComposeObjectErrorCasesV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } testComposeObjectErrorCasesWrapper(c) } func testComposeMultipleSources(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{ "destination": "", "sourceList": "", } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Upload a small source object const srcSize = 1024 * 1024 * 5 buf := bytes.Repeat([]byte("1"), srcSize) _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(srcSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // We will append 10 copies of the object. srcs := []minio.CopySrcOptions{} for i := 0; i < 10; i++ { srcs = append(srcs, minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", }) } // make the last part very small srcs[9].MatchRange = true args["sourceList"] = srcs dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject", } args["destination"] = dst ui, err := c.ComposeObject(context.Background(), dst, srcs...) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } if ui.Size != 9*srcSize+1 { logError(testName, function, args, startTime, "", "ComposeObject returned unexpected size", err) return } objProps, err := c.StatObject(context.Background(), bucketName, "dstObject", minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if objProps.Size != 9*srcSize+1 { logError(testName, function, args, startTime, "", "Size mismatched! Expected "+string(10000*srcSize)+" got "+string(objProps.Size), err) return } logSuccess(testName, function, args, startTime) } // Test concatenating multiple 10K objects V2 func testCompose10KSourcesV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } testComposeMultipleSources(c) } func testEncryptedEmptyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, objectSize, opts)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) sse := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"object")) // 1. create an sse-c encrypted object to copy by uploading const srcSize = 0 var buf []byte // Empty buffer args["objectName"] = "object" _, err = c.PutObject(context.Background(), bucketName, "object", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ServerSideEncryption: sse}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } // 2. Test CopyObject for an empty object src := minio.CopySrcOptions{ Bucket: bucketName, Object: "object", Encryption: sse, } dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "new-object", Encryption: sse, } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { function = "CopyObject(dst, src)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject failed", err) return } // 3. Test Key rotation newSSE := encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"new-object")) src = minio.CopySrcOptions{ Bucket: bucketName, Object: "new-object", Encryption: sse, } dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "new-object", Encryption: newSSE, } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { function = "CopyObject(dst, src)" logError(testName, function, map[string]interface{}{}, startTime, "", "CopyObject with key rotation failed", err) return } // 4. Download the object. reader, err := c.GetObject(context.Background(), bucketName, "new-object", minio.GetObjectOptions{ServerSideEncryption: newSSE}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer reader.Close() decBytes, err := io.ReadAll(reader) if err != nil { logError(testName, function, map[string]interface{}{}, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, map[string]interface{}{}, startTime, "", "Downloaded object doesn't match the empty encrypted object", err) return } delete(args, "objectName") logSuccess(testName, function, args, startTime) } func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc, sseDst encrypt.ServerSide) { // initialize logging params startTime := time.Now() testName := getFuncNameLoc(2) function := "CopyObject(destination, source)" args := map[string]interface{}{} args["testName"] = testName var srcEncryption, dstEncryption encrypt.ServerSide // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // 1. create an sse-c encrypted object to copy by uploading const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB // Calculate the CRC32C checksum for the object meta := map[string]string{} h := minio.ChecksumCRC32C.Hasher() h.Reset() h.Write(buf) meta[minio.ChecksumCRC32C.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ ServerSideEncryption: sseSrc, DisableMultipart: true, DisableContentSha256: true, UserMetadata: meta, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } if sseSrc != nil && sseSrc.Type() != encrypt.S3 { srcEncryption = sseSrc } // 2. copy object and change encryption key src := minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", Encryption: srcEncryption, } args["source"] = src dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject", Encryption: sseDst, } args["destination"] = dst _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } if sseDst != nil && sseDst.Type() != encrypt.S3 { dstEncryption = sseDst } // 3. get copied object and check if content is equal coreClient := minio.Core{Client: c} reader, oi, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption, Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } decBytes, err := io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } reader.Close() err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="}) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch on dstObject", err) return } // Test key rotation for source object in-place. var newSSE encrypt.ServerSide if sseSrc != nil && sseSrc.Type() == encrypt.SSEC { newSSE = encrypt.DefaultPBKDF([]byte("Don't Panic"), []byte(bucketName+"srcObject")) // replace key } if sseSrc != nil && sseSrc.Type() == encrypt.S3 { newSSE = encrypt.NewSSE() } if newSSE != nil { dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObject", Encryption: newSSE, } args["destination"] = dst _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } // Get copied object and check if content is equal reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE, Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } decBytes, err = io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } reader.Close() err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="}) if err != nil { fmt.Printf("srcObject objectInfo: %+v\n", oi) logError(testName, function, args, startTime, "", "Checksum mismatch on srcObject for in-place", err) return } // Test in-place decryption. dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObject", } args["destination"] = dst src = minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", Encryption: newSSE, } args["source"] = src _, err = c.CopyObject(context.Background(), dst, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject Key rotation failed", err) return } } // Get copied decrypted object and check if content is equal reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{Checksum: true}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } defer reader.Close() decBytes, err = io.ReadAll(reader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(decBytes, buf) { logError(testName, function, args, startTime, "", "Downloaded object mismatched for encrypted object", err) return } err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="}) if err != nil { logError(testName, function, args, startTime, "", "Checksum mismatch for decrypted object", err) return } logSuccess(testName, function, args, startTime) } // Test encrypted copy object func testUnencryptedToSSECCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, nil, sseDst) } // Test encrypted copy object func testUnencryptedToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") var sseSrc encrypt.ServerSide sseDst := encrypt.NewSSE() testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testUnencryptedToUnencryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") var sseSrc, sseDst encrypt.ServerSide testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSECToSSECCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSECToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) sseDst := encrypt.NewSSE() testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSECToUnencryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) var sseDst encrypt.ServerSide testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSES3ToSSECCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.NewSSE() sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSES3ToSSES3CopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.NewSSE() sseDst := encrypt.NewSSE() testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedSSES3ToUnencryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.NewSSE() var sseDst encrypt.ServerSide testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } // Test encrypted copy object func testEncryptedCopyObjectV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true, TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") sseSrc := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"srcObject")) sseDst := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+"dstObject")) testEncryptedCopyObjectWrapper(c, bucketName, sseSrc, sseDst) } func testDecryptedCopyObject() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } bucketName, objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-"), "object" if err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}); err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) encryption := encrypt.DefaultPBKDF([]byte("correct horse battery staple"), []byte(bucketName+objectName)) _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(bytes.Repeat([]byte("a"), 1024*1024)), 1024*1024, minio.PutObjectOptions{ ServerSideEncryption: encryption, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } src := minio.CopySrcOptions{ Bucket: bucketName, Object: objectName, Encryption: encrypt.SSECopy(encryption), } args["source"] = src dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "decrypted-" + objectName, } args["destination"] = dst if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } if _, err = c.GetObject(context.Background(), bucketName, "decrypted-"+objectName, minio.GetObjectOptions{}); err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } logSuccess(testName, function, args, startTime) } func testSSECMultipartEncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 6MB of data buf := bytes.Repeat([]byte("abcdef"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) // Upload a 6MB object using multipart mechanism uploadID, err := c.NewMultipartUpload(context.Background(), bucketName, objectName, minio.PutObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } var completeParts []minio.CompletePart part, err := c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 1, bytes.NewReader(buf[:5*1024*1024]), 5*1024*1024, minio.PutObjectPartOptions{SSE: srcencryption}, ) if err != nil { logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) return } completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) part, err = c.PutObjectPart(context.Background(), bucketName, objectName, uploadID, 2, bytes.NewReader(buf[5*1024*1024:]), 1024*1024, minio.PutObjectPartOptions{SSE: srcencryption}, ) if err != nil { logError(testName, function, args, startTime, "", "PutObjectPart call failed", err) return } completeParts = append(completeParts, minio.CompletePart{PartNumber: part.PartNumber, ETag: part.ETag}) // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), bucketName, objectName, uploadID, completeParts, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err = c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = objInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err = c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (6*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 6*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 6*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 6MB", err) return } getOpts.SetRange(6*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 6*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:6*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 6MB", err) return } if getBuf[6*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation func testSSECEncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, ServerSideEncryption: srcencryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for SSEC encrypted to unencrypted copy func testSSECEncryptedToUnencryptedCopyPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcencryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" var dstencryption encrypt.ServerSide uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for SSEC encrypted to SSE-S3 encrypted copy func testSSECEncryptedToSSES3CopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcencryption := encrypt.DefaultPBKDF([]byte(password), []byte(bucketName+objectName)) putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, ServerSideEncryption: srcencryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) encrypt.SSECopy(srcencryption).Marshal(header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to SSEC encryption copy part func testUnencryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{TrailingHeaders: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy func testUnencryptedToUnencryptedCopyPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") putmetadata := map[string]string{ "Content-Type": "binary/octet-stream", } opts := minio.PutObjectOptions{ UserMetadata: putmetadata, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy func testUnencryptedToSSES3CopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for SSE-S3 to SSEC encryption copy part func testSSES3EncryptedToSSECCopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") password := "correct horse battery staple" srcEncryption := encrypt.NewSSE() opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcEncryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.DefaultPBKDF([]byte(password), []byte(destBucketName+destObjectName)) uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{ServerSideEncryption: dstencryption} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to unencrypted copy func testSSES3EncryptedToUnencryptedCopyPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") srcEncryption := encrypt.NewSSE() opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcEncryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } // Test Core CopyObjectPart implementation for unencrypted to SSE-S3 encrypted copy func testSSES3EncryptedToSSES3CopyObjectPart() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObjectPart(destination, source)" args := map[string]interface{}{} client, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Instantiate new core client object. c := minio.Core{Client: client} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, client) // Make a buffer with 5MB of data buf := bytes.Repeat([]byte("abcde"), 1024*1024) // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") srcEncryption := encrypt.NewSSE() opts := minio.PutObjectOptions{ UserMetadata: map[string]string{ "Content-Type": "binary/octet-stream", }, ServerSideEncryption: srcEncryption, } uploadInfo, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), "", "", opts) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{ServerSideEncryption: srcEncryption}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", fmt.Sprintf("Error: number of bytes does not match, want %v, got %v\n", len(buf), st.Size), err) return } destBucketName := bucketName destObjectName := objectName + "-dest" dstencryption := encrypt.NewSSE() uploadID, err := c.NewMultipartUpload(context.Background(), destBucketName, destObjectName, minio.PutObjectOptions{ServerSideEncryption: dstencryption}) if err != nil { logError(testName, function, args, startTime, "", "NewMultipartUpload call failed", err) return } // Content of the destination object will be two copies of // `objectName` concatenated, followed by first byte of // `objectName`. metadata := make(map[string]string) header := make(http.Header) dstencryption.Marshal(header) for k, v := range header { metadata[k] = v[0] } metadata["x-amz-copy-source-if-match"] = uploadInfo.ETag // First of three parts fstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 1, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Second of three parts sndPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 2, 0, -1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Last of three parts lstPart, err := c.CopyObjectPart(context.Background(), bucketName, objectName, destBucketName, destObjectName, uploadID, 3, 0, 1, metadata) if err != nil { logError(testName, function, args, startTime, "", "CopyObjectPart call failed", err) return } // Complete the multipart upload _, err = c.CompleteMultipartUpload(context.Background(), destBucketName, destObjectName, uploadID, []minio.CompletePart{fstPart, sndPart, lstPart}, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "CompleteMultipartUpload call failed", err) return } // Stat the object and check its length matches objInfo, err := c.StatObject(context.Background(), destBucketName, destObjectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject call failed", err) return } if objInfo.Size != (5*1024*1024)*2+1 { logError(testName, function, args, startTime, "", "Destination object has incorrect size!", err) return } // Now we read the data back getOpts := minio.GetObjectOptions{} getOpts.SetRange(0, 5*1024*1024-1) r, _, _, err := c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf := make([]byte, 5*1024*1024) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf, buf) { logError(testName, function, args, startTime, "", "Got unexpected data in first 5MB", err) return } getOpts.SetRange(5*1024*1024, 0) r, _, _, err = c.GetObject(context.Background(), destBucketName, destObjectName, getOpts) if err != nil { logError(testName, function, args, startTime, "", "GetObject call failed", err) return } getBuf = make([]byte, 5*1024*1024+1) _, err = readFull(r, getBuf) if err != nil { logError(testName, function, args, startTime, "", "Read buffer failed", err) return } if !bytes.Equal(getBuf[:5*1024*1024], buf) { logError(testName, function, args, startTime, "", "Got unexpected data in second 5MB", err) return } if getBuf[5*1024*1024] != buf[0] { logError(testName, function, args, startTime, "", "Got unexpected data in last byte of copied object!", err) return } logSuccess(testName, function, args, startTime) // Do not need to remove destBucketName its same as bucketName. } func testUserMetadataCopying() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } testUserMetadataCopyingWrapper(c) } func testUserMetadataCopyingWrapper(c *minio.Client) { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") // Make a new bucket in 'us-east-1' (source bucket). err := c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return h } h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-meta-") { h.Add(k, vs[0]) } } return h } // 1. create a client encrypted object to copy by uploading const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB metadata := make(http.Header) metadata.Set("x-amz-meta-myheader", "myvalue") m := make(map[string]string) m["x-amz-meta-myheader"] = "myvalue" _, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{UserMetadata: m}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithMetadata failed", err) return } if !reflect.DeepEqual(metadata, fetchMeta("srcObject")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 2. create source src := minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObject", } // 2.1 create destination with metadata set dst1 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-1", UserMetadata: map[string]string{"notmyheader": "notmyvalue"}, ReplaceMetadata: true, } // 3. Check that copying to an object with metadata set resets // the headers on the copy. args["source"] = src args["destination"] = dst1 _, err = c.CopyObject(context.Background(), dst1, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } expectedHeaders := make(http.Header) expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-1")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 4. create destination with no metadata set and same source dst2 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-2", } // 5. Check that copying to an object with no metadata set, // copies metadata. args["source"] = src args["destination"] = dst2 _, err = c.CopyObject(context.Background(), dst2, src) if err != nil { logError(testName, function, args, startTime, "", "CopyObject failed", err) return } expectedHeaders = metadata if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-2")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 6. Compose a pair of sources. dst3 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-3", ReplaceMetadata: true, } function = "ComposeObject(destination, sources)" args["source"] = []minio.CopySrcOptions{src, src} args["destination"] = dst3 _, err = c.ComposeObject(context.Background(), dst3, src, src) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } // Check that no headers are copied in this case if !reflect.DeepEqual(make(http.Header), fetchMeta("dstObject-3")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } // 7. Compose a pair of sources with dest user metadata set. dst4 := minio.CopyDestOptions{ Bucket: bucketName, Object: "dstObject-4", UserMetadata: map[string]string{"notmyheader": "notmyvalue"}, ReplaceMetadata: true, } function = "ComposeObject(destination, sources)" args["source"] = []minio.CopySrcOptions{src, src} args["destination"] = dst4 _, err = c.ComposeObject(context.Background(), dst4, src, src) if err != nil { logError(testName, function, args, startTime, "", "ComposeObject failed", err) return } // Check that no headers are copied in this case expectedHeaders = make(http.Header) expectedHeaders.Set("x-amz-meta-notmyheader", "notmyvalue") if !reflect.DeepEqual(expectedHeaders, fetchMeta("dstObject-4")) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } logSuccess(testName, function, args, startTime) } func testUserMetadataCopyingV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "CopyObject(destination, source)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } testUserMetadataCopyingWrapper(c) } func testStorageClassMetadataPutObject() { // initialize logging params startTime := time.Now() function := "testStorageClassMetadataPutObject()" args := map[string]interface{}{} testName := getFuncName() c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return h } h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { for _, v := range vs { h.Add(k, v) } } } return h } metadata := make(http.Header) metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") emptyMetadata := make(http.Header) const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Get the returned metadata returnedMeta := fetchMeta("srcObjectRRSClass") // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } metadata = make(http.Header) metadata.Set("x-amz-storage-class", "STANDARD") _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClass")) { logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) return } logSuccess(testName, function, args, startTime) } func testStorageClassInvalidMetadataPutObject() { // initialize logging params startTime := time.Now() function := "testStorageClassInvalidMetadataPutObject()" args := map[string]interface{}{} testName := getFuncName() c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 1MiB _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "INVALID_STORAGE_CLASS"}) if err == nil { logError(testName, function, args, startTime, "", "PutObject with invalid storage class passed, was expected to fail", err) return } logSuccess(testName, function, args, startTime) } func testStorageClassMetadataCopyObject() { // initialize logging params startTime := time.Now() function := "testStorageClassMetadataCopyObject()" args := map[string]interface{}{} testName := getFuncName() c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") // Make a new bucket in 'us-east-1' (source bucket). err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) fetchMeta := func(object string) (h http.Header) { objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{}) args["bucket"] = bucketName args["object"] = object if err != nil { logError(testName, function, args, startTime, "", "Stat failed", err) return h } h = make(http.Header) for k, vs := range objInfo.Metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-storage-class") { for _, v := range vs { h.Add(k, v) } } } return h } metadata := make(http.Header) metadata.Set("x-amz-storage-class", "REDUCED_REDUNDANCY") emptyMetadata := make(http.Header) const srcSize = 1024 * 1024 buf := bytes.Repeat([]byte("abcde"), srcSize) // Put an object with RRS Storage class _, err = c.PutObject(context.Background(), bucketName, "srcObjectRRSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "REDUCED_REDUNDANCY"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Make server side copy of object uploaded in previous step src := minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObjectRRSClass", } dst := minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObjectRRSClassCopy", } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on RRS", err) return } // Get the returned metadata returnedMeta := fetchMeta("srcObjectRRSClassCopy") // The response metada should either be equal to metadata (with REDUCED_REDUNDANCY) or emptyMetadata (in case of gateways) if !reflect.DeepEqual(metadata, returnedMeta) && !reflect.DeepEqual(emptyMetadata, returnedMeta) { logError(testName, function, args, startTime, "", "Metadata match failed", err) return } metadata = make(http.Header) metadata.Set("x-amz-storage-class", "STANDARD") // Put an object with Standard Storage class _, err = c.PutObject(context.Background(), bucketName, "srcObjectSSClass", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{StorageClass: "STANDARD"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Make server side copy of object uploaded in previous step src = minio.CopySrcOptions{ Bucket: bucketName, Object: "srcObjectSSClass", } dst = minio.CopyDestOptions{ Bucket: bucketName, Object: "srcObjectSSClassCopy", } if _, err = c.CopyObject(context.Background(), dst, src); err != nil { logError(testName, function, args, startTime, "", "CopyObject failed on SS", err) return } // Fetch the meta data of copied object if reflect.DeepEqual(metadata, fetchMeta("srcObjectSSClassCopy")) { logError(testName, function, args, startTime, "", "Metadata verification failed, STANDARD storage class should not be a part of response metadata", err) return } logSuccess(testName, function, args, startTime) } // Test put object with size -1 byte object. func testPutObjectNoLengthV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": -1, "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := bucketName + "unique" args["objectName"] = objectName bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() args["size"] = bufSize // Upload an object. _, err = c.PutObject(context.Background(), bucketName, objectName, reader, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Expected upload object size "+string(bufSize)+" got "+string(st.Size), err) return } logSuccess(testName, function, args, startTime) } // Test put objects of unknown size. func testPutObjectsUnknownV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader,size,opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": "", "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Issues are revealed by trying to upload multiple files of unknown size // sequentially (on 4GB machines) for i := 1; i <= 4; i++ { // Simulate that we could be receiving byte slices of data that we want // to upload as a file rpipe, wpipe := io.Pipe() defer rpipe.Close() go func() { b := []byte("test") wpipe.Write(b) wpipe.Close() }() // Upload the object. objectName := fmt.Sprintf("%sunique%d", bucketName, i) args["objectName"] = objectName ui, err := c.PutObject(context.Background(), bucketName, objectName, rpipe, -1, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectStreaming failed", err) return } if ui.Size != 4 { logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(ui.Size), nil) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObjectStreaming failed", err) return } if st.Size != int64(4) { logError(testName, function, args, startTime, "", "Expected upload object size "+string(4)+" got "+string(st.Size), err) return } } logSuccess(testName, function, args, startTime) } // Test put object with 0 byte object. func testPutObject0ByteV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": 0, "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := bucketName + "unique" args["objectName"] = objectName args["opts"] = minio.PutObjectOptions{} // Upload an object. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err) return } if st.Size != 0 { logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err) return } logSuccess(testName, function, args, startTime) } // Test put object with 0 byte object with non-US-ASCII characters. func testPutObjectMetadataNonUSASCIIV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "bucketName": "", "objectName": "", "size": 0, "opts": "", } metadata := map[string]string{ "test-zh": "你好", "test-ja": "こんにちは", "test-ko": "안녕하세요", "test-ru": "Здравствуй", "test-de": "Hallo", "test-it": "Ciao", "test-pt": "Olá", "test-ar": "مرحبا", "test-hi": "नमस्ते", "test-hu": "Helló", "test-ro": "Bună", "test-be": "Прывiтанне", "test-sl": "Pozdravljen", "test-sr": "Здраво", "test-bg": "Здравейте", "test-uk": "Привіт", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) objectName := bucketName + "unique" args["objectName"] = objectName args["opts"] = minio.PutObjectOptions{} // Upload an object. _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{ UserMetadata: metadata, }) if err != nil { logError(testName, function, args, startTime, "", "PutObjectWithSize failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObjectWithSize failed", err) return } if st.Size != 0 { logError(testName, function, args, startTime, "", "Expected upload object size 0 but got "+string(st.Size), err) return } for k, v := range metadata { if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { continue } if st.Metadata.Get(http.CanonicalHeaderKey("X-Amz-Meta-"+k)) != v { logError(testName, function, args, startTime, "", "Expected upload object metadata "+k+": "+v+" but got "+st.Metadata.Get(http.CanonicalHeaderKey("X-Amz-Meta-"+k)), err) return } } logSuccess(testName, function, args, startTime) } // Test expected error cases func testComposeObjectErrorCases() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } testComposeObjectErrorCasesWrapper(c) } // Test concatenating multiple 10K objects V4 func testCompose10KSources() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ComposeObject(destination, sourceList)" args := map[string]interface{}{} c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } testComposeMultipleSources(c) } // Tests comprehensive list of all methods. func testFunctionalV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "testFunctionalV2()" functionAll := "" args := map[string]interface{}{} c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v2 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") location := "us-east-1" // Make a new bucket. function = "MakeBucket(bucketName, location)" functionAll = "MakeBucket(bucketName, location)" args = map[string]interface{}{ "bucketName": bucketName, "location": location, } err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: location}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Generate a random file name. fileName := randString(60, rand.NewSource(time.Now().UnixNano()), "") file, err := os.Create(fileName) if err != nil { logError(testName, function, args, startTime, "", "file create failed", err) return } for i := 0; i < 3; i++ { buf := make([]byte, rand.Intn(1<<19)) _, err = file.Write(buf) if err != nil { logError(testName, function, args, startTime, "", "file write failed", err) return } } file.Close() // Verify if bucket exits and you have access. var exists bool function = "BucketExists(bucketName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, } exists, err = c.BucketExists(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "BucketExists failed", err) return } if !exists { logError(testName, function, args, startTime, "", "Could not find existing bucket "+bucketName, err) return } // Make the bucket 'public read/write'. function = "SetBucketPolicy(bucketName, bucketPolicy)" functionAll += ", " + function readWritePolicy := `{"Version": "2012-10-17","Statement": [{"Action": ["s3:ListBucketMultipartUploads", "s3:ListBucket"],"Effect": "Allow","Principal": {"AWS": ["*"]},"Resource": ["arn:aws:s3:::` + bucketName + `"],"Sid": ""}]}` args = map[string]interface{}{ "bucketName": bucketName, "bucketPolicy": readWritePolicy, } err = c.SetBucketPolicy(context.Background(), bucketName, readWritePolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // List all buckets. function = "ListBuckets()" functionAll += ", " + function args = nil buckets, err := c.ListBuckets(context.Background()) if len(buckets) == 0 { logError(testName, function, args, startTime, "", "List buckets cannot be empty", err) return } if err != nil { logError(testName, function, args, startTime, "", "ListBuckets failed", err) return } // Verify if previously created bucket is listed in list buckets. bucketFound := false for _, bucket := range buckets { if bucket.Name == bucketName { bucketFound = true } } // If bucket not found error out. if !bucketFound { logError(testName, function, args, startTime, "", "Bucket "+bucketName+"not found", err) return } objectName := bucketName + "unique" // Generate data buf := bytes.Repeat([]byte("n"), rand.Intn(1<<19)) args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "contentType": "", } _, err = c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err) return } objectNameNoLength := objectName + "-nolength" args["objectName"] = objectNameNoLength _, err = c.PutObject(context.Background(), bucketName, objectNameNoLength, bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } st, err = c.StatObject(context.Background(), bucketName, objectNameNoLength, minio.StatObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "StatObject failed", err) return } if st.Size != int64(len(buf)) { logError(testName, function, args, startTime, "", "Expected uploaded object length "+string(len(buf))+" got "+string(st.Size), err) return } // Instantiate a done channel to close all listing. doneCh := make(chan struct{}) defer close(doneCh) objFound := false isRecursive := true // Recursive is true. function = "ListObjects(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for obj := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Prefix: objectName, Recursive: isRecursive}) { if obj.Key == objectName { objFound = true break } } if !objFound { logError(testName, function, args, startTime, "", "Could not find existing object "+objectName, err) return } incompObjNotFound := true function = "ListIncompleteUploads(bucketName, objectName, isRecursive, doneCh)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "isRecursive": isRecursive, } for objIncompl := range c.ListIncompleteUploads(context.Background(), bucketName, objectName, isRecursive) { if objIncompl.Key != "" { incompObjNotFound = false break } } if !incompObjNotFound { logError(testName, function, args, startTime, "", "Unexpected dangling incomplete upload found", err) return } function = "GetObject(bucketName, objectName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, } newReader, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } newReadBytes, err := io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } function = "FGetObject(bucketName, objectName, fileName)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "fileName": fileName + "-f", } err = c.FGetObject(context.Background(), bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FgetObject failed", err) return } // Generate presigned HEAD object url. function = "PresignedHeadObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedHeadURL, err := c.PresignedHeadObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject failed", err) return } httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, Transport: createHTTPTransport(), } req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) return } // Verify if presigned url works. resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedHeadObject URL head request failed", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedHeadObject URL returns status "+string(resp.StatusCode), err) return } if resp.Header.Get("ETag") == "" { logError(testName, function, args, startTime, "", "Got empty ETag", err) return } resp.Body.Close() // Generate presigned GET object url. function = "PresignedGetObject(bucketName, objectName, expires, reqParams)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName, "expires": 3600 * time.Second, } presignedGetURL, err := c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return } newPresignedBytes, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } resp.Body.Close() if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } // Set request parameters. reqParams := make(url.Values) reqParams.Set("response-content-disposition", "attachment; filename=\"test.txt\"") // Generate presigned GET object url. args["reqParams"] = reqParams presignedGetURL, err = c.PresignedGetObject(context.Background(), bucketName, objectName, 3600*time.Second, reqParams) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject failed", err) return } // Verify if presigned url works. req, err = http.NewRequest(http.MethodGet, presignedGetURL.String(), nil) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject request incorrect", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "PresignedGetObject response incorrect", err) return } if resp.StatusCode != http.StatusOK { logError(testName, function, args, startTime, "", "PresignedGetObject URL returns status "+string(resp.StatusCode), err) return } newPresignedBytes, err = io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed", err) return } if !bytes.Equal(newPresignedBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch", err) return } // Verify content disposition. if resp.Header.Get("Content-Disposition") != "attachment; filename=\"test.txt\"" { logError(testName, function, args, startTime, "", "wrong Content-Disposition received ", err) return } function = "PresignedPutObject(bucketName, objectName, expires)" functionAll += ", " + function args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", "expires": 3600 * time.Second, } presignedPutURL, err := c.PresignedPutObject(context.Background(), bucketName, objectName+"-presigned", 3600*time.Second) if err != nil { logError(testName, function, args, startTime, "", "PresignedPutObject failed", err) return } // Generate data more than 32K buf = bytes.Repeat([]byte("1"), rand.Intn(1<<10)+32*1024) req, err = http.NewRequest(http.MethodPut, presignedPutURL.String(), bytes.NewReader(buf)) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) return } resp, err = httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request to PresignedPutObject URL failed", err) return } // Download the uploaded object to verify args = map[string]interface{}{ "bucketName": bucketName, "objectName": objectName + "-presigned", } newReader, err = c.GetObject(context.Background(), bucketName, objectName+"-presigned", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject of uploaded presigned object failed", err) return } newReadBytes, err = io.ReadAll(newReader) if err != nil { logError(testName, function, args, startTime, "", "ReadAll failed during get on presigned put object", err) return } newReader.Close() if !bytes.Equal(newReadBytes, buf) { logError(testName, function, args, startTime, "", "Bytes mismatch on presigned object upload verification", err) return } function = "PresignHeader(method, bucketName, objectName, expires, reqParams, extraHeaders)" functionAll += ", " + function presignExtraHeaders := map[string][]string{ "mysecret": {"abcxxx"}, } args = map[string]interface{}{ "method": "PUT", "bucketName": bucketName, "objectName": objectName + "-presign-custom", "expires": 3600 * time.Second, "extraHeaders": presignExtraHeaders, } _, err = c.PresignHeader(context.Background(), "PUT", bucketName, objectName+"-presign-custom", 3600*time.Second, nil, presignExtraHeaders) if err == nil { logError(testName, function, args, startTime, "", "Presigned with extra headers succeeded", err) return } os.Remove(fileName) os.Remove(fileName + "-f") logSuccess(testName, functionAll, args, startTime) } // Test get object with GetObject with context func testGetObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx cancel() r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err) return } if _, err = r.Stat(); err == nil { logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err) return } r.Close() ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() // Read the data back r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "object Stat call failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match: want "+string(bufSize)+", got"+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", "object Close() call failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with FGetObject with a user provided context func testFGetObjectContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FGetObject(ctx, bucketName, objectName, fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "fileName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-1-MB"] reader := getDataReader("datafile-1-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() fileName := "tempfile-context" args["fileName"] = fileName // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err == nil { logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err) return } if err = os.Remove(fileName + "-fcontext"); err != nil { logError(testName, function, args, startTime, "", "Remove file failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with GetObject with a user provided context func testGetObjectRanges() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(ctx, bucketName, objectName, fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "fileName": "", } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) defer cancel() rng := rand.NewSource(time.Now().UnixNano()) c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rng, "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() // Save the data objectName := randString(60, rng, "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } // Read the data back tests := []struct { start int64 end int64 }{ { start: 1024, end: 1024 + 1<<20, }, { start: 20e6, end: 20e6 + 10000, }, { start: 40e6, end: 40e6 + 10000, }, { start: 60e6, end: 60e6 + 10000, }, { start: 80e6, end: 80e6 + 10000, }, { start: 120e6, end: int64(bufSize), }, } for _, test := range tests { wantRC := getDataReader("datafile-129-MB") io.CopyN(io.Discard, wantRC, test.start) want := mustCrcReader(io.LimitReader(wantRC, test.end-test.start+1)) opts := minio.GetObjectOptions{} opts.SetRange(test.start, test.end) args["opts"] = fmt.Sprintf("%+v", test) obj, err := c.GetObject(ctx, bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "FGetObject with long timeout failed", err) return } err = crcMatches(obj, want) if err != nil { logError(testName, function, args, startTime, "", fmt.Sprintf("GetObject offset %d -> %d", test.start, test.end), err) return } } logSuccess(testName, function, args, startTime) } // Test get object ACLs with GetObjectACL with custom provided context func testGetObjectACLContext() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObjectACL(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-1-MB"] reader := getDataReader("datafile-1-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Add meta data to add a canned acl metaData := map[string]string{ "X-Amz-Acl": "public-read-write", } _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: "binary/octet-stream", UserMetadata: metaData, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) args["ctx"] = ctx defer cancel() // Read the data back objectInfo, getObjectACLErr := c.GetObjectACL(ctx, bucketName, objectName) if getObjectACLErr != nil { logError(testName, function, args, startTime, "", "GetObjectACL failed. ", getObjectACLErr) return } s, ok := objectInfo.Metadata["X-Amz-Acl"] if !ok { logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Acl\"", nil) return } if len(s) != 1 { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" canned acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } // Do a very limited testing if this is not AWS S3 if os.Getenv(serverEndpoint) != "s3.amazonaws.com" { if s[0] != "private" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"private\" but got"+fmt.Sprintf("%q", s[0]), nil) return } logSuccess(testName, function, args, startTime) return } if s[0] != "public-read-write" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Acl\" expected \"public-read-write\" but got"+fmt.Sprintf("%q", s[0]), nil) return } bufSize = dataFileMap["datafile-1-MB"] reader2 := getDataReader("datafile-1-MB") defer reader2.Close() // Save the data objectName = randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Add meta data to add a canned acl metaData = map[string]string{ "X-Amz-Grant-Read": "id=fooread@minio.go", "X-Amz-Grant-Write": "id=foowrite@minio.go", } _, err = c.PutObject(context.Background(), bucketName, objectName, reader2, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", UserMetadata: metaData}) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) return } ctx, cancel = context.WithTimeout(context.Background(), 10*time.Second) args["ctx"] = ctx defer cancel() // Read the data back objectInfo, getObjectACLErr = c.GetObjectACL(ctx, bucketName, objectName) if getObjectACLErr == nil { logError(testName, function, args, startTime, "", "GetObjectACL fail", getObjectACLErr) return } if len(objectInfo.Metadata) != 3 { logError(testName, function, args, startTime, "", "GetObjectACL fail expected \"3\" ACLs but got "+fmt.Sprintf(`"%d"`, len(objectInfo.Metadata)), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Read"] if !ok { logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Read\"", nil) return } if len(s) != 1 { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "fooread@minio.go" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Read\" acl expected \"fooread@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } s, ok = objectInfo.Metadata["X-Amz-Grant-Write"] if !ok { logError(testName, function, args, startTime, "", "GetObjectACL fail unable to find \"X-Amz-Grant-Write\"", nil) return } if len(s) != 1 { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"1\" got "+fmt.Sprintf(`"%d"`, len(s)), nil) return } if s[0] != "foowrite@minio.go" { logError(testName, function, args, startTime, "", "GetObjectACL fail \"X-Amz-Grant-Write\" acl expected \"foowrite@minio.go\" got "+fmt.Sprintf("%q", s), nil) return } logSuccess(testName, function, args, startTime) } // Test validates putObject with context to see if request cancellation is honored for V2. func testPutObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "PutObject(ctx, bucketName, objectName, reader, size, opts)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "size": "", "opts": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Make a new bucket. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datatfile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() objectName := fmt.Sprintf("test-file-%v", rand.Uint32()) args["objectName"] = objectName ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) args["ctx"] = ctx args["size"] = bufSize defer cancel() _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject with short timeout failed", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) args["ctx"] = ctx defer cancel() reader = getDataReader("datafile-33-kB") defer reader.Close() _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject with long timeout failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with GetObject with custom context func testGetObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetObject(ctx, bucketName, objectName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx cancel() r, err := c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject failed unexpectedly", err) return } if _, err = r.Stat(); err == nil { logError(testName, function, args, startTime, "", "GetObject should fail on short timeout", err) return } r.Close() ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back r, err = c.GetObject(ctx, bucketName, objectName, minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "GetObject shouldn't fail on longer timeout", err) return } st, err := r.Stat() if err != nil { logError(testName, function, args, startTime, "", "object Stat call failed", err) return } if st.Size != int64(bufSize) { logError(testName, function, args, startTime, "", "Number of bytes in stat does not match, expected "+string(bufSize)+" got "+string(st.Size), err) return } if err := r.Close(); err != nil { logError(testName, function, args, startTime, "", " object Close() call failed", err) return } logSuccess(testName, function, args, startTime) } // Test get object with FGetObject with custom context func testFGetObjectContextV2() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "FGetObject(ctx, bucketName, objectName,fileName)" args := map[string]interface{}{ "ctx": "", "bucketName": "", "objectName": "", "fileName": "", } c, err := NewClient(ClientConfig{CredsV2: true}) if err != nil { logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket call failed", err) return } defer cleanupBucket(bucketName, c) bufSize := dataFileMap["datatfile-1-MB"] reader := getDataReader("datafile-1-MB") defer reader.Close() // Save the data objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } ctx, cancel := context.WithTimeout(context.Background(), 1*time.Nanosecond) args["ctx"] = ctx defer cancel() fileName := "tempfile-context" args["fileName"] = fileName // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-f", minio.GetObjectOptions{}) if err == nil { logError(testName, function, args, startTime, "", "FGetObject should fail on short timeout", err) return } ctx, cancel = context.WithTimeout(context.Background(), 1*time.Hour) defer cancel() // Read the data back err = c.FGetObject(ctx, bucketName, objectName, fileName+"-fcontext", minio.GetObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "FGetObject call shouldn't fail on long timeout", err) return } if err = os.Remove(fileName + "-fcontext"); err != nil { logError(testName, function, args, startTime, "", "Remove file failed", err) return } logSuccess(testName, function, args, startTime) } // Test list object v1 and V2 func testListObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "ListObjects(bucketName, objectPrefix, recursive, doneCh)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", "recursive": "true", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) testObjects := []struct { name string storageClass string }{ // Special characters {"foo bar", "STANDARD"}, {"foo-%", "STANDARD"}, {"random-object-1", "STANDARD"}, {"random-object-2", "REDUCED_REDUNDANCY"}, } for i, object := range testObjects { bufSize := dataFileMap["datafile-33-kB"] reader := getDataReader("datafile-33-kB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, object.name, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream", StorageClass: object.storageClass}) if err != nil { logError(testName, function, args, startTime, "", fmt.Sprintf("PutObject %d call failed", i+1), err) return } } testList := func(listFn func(context.Context, string, minio.ListObjectsOptions) <-chan minio.ObjectInfo, bucket string, opts minio.ListObjectsOptions) { var objCursor int // check for object name and storage-class from listing object result for objInfo := range listFn(context.Background(), bucket, opts) { if objInfo.Err != nil { logError(testName, function, args, startTime, "", "ListObjects failed unexpectedly", err) return } if objInfo.Key != testObjects[objCursor].name { logError(testName, function, args, startTime, "", "ListObjects does not return expected object name", err) return } if objInfo.StorageClass != testObjects[objCursor].storageClass { // Ignored as Gateways (Azure/GCS etc) wont return storage class logIgnored(testName, function, args, startTime, "ListObjects doesn't return expected storage class") } objCursor++ } if objCursor != len(testObjects) { logError(testName, function, args, startTime, "", "ListObjects returned unexpected number of items", errors.New("")) return } } testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, UseV1: true}) testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true}) testList(c.ListObjects, bucketName, minio.ListObjectsOptions{Recursive: true, WithMetadata: true}) logSuccess(testName, function, args, startTime) } // testCors is runnable against S3 itself. // Just provide the env var MINIO_GO_TEST_BUCKET_CORS with bucket that is public and WILL BE DELETED. // Recreate this manually each time. Minio-go SDK does not support calling // SetPublicBucket (put-public-access-block) on S3, otherwise we could script the whole thing. func testCors() { ctx := context.Background() startTime := time.Now() testName := getFuncName() function := "SetBucketCors(bucketName, cors)" args := map[string]interface{}{ "bucketName": "", "cors": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Create or reuse a bucket that will get cors settings applied to it and deleted when done bucketName := os.Getenv("MINIO_GO_TEST_BUCKET_CORS") if bucketName == "" { bucketName = randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } } args["bucketName"] = bucketName defer cleanupBucket(bucketName, c) publicPolicy := `{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:*"],"Resource":["arn:aws:s3:::` + bucketName + `", "arn:aws:s3:::` + bucketName + `/*"]}]}` err = c.SetBucketPolicy(ctx, bucketName, publicPolicy) if err != nil { logError(testName, function, args, startTime, "", "SetBucketPolicy failed", err) return } // Upload an object for testing. objectContents := `some-text-file-contents` reader := strings.NewReader(objectContents) bufSize := int64(len(objectContents)) objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName _, err = c.PutObject(ctx, bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"}) if err != nil { logError(testName, function, args, startTime, "", "PutObject call failed", err) return } bucketURL := c.EndpointURL().String() + "/" + bucketName + "/" objectURL := bucketURL + objectName httpClient := &http.Client{ Timeout: 30 * time.Second, Transport: createHTTPTransport(), } errStrAccessForbidden := `AccessForbiddenCORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted` testCases := []struct { name string // Cors rules to apply applyCorsRules []cors.Rule // Outbound request info method string url string headers map[string]string // Wanted response wantStatus int wantHeaders map[string]string wantBodyContains string }{ { name: "apply bucket rules", applyCorsRules: []cors.Rule{ { AllowedOrigin: []string{"https"}, // S3 documents 'https' origin, but it does not actually work, see test below. AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"*"}, }, { AllowedOrigin: []string{"http://www.example1.com"}, AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"*"}, ExposeHeader: []string{"x-amz-server-side-encryption", "x-amz-request-id"}, MaxAgeSeconds: 3600, }, { AllowedOrigin: []string{"http://www.example2.com"}, AllowedMethod: []string{"POST"}, AllowedHeader: []string{"X-My-Special-Header"}, ExposeHeader: []string{"X-AMZ-Request-ID"}, }, { AllowedOrigin: []string{"http://www.example3.com"}, AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"X-Example-3-Special-Header"}, MaxAgeSeconds: 10, }, { AllowedOrigin: []string{"*"}, AllowedMethod: []string{"GET"}, AllowedHeader: []string{"*"}, ExposeHeader: []string{"x-amz-request-id", "X-AMZ-server-side-encryption"}, MaxAgeSeconds: 3600, }, { AllowedOrigin: []string{"http://multiplemethodstest.com"}, AllowedMethod: []string{"POST", "PUT", "DELETE"}, AllowedHeader: []string{"x-abc-*", "x-def-*"}, }, { AllowedOrigin: []string{"http://UPPERCASEEXAMPLE.com"}, AllowedMethod: []string{"DELETE"}, }, { AllowedOrigin: []string{"https://*"}, AllowedMethod: []string{"DELETE"}, AllowedHeader: []string{"x-abc-*", "x-def-*"}, }, }, }, { name: "preflight to object url matches example1 rule", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", // S3 additionally sets the following headers here, MinIO follows fetch spec and does not: // "Access-Control-Expose-Headers": "", }, }, { name: "preflight to bucket url matches example1 rule", method: http.MethodOptions, url: bucketURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", }, }, { name: "preflight matches example2 rule with header given", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example2.com", "Access-Control-Request-Method": "POST", "Access-Control-Request-Headers": "X-My-Special-Header", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example2.com", "Access-Control-Allow-Methods": "POST", "Access-Control-Allow-Headers": "x-my-special-header", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "", "Content-Length": "0", }, }, { name: "preflight matches example2 rule with no header given", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example2.com", "Access-Control-Request-Method": "POST", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example2.com", "Access-Control-Allow-Methods": "POST", "Access-Control-Allow-Headers": "", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "", "Content-Length": "0", }, }, { name: "preflight matches wildcard origin rule", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.couldbeanything.com", "Access-Control-Request-Method": "GET", "Access-Control-Request-Headers": "x-custom-header,x-other-custom-header", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Methods": "GET", "Access-Control-Allow-Headers": "x-custom-header,x-other-custom-header", "Access-Control-Allow-Credentials": "", "Access-Control-Max-Age": "3600", "Content-Length": "0", }, }, { name: "preflight does not match any rule", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.couldbeanything.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight does not match example1 rule because of method", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "POST", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "s3 processes cors rules even when request is not preflight if cors headers present test get", method: http.MethodGet, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", "Access-Control-Request-Method": "PUT", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", // S3 additionally sets the following headers here, MinIO follows fetch spec and does not: // "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", // "Access-Control-Allow-Methods": "PUT", // "Access-Control-Max-Age": "3600", }, }, { name: "s3 processes cors rules even when request is not preflight if cors headers present test put", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "GET", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Origin": "*", "Access-Control-Expose-Headers": "x-amz-request-id,x-amz-server-side-encryption", // S3 additionally sets the following headers here, MinIO follows fetch spec and does not: // "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", // "Access-Control-Allow-Methods": "PUT", // "Access-Control-Max-Age": "3600", }, }, { name: "s3 processes cors rules even when request is not preflight but there is no rule match", method: http.MethodGet, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Headers": "x-another-header,x-could-be-anything", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Allow-Credentials": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "get request matches wildcard origin rule and returns cors headers", method: http.MethodGet, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "x-amz-request-id,X-AMZ-server-side-encryption", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "3600", // "Access-Control-Allow-Methods": "GET", }, }, { name: "head request does not match rule and returns no cors headers", method: http.MethodHead, url: objectURL, headers: map[string]string{ "Origin": "http://www.nomatchingdomainfound.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put request with origin does not match rule and returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.nomatchingdomainfound.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put request with no origin does not match rule and returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{}, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight for delete request with wildcard origin does not match", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.notsecureexample.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight for delete request with wildcard https origin matches secureexample", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.secureexample.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "DELETE", "Access-Control-Allow-Origin": "https://www.secureexample.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight for delete request matches secureexample with wildcard https origin and request headers", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.secureexample.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-abc-1,x-abc-second,x-def-1", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "DELETE", "Access-Control-Allow-Origin": "https://www.secureexample.com", "Access-Control-Allow-Headers": "x-abc-1,x-abc-second,x-def-1", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight for delete request matches secureexample rejected because request header does not match", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.secureexample.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-abc-1,x-abc-second,x-def-1,x-does-not-match", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight with https origin is documented by s3 as matching but it does not match", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "https://www.securebutdoesnotmatch.com", "Access-Control-Request-Method": "PUT", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "put no origin no match returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{}, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put with origin match example1 returns cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "3600", // "Access-Control-Allow-Methods": "PUT", }, }, { name: "put with origin and header match example1 returns cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "x-could-be-anything": "myvalue", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "3600", // "Access-Control-Allow-Methods": "PUT", }, }, { name: "put no match found returns no cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.unmatchingdomain.com", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "", "Access-Control-Allow-Methods": "", "Access-Control-Allow-Origin": "", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "put with origin match example3 returns cors headers", method: http.MethodPut, url: objectURL, headers: map[string]string{ "Origin": "http://www.example3.com", "X-My-Special-Header": "myvalue", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://www.example3.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Max-Age": "10", // "Access-Control-Allow-Methods": "PUT", }, }, { name: "preflight matches example1 rule headers case is incorrect", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", // Fetch standard guarantees that these are sent lowercase, here we test what happens when they are not. "Access-Control-Request-Headers": "X-Another-Header,X-Could-Be-Anything", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "x-another-header,x-could-be-anything", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", }, }, { name: "preflight matches example1 rule headers are not sorted", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://www.example1.com", "Access-Control-Request-Method": "PUT", // Fetch standard guarantees that these are sorted, test what happens when they are not. "Access-Control-Request-Headers": "a-customer-header,b-should-be-last", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Origin": "http://www.example1.com", "Access-Control-Allow-Methods": "PUT", "Access-Control-Allow-Headers": "a-customer-header,b-should-be-last", "Access-Control-Allow-Credentials": "true", "Access-Control-Max-Age": "3600", "Content-Length": "0", // S3 returns the following headers, MinIO follows fetch spec and does not: // "Access-Control-Expose-Headers": "x-amz-server-side-encryption,x-amz-request-id", }, }, { name: "preflight with case sensitivity in origin matches uppercase", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://UPPERCASEEXAMPLE.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Methods": "DELETE", "Access-Control-Allow-Origin": "http://UPPERCASEEXAMPLE.com", "Access-Control-Allow-Headers": "", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", }, }, { name: "preflight with case sensitivity in origin does not match when lowercase", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://uppercaseexample.com", "Access-Control-Request-Method": "DELETE", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight match upper case with unknown header but no header restrictions", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://UPPERCASEEXAMPLE.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-unknown-1", }, wantStatus: http.StatusForbidden, wantBodyContains: errStrAccessForbidden, }, { name: "preflight for delete request matches multiplemethodstest.com origin and request headers", method: http.MethodOptions, url: objectURL, headers: map[string]string{ "Origin": "http://multiplemethodstest.com", "Access-Control-Request-Method": "DELETE", "Access-Control-Request-Headers": "x-abc-1", }, wantStatus: http.StatusOK, wantHeaders: map[string]string{ "Access-Control-Allow-Credentials": "true", "Access-Control-Allow-Origin": "http://multiplemethodstest.com", "Access-Control-Allow-Headers": "x-abc-1", "Access-Control-Expose-Headers": "", "Access-Control-Max-Age": "", // S3 returns POST, PUT, DELETE here, MinIO does not as spec does not require it. // "Access-Control-Allow-Methods": "DELETE", }, }, { name: "delete request goes ahead because cors is only for browsers and does not block on the server side", method: http.MethodDelete, url: objectURL, headers: map[string]string{ "Origin": "http://www.justrandom.com", }, wantStatus: http.StatusNoContent, }, } for i, test := range testCases { testName := fmt.Sprintf("%s_%d_%s", testName, i+1, strings.ReplaceAll(test.name, " ", "_")) // Apply the CORS rules if test.applyCorsRules != nil { corsConfig := &cors.Config{ CORSRules: test.applyCorsRules, } err = c.SetBucketCors(ctx, bucketName, corsConfig) if err != nil { logError(testName, function, args, startTime, "", "SetBucketCors failed to apply", err) return } } // Make request if test.method != "" && test.url != "" { req, err := http.NewRequestWithContext(ctx, test.method, test.url, nil) if err != nil { logError(testName, function, args, startTime, "", "HTTP request creation failed", err) return } req.Header.Set("User-Agent", "MinIO-go-FunctionalTest/"+appVersion) for k, v := range test.headers { req.Header.Set(k, v) } resp, err := httpClient.Do(req) if err != nil { logError(testName, function, args, startTime, "", "HTTP request failed", err) return } defer resp.Body.Close() // Check returned status code if resp.StatusCode != test.wantStatus { errStr := fmt.Sprintf(" incorrect status code in response, want: %d, got: %d", test.wantStatus, resp.StatusCode) logError(testName, function, args, startTime, "", errStr, nil) return } // Check returned body if test.wantBodyContains != "" { body, err := io.ReadAll(resp.Body) if err != nil { logError(testName, function, args, startTime, "", "Failed to read response body", err) return } if !strings.Contains(string(body), test.wantBodyContains) { errStr := fmt.Sprintf(" incorrect body in response, want: %s, in got: %s", test.wantBodyContains, string(body)) logError(testName, function, args, startTime, "", errStr, nil) return } } // Check returned response headers for k, v := range test.wantHeaders { gotVal := resp.Header.Get(k) if k == "Access-Control-Expose-Headers" { // MinIO returns this in canonical form, S3 does not. gotVal = strings.ToLower(gotVal) v = strings.ToLower(v) } // Remove all spaces, S3 adds spaces after CSV values in headers, MinIO does not. gotVal = strings.ReplaceAll(gotVal, " ", "") if gotVal != v { errStr := fmt.Sprintf(" incorrect header in response, want: %s: '%s', got: '%s'", k, v, gotVal) logError(testName, function, args, startTime, "", errStr, nil) return } } } logSuccess(testName, function, args, startTime) } logSuccess(testName, function, args, startTime) } func testCorsSetGetDelete() { ctx := context.Background() startTime := time.Now() testName := getFuncName() function := "SetBucketCors(bucketName, cors)" args := map[string]interface{}{ "bucketName": "", "cors": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } defer cleanupBucket(bucketName, c) // Set the CORS rules on the new bucket corsRules := []cors.Rule{ { AllowedOrigin: []string{"http://www.example1.com"}, AllowedMethod: []string{"PUT"}, AllowedHeader: []string{"*"}, }, { AllowedOrigin: []string{"http://www.example2.com"}, AllowedMethod: []string{"POST"}, AllowedHeader: []string{"X-My-Special-Header"}, }, { AllowedOrigin: []string{"*"}, AllowedMethod: []string{"GET"}, AllowedHeader: []string{"*"}, }, } corsConfig := cors.NewConfig(corsRules) err = c.SetBucketCors(ctx, bucketName, corsConfig) if err != nil { logError(testName, function, args, startTime, "", "SetBucketCors failed to apply", err) return } // Get the rules and check they match what we set gotCorsConfig, err := c.GetBucketCors(ctx, bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketCors failed", err) return } if !reflect.DeepEqual(corsConfig, gotCorsConfig) { msg := fmt.Sprintf("GetBucketCors returned unexpected rules, expected: %+v, got: %+v", corsConfig, gotCorsConfig) logError(testName, function, args, startTime, "", msg, nil) return } // Delete the rules err = c.SetBucketCors(ctx, bucketName, nil) if err != nil { logError(testName, function, args, startTime, "", "SetBucketCors failed to delete", err) return } // Get the rules and check they are now empty gotCorsConfig, err = c.GetBucketCors(ctx, bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketCors failed", err) return } if gotCorsConfig != nil { logError(testName, function, args, startTime, "", "GetBucketCors returned unexpected rules", nil) return } logSuccess(testName, function, args, startTime) } // Test deleting multiple objects with object retention set in Governance mode func testRemoveObjects() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh, opts)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", "recursive": "true", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) return } // Replace with smaller... bufSize = dataFileMap["datafile-10-kB"] reader = getDataReader("datafile-10-kB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) } t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: false, RetainUntilDate: &t, Mode: &m, } err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "Error setting retention", err) return } objectsCh := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh) // List all objects from a bucket-name with a matching prefix. for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) { if object.Err != nil { logError(testName, function, args, startTime, "", "Error listing objects", object.Err) return } objectsCh <- object } }() for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh, minio.RemoveObjectsOptions{}) { // Error is expected here because Retention is set on the object // and RemoveObjects is called without Bypass Governance if rErr.Err == nil { logError(testName, function, args, startTime, "", "Expected error during deletion", nil) return } } objectsCh1 := make(chan minio.ObjectInfo) // Send object names that are needed to be removed to objectsCh go func() { defer close(objectsCh1) // List all objects from a bucket-name with a matching prefix. for object := range c.ListObjects(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) { if object.Err != nil { logError(testName, function, args, startTime, "", "Error listing objects", object.Err) return } objectsCh1 <- object } }() opts1 := minio.RemoveObjectsOptions{ GovernanceBypass: true, } for rErr := range c.RemoveObjects(context.Background(), bucketName, objectsCh1, opts1) { // Error is not expected here because Retention is set on the object // and RemoveObjects is called with Bypass Governance logError(testName, function, args, startTime, "", "Error detected during deletion", rErr.Err) return } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test deleting multiple objects with object retention set in Governance mode, via iterators func testRemoveObjectsIter() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveObjects(bucketName, objectsCh, opts)" args := map[string]interface{}{ "bucketName": "", "objectPrefix": "", "recursive": "true", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") args["objectName"] = objectName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } bufSize := dataFileMap["datafile-129-MB"] reader := getDataReader("datafile-129-MB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) return } // Replace with smaller... bufSize = dataFileMap["datafile-10-kB"] reader = getDataReader("datafile-10-kB") defer reader.Close() _, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error uploading object", err) } t := time.Date(2030, time.April, 25, 14, 0, 0, 0, time.UTC) m := minio.RetentionMode(minio.Governance) opts := minio.PutObjectRetentionOptions{ GovernanceBypass: false, RetainUntilDate: &t, Mode: &m, } err = c.PutObjectRetention(context.Background(), bucketName, objectName, opts) if err != nil { logError(testName, function, args, startTime, "", "Error setting retention", err) return } objectsIter := c.ListObjectsIter(context.Background(), bucketName, minio.ListObjectsOptions{ WithVersions: true, Recursive: true, }) results, err := c.RemoveObjectsWithIter(context.Background(), bucketName, objectsIter, minio.RemoveObjectsOptions{}) if err != nil { logError(testName, function, args, startTime, "", "Error sending delete request", err) return } for result := range results { if result.Err != nil { // Error is expected here because Retention is set on the object // and RemoveObjects is called without Bypass Governance break } logError(testName, function, args, startTime, "", "Expected error during deletion", nil) return } objectsIter = c.ListObjectsIter(context.Background(), bucketName, minio.ListObjectsOptions{UseV1: true, Recursive: true}) results, err = c.RemoveObjectsWithIter(context.Background(), bucketName, objectsIter, minio.RemoveObjectsOptions{ GovernanceBypass: true, }) if err != nil { logError(testName, function, args, startTime, "", "Error sending delete request", err) return } for result := range results { if result.Err != nil { // Error is not expected here because Retention is set on the object // and RemoveObjects is called with Bypass Governance logError(testName, function, args, startTime, "", "Error detected during deletion", result.Err) return } } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test get bucket tags func testGetBucketTagging() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "GetBucketTagging(bucketName)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server failed", err) return } if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test setting tags for bucket func testSetBucketTagging() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "SetBucketTagging(bucketName, tags)" args := map[string]interface{}{ "bucketName": "", "tags": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server", err) return } tag := randString(60, rand.NewSource(time.Now().UnixNano()), "") expectedValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") t, err := tags.MapToBucketTags(map[string]string{ tag: expectedValue, }) args["tags"] = t.String() if err != nil { logError(testName, function, args, startTime, "", "tags.MapToBucketTags failed", err) return } err = c.SetBucketTagging(context.Background(), bucketName, t) if err != nil { logError(testName, function, args, startTime, "", "SetBucketTagging failed", err) return } tagging, err := c.GetBucketTagging(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketTagging failed", err) return } if tagging.ToMap()[tag] != expectedValue { msg := fmt.Sprintf("Tag %s; got value %s; wanted %s", tag, tagging.ToMap()[tag], expectedValue) logError(testName, function, args, startTime, "", msg, err) return } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Test removing bucket tags func testRemoveBucketTagging() { // initialize logging params startTime := time.Now() testName := getFuncName() function := "RemoveBucketTagging(bucketName)" args := map[string]interface{}{ "bucketName": "", } c, err := NewClient(ClientConfig{}) if err != nil { logError(testName, function, args, startTime, "", "MinIO client v4 object creation failed", err) return } // Generate a new random bucket name. bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") args["bucketName"] = bucketName // Make a new bucket. err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1", ObjectLocking: true}) if err != nil { logError(testName, function, args, startTime, "", "MakeBucket failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server", err) return } tag := randString(60, rand.NewSource(time.Now().UnixNano()), "") expectedValue := randString(60, rand.NewSource(time.Now().UnixNano()), "") t, err := tags.MapToBucketTags(map[string]string{ tag: expectedValue, }) if err != nil { logError(testName, function, args, startTime, "", "tags.MapToBucketTags failed", err) return } err = c.SetBucketTagging(context.Background(), bucketName, t) if err != nil { logError(testName, function, args, startTime, "", "SetBucketTagging failed", err) return } tagging, err := c.GetBucketTagging(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "GetBucketTagging failed", err) return } if tagging.ToMap()[tag] != expectedValue { msg := fmt.Sprintf("Tag %s; got value %s; wanted %s", tag, tagging.ToMap()[tag], expectedValue) logError(testName, function, args, startTime, "", msg, err) return } err = c.RemoveBucketTagging(context.Background(), bucketName) if err != nil { logError(testName, function, args, startTime, "", "RemoveBucketTagging failed", err) return } _, err = c.GetBucketTagging(context.Background(), bucketName) if minio.ToErrorResponse(err).Code != minio.NoSuchTagSet { logError(testName, function, args, startTime, "", "Invalid error from server", err) return } // Delete all objects and buckets if err = cleanupVersionedBucket(bucketName, c); err != nil { logError(testName, function, args, startTime, "", "CleanupBucket failed", err) return } logSuccess(testName, function, args, startTime) } // Convert string to bool and always return false if any error func mustParseBool(str string) bool { b, err := strconv.ParseBool(str) if err != nil { return false } return b } // wantChecksums is a map of expected checksums for an object. type wantChecksums map[minio.ChecksumType]string // cmpChecksum compares the checksums of an object against expected values. func cmpChecksum(oi minio.ObjectInfo, chksums wantChecksums) error { if oi.ChecksumCRC64NVME != chksums[minio.ChecksumCRC64NVME] { return fmt.Errorf("Checksum mismatch for CRC64NVME, want: %s, got: %s", chksums[minio.ChecksumCRC64NVME], oi.ChecksumCRC64NVME) } if oi.ChecksumCRC32C != chksums[minio.ChecksumCRC32C] { return fmt.Errorf("Checksum mismatch for CRC32C, want: %s, got: %s", chksums[minio.ChecksumCRC32C], oi.ChecksumCRC32C) } if oi.ChecksumCRC32 != chksums[minio.ChecksumCRC32] { return fmt.Errorf("Checksum mismatch for CRC32, want: %s, got: %s", chksums[minio.ChecksumCRC32], oi.ChecksumCRC32) } if oi.ChecksumSHA1 != chksums[minio.ChecksumSHA1] { return fmt.Errorf("Checksum mismatch for SHA1, want: %s, got: %s", chksums[minio.ChecksumSHA1], oi.ChecksumSHA1) } if oi.ChecksumSHA256 != chksums[minio.ChecksumSHA256] { return fmt.Errorf("Checksum mismatch for SHA256, want: %s, got: %s", chksums[minio.ChecksumSHA256], oi.ChecksumSHA256) } return nil } func main() { slog.SetDefault(slog.New(slog.NewJSONHandler( os.Stdout, &slog.HandlerOptions{ Level: slog.LevelInfo, ReplaceAttr: func(groups []string, a slog.Attr) slog.Attr { if a.Key == slog.MessageKey || a.Value.String() == "" { return slog.Attr{} } return a }, }, ))) tls := mustParseBool(os.Getenv(enableHTTPS)) kms := mustParseBool(os.Getenv(enableKMS)) if os.Getenv(enableKMS) == "" { // Default to KMS tests. kms = true } // execute tests if isFullMode() { testPutObjectWithChecksums() testPutObjectWithTrailingChecksums() testPutMultipartObjectWithChecksums() testCopyObjectWithChecksums() testReplaceObjectWithChecksums() testCorsSetGetDelete() testCors() testListMultipartUpload() testGetObjectAttributes() testGetObjectAttributesErrorCases() testMakeBucketErrorV2() testGetObjectClosedTwiceV2() testFPutObjectV2() testMakeBucketRegionsV2() testGetObjectReadSeekFunctionalV2() testGetObjectReadAtFunctionalV2() testGetObjectRanges() testCopyObjectV2() testFunctionalV2() testComposeObjectErrorCasesV2() testCompose10KSourcesV2() testUserMetadataCopyingV2() testPutObject0ByteV2() testPutObjectMetadataNonUSASCIIV2() testPutObjectNoLengthV2() testPutObjectsUnknownV2() testPutObjectWithAutoChecksums() testGetObjectContextV2() testFPutObjectContextV2() testFGetObjectContextV2() testPutObjectContextV2() testPutObjectWithVersioning() testMakeBucketError() testMakeBucketRegions() testPutObjectWithMetadata() testPutObjectReadAt() testPutObjectStreaming() testPutObjectPreconditionOnNonExistent() testGetObjectSeekEnd() testGetObjectClosedTwice() testGetObjectS3Zip() testRemoveMultipleObjects() testRemoveMultipleObjectsWithResult() testRemoveMultipleObjectsIter() testFPutObjectMultipart() testFPutObject() testGetObjectReadSeekFunctional() testGetObjectReadAtFunctional() testGetObjectReadAtWhenEOFWasReached() testPresignedPostPolicy() testPresignedPostPolicyWrongFile() testPresignedPostPolicyEmptyFileName() testCopyObject() testComposeObjectErrorCases() testCompose10KSources() testUserMetadataCopying() testBucketNotification() testFunctional() testGetObjectModified() testPutObjectUploadSeekedObject() testGetObjectContext() testFPutObjectContext() testFGetObjectContext() testGetObjectACLContext() testPutObjectContext() testStorageClassMetadataPutObject() testStorageClassInvalidMetadataPutObject() testStorageClassMetadataCopyObject() testPutObjectWithContentLanguage() testListObjects() testRemoveObjects() testRemoveObjectsIter() testListObjectVersions() testStatObjectWithVersioning() testGetObjectWithVersioning() testCopyObjectWithVersioning() testConcurrentCopyObjectWithVersioning() testComposeObjectWithVersioning() testRemoveObjectWithVersioning() testRemoveObjectsWithVersioning() testObjectTaggingWithVersioning() testTrailingChecksums() testPutObjectWithAutomaticChecksums() testGetBucketTagging() testSetBucketTagging() testRemoveBucketTagging() // SSE-C tests will only work over TLS connection. if tls { testGetObjectAttributesSSECEncryption() testSSECEncryptionPutGet() testSSECEncryptionFPut() testSSECEncryptedGetObjectReadAtFunctional() testSSECEncryptedGetObjectReadSeekFunctional() testEncryptedCopyObjectV2() testEncryptedSSECToSSECCopyObject() testEncryptedSSECToUnencryptedCopyObject() testUnencryptedToSSECCopyObject() testUnencryptedToUnencryptedCopyObject() testEncryptedEmptyObject() testDecryptedCopyObject() testSSECEncryptedToSSECCopyObjectPart() testSSECMultipartEncryptedToSSECCopyObjectPart() testSSECEncryptedToUnencryptedCopyPart() testUnencryptedToSSECCopyObjectPart() testUnencryptedToUnencryptedCopyPart() testEncryptedSSECToSSES3CopyObject() testEncryptedSSES3ToSSECCopyObject() testSSECEncryptedToSSES3CopyObjectPart() testSSES3EncryptedToSSECCopyObjectPart() } // KMS tests if kms { testSSES3EncryptionPutGet() testSSES3EncryptionFPut() testSSES3EncryptedGetObjectReadAtFunctional() testSSES3EncryptedGetObjectReadSeekFunctional() testEncryptedSSES3ToSSES3CopyObject() testEncryptedSSES3ToUnencryptedCopyObject() testUnencryptedToSSES3CopyObject() testUnencryptedToSSES3CopyObjectPart() testSSES3EncryptedToUnencryptedCopyPart() testSSES3EncryptedToSSES3CopyObjectPart() } } else { testFunctional() testFunctionalV2() } }