[full-ci] chore: bump reva to v2.44.0 update opencloud version 6.2.0 (#2734)

This commit is contained in:
Viktor Scharf
2026-05-11 17:08:39 +02:00
committed by GitHub
parent a84820865e
commit cf0ec50da9
171 changed files with 15555 additions and 4260 deletions
+5
View File
@@ -175,6 +175,11 @@ func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName stri
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+17 -5
View File
@@ -150,6 +150,11 @@ type UploadInfo struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string
ChecksumXXHash3 string
ChecksumXXHash128 string
ChecksumMode string
}
@@ -168,11 +173,12 @@ type ObjectInfo struct {
// each parts concatenated into one string.
ETag string `json:"etag"`
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
ContentEncoding string `json:"contentEncoding"` // A standard MIME type describing encoding of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
// Collection of additional metadata on the object.
// eg: x-amz-meta-*, content-encoding etc.
@@ -226,6 +232,12 @@ type ObjectInfo struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string
ChecksumXXHash3 string
ChecksumXXHash128 string
ChecksumAlgorithm string
ChecksumMode string `xml:"ChecksumType"`
Internal *struct {
+96 -10
View File
@@ -23,6 +23,7 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -88,10 +89,17 @@ type ObjectAttributesResponse struct {
StorageClass string
ObjectSize int
Checksum struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
ChecksumType string `xml:",omitempty"`
}
ObjectParts struct {
PartsCount int
@@ -105,12 +113,90 @@ type ObjectAttributesResponse struct {
// ObjectAttributePart is used by ObjectAttributesResponse to describe an object part
type ObjectAttributePart struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
PartNumber int
Size int
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
PartNumber int
Size int
}
// ChecksumMap returns a map of checksums for the object.
func (o *ObjectAttributesResponse) ChecksumMap() map[string]string {
res := make(map[string]string)
setif := func(typ ChecksumType, value string) {
if value != "" {
res[typ.Key()] = value
}
}
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
return res
}
// ChecksumMode returns the checksum mode of the object.
// If unable to determine, returns ChecksumUnknownMode.
func (o *ObjectAttributesResponse) ChecksumMode() ChecksumMode {
t := o.ChecksumType()
if !t.IsSet() {
return ChecksumUnknownMode
}
switch o.Checksum.ChecksumType {
case amzChecksumModeComposite:
return ChecksumCompositeMode
case amzChecksumModeFullObject:
return ChecksumFullObjectMode
case "":
// Likely not supported by the server.
if o.Checksum.ChecksumCRC64NVME != "" || !strings.ContainsRune(o.ETag, '-') {
// Always full object.
return ChecksumFullObjectMode
}
if !t.CanMergeCRC() {
// Only composite possible.
return ChecksumCompositeMode
}
}
return ChecksumUnknownMode
}
// ChecksumType returns the checksum type of the object.
// If none is set, returns ChecksumNone.
func (o *ObjectAttributesResponse) ChecksumType() ChecksumType {
t := ChecksumNone
setif := func(typ ChecksumType, value string) {
if value != "" {
t = typ
}
}
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
if t.IsSet() && o.Checksum.ChecksumType == amzChecksumModeFullObject {
t |= ChecksumFullObject
}
return t
}
func (o *ObjectAttributes) parseResponse(resp *http.Response) (err error) {
+7 -1
View File
@@ -404,7 +404,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
}
var (
keyMarker = ""
keyMarker = opts.StartAfter
versionIDMarker = ""
preName = ""
preKey = ""
@@ -432,12 +432,18 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
UserMetadata: version.UserMetadata,
Internal: version.Internal,
NumVersions: numVersions,
ChecksumAlgorithm: version.ChecksumAlgorithm,
ChecksumMode: version.ChecksumType,
ChecksumCRC32: version.ChecksumCRC32,
ChecksumCRC32C: version.ChecksumCRC32C,
ChecksumSHA1: version.ChecksumSHA1,
ChecksumSHA256: version.ChecksumSHA256,
ChecksumCRC64NVME: version.ChecksumCRC64NVME,
ChecksumMD5: version.ChecksumMD5,
ChecksumSHA512: version.ChecksumSHA512,
ChecksumXXHash64: version.ChecksumXXHash64,
ChecksumXXHash3: version.ChecksumXXHash3,
ChecksumXXHash128: version.ChecksumXXHash128,
}
if !yield(info) {
return false
+15
View File
@@ -192,6 +192,11 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -353,6 +358,11 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
}
objPart.Size = p.size
objPart.PartNumber = p.partNumber
@@ -458,6 +468,11 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME,
ChecksumMD5: completeMultipartUploadResult.ChecksumMD5,
ChecksumSHA512: completeMultipartUploadResult.ChecksumSHA512,
ChecksumXXHash64: completeMultipartUploadResult.ChecksumXXHash64,
ChecksumXXHash3: completeMultipartUploadResult.ChecksumXXHash3,
ChecksumXXHash128: completeMultipartUploadResult.ChecksumXXHash128,
ChecksumMode: completeMultipartUploadResult.ChecksumType,
}, nil
}
+20
View File
@@ -259,6 +259,11 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME,
ChecksumMD5: uploadRes.Part.ChecksumMD5,
ChecksumSHA512: uploadRes.Part.ChecksumSHA512,
ChecksumXXHash64: uploadRes.Part.ChecksumXXHash64,
ChecksumXXHash3: uploadRes.Part.ChecksumXXHash3,
ChecksumXXHash128: uploadRes.Part.ChecksumXXHash128,
})
}
}
@@ -418,6 +423,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -617,6 +627,11 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -796,6 +811,11 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+5
View File
@@ -502,6 +502,11 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
+41
View File
@@ -113,6 +113,12 @@ type Version struct {
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
ChecksumAlgorithm string `xml:",omitempty"`
ChecksumType string `xml:",omitempty"`
isDeleteMarker bool
@@ -297,6 +303,11 @@ type ObjectPart struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
}
// Checksum will return the checksum for the given type.
@@ -313,6 +324,16 @@ func (c ObjectPart) Checksum(t ChecksumType) string {
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
case t.Is(ChecksumMD5):
return c.ChecksumMD5
case t.Is(ChecksumSHA512):
return c.ChecksumSHA512
case t.Is(ChecksumXXHash64):
return c.ChecksumXXHash64
case t.Is(ChecksumXXHash3):
return c.ChecksumXXHash3
case t.Is(ChecksumXXHash128):
return c.ChecksumXXHash128
}
return ""
}
@@ -382,6 +403,11 @@ type completeMultipartUploadResult struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string `xml:"ChecksumXXHASH64"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128"`
ChecksumType string
}
@@ -398,6 +424,11 @@ type CompletePart struct {
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
}
// Checksum will return the checksum for the given type.
@@ -414,6 +445,16 @@ func (c CompletePart) Checksum(t ChecksumType) string {
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
case t.Is(ChecksumMD5):
return c.ChecksumMD5
case t.Is(ChecksumSHA512):
return c.ChecksumSHA512
case t.Is(ChecksumXXHash64):
return c.ChecksumXXHash64
case t.Is(ChecksumXXHash3):
return c.ChecksumXXHash3
case t.Is(ChecksumXXHash128):
return c.ChecksumXXHash128
}
return ""
}
+17 -9
View File
@@ -52,6 +52,12 @@ type UpdateObjectEncryptionOptions struct {
VersionID string
}
// UpdateObjectEncryptionResult holds the result of an UpdateObjectEncryption call.
type UpdateObjectEncryptionResult struct {
// VersionID is the version ID of the object that was updated, if versioning is enabled.
VersionID string
}
// UpdateObjectEncryption changes the encryption configuration of an existing object in-place.
// The object must already be encrypted with SSE-S3 or SSE-KMS. SSE-C objects are not supported.
// This operation rotates the data encryption key envelope without re-reading/re-writing object data.
@@ -62,19 +68,19 @@ type UpdateObjectEncryptionOptions struct {
// - objectName: Name of the object
// - opts: Options including KMSKeyArn (required), optional BucketKeyEnabled, and optional VersionID
//
// Returns an error if the operation fails.
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) error {
// Returns the version ID of the updated object (if versioning is enabled) and an error if the operation fails.
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) (UpdateObjectEncryptionResult, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if opts.KMSKeyArn == "" {
return errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
return UpdateObjectEncryptionResult{}, errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
}
// Get resources properly escaped and lined up before
@@ -96,7 +102,7 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
bodyData, err := xml.Marshal(reqBody)
if err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
reqMetadata := requestMetadata{
@@ -113,10 +119,12 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, objectName)
return UpdateObjectEncryptionResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
}
return nil
return UpdateObjectEncryptionResult{
VersionID: resp.Header.Get(amzVersionID),
}, nil
}
+13 -1
View File
@@ -198,6 +198,9 @@ func New(endpoint string, opts *Options) (*Client, error) {
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
// for backwards compatibility.
clnt.s3DualstackEnabled = true
} else if s3utils.IsAmazonOutpostsEndpoint(*clnt.endpointURL) {
// S3 on Outposts uses signature v4 with service name s3-outposts.
clnt.overrideSignerType = credentials.SignatureV4
}
return clnt, nil
@@ -912,7 +915,11 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
} else if signerType.IsV4() {
// Presign URL with signature v4.
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.PreSignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
} else {
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
}
}
return req, nil
}
@@ -971,6 +978,9 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
req = signer.StreamingSignV4Express(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.StreamingSignV4Outposts(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
} else {
req = signer.StreamingSignV4(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
@@ -991,6 +1001,8 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.SignV4TrailerOutposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
} else {
// Add signature version '4' authorization header.
req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
+6 -1
View File
@@ -209,6 +209,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
}
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
region := getDefaultLocation(*c.endpointURL, c.region)
req = signer.SignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, region)
} else {
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
}
return req, nil
}
+67 -4
View File
@@ -18,8 +18,10 @@
package minio
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/binary"
"errors"
@@ -30,8 +32,10 @@ import (
"sort"
"strings"
"github.com/cespare/xxhash/v2"
"github.com/klauspost/crc32"
"github.com/minio/crc64nvme"
"github.com/zeebo/xxh3"
)
// ChecksumMode contains information about the checksum mode on the object
@@ -49,6 +53,9 @@ const (
// checksumModeMask is a mask for valid checksum mode types.
checksumModeMask = checksumLastMode - 1
// ChecksumUnknownMode indicates no or unknown checksum mode.
ChecksumUnknownMode ChecksumMode = 0
)
// Is returns if c is all of t.
@@ -64,9 +71,9 @@ func (c ChecksumMode) Key() string {
func (c ChecksumMode) String() string {
switch c & checksumModeMask {
case ChecksumFullObjectMode:
return "FULL_OBJECT"
return amzChecksumModeFullObject
case ChecksumCompositeMode:
return "COMPOSITE"
return amzChecksumModeComposite
}
return ""
}
@@ -86,6 +93,16 @@ const (
ChecksumCRC32C
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
ChecksumCRC64NVME
// ChecksumMD5 indicates an MD5 checksum.
ChecksumMD5
// ChecksumSHA512 indicates a SHA-512 checksum.
ChecksumSHA512
// ChecksumXXHash64 indicates an XXHash64 checksum.
ChecksumXXHash64
// ChecksumXXHash3 indicates an XXH3-64 checksum.
ChecksumXXHash3
// ChecksumXXHash128 indicates an XXH3-128 checksum.
ChecksumXXHash128
// Keep after all valid checksums
checksumLast
@@ -112,7 +129,15 @@ const (
amzChecksumSHA1 = "x-amz-checksum-sha1"
amzChecksumSHA256 = "x-amz-checksum-sha256"
amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme"
amzChecksumMD5 = "x-amz-checksum-md5"
amzChecksumSHA512 = "x-amz-checksum-sha512"
amzChecksumXXHash64 = "x-amz-checksum-xxhash64"
amzChecksumXXHash3 = "x-amz-checksum-xxhash3"
amzChecksumXXHash128 = "x-amz-checksum-xxhash128"
amzChecksumMode = "x-amz-checksum-type"
amzChecksumModeComposite = "COMPOSITE"
amzChecksumModeFullObject = "FULL_OBJECT"
)
// Base returns the base type, without modifiers.
@@ -139,6 +164,16 @@ func (c ChecksumType) Key() string {
return amzChecksumSHA256
case ChecksumCRC64NVME:
return amzChecksumCRC64NVME
case ChecksumMD5:
return amzChecksumMD5
case ChecksumSHA512:
return amzChecksumSHA512
case ChecksumXXHash64:
return amzChecksumXXHash64
case ChecksumXXHash3:
return amzChecksumXXHash3
case ChecksumXXHash128:
return amzChecksumXXHash128
}
return ""
}
@@ -146,7 +181,8 @@ func (c ChecksumType) Key() string {
// CanComposite will return if the checksum type can be used for composite multipart upload on AWS.
func (c ChecksumType) CanComposite() bool {
switch c & checksumMask {
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C:
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C,
ChecksumMD5, ChecksumSHA512, ChecksumXXHash64, ChecksumXXHash3, ChecksumXXHash128:
return true
}
return false
@@ -186,6 +222,14 @@ func (c ChecksumType) RawByteLen() int {
return sha256.Size
case ChecksumCRC64NVME:
return crc64nvme.Size
case ChecksumXXHash64, ChecksumXXHash3:
return 8
case ChecksumMD5:
return md5.Size
case ChecksumSHA512:
return sha512.Size
case ChecksumXXHash128:
return 16
}
return 0
}
@@ -206,6 +250,16 @@ func (c ChecksumType) Hasher() hash.Hash {
return sha256.New()
case ChecksumCRC64NVME:
return crc64nvme.New()
case ChecksumMD5:
return md5.New()
case ChecksumSHA512:
return sha512.New()
case ChecksumXXHash64:
return xxhash.New()
case ChecksumXXHash3:
return xxh3.New()
case ChecksumXXHash128:
return xxh3.New128()
}
return nil
}
@@ -233,7 +287,6 @@ func (c ChecksumType) EncodeToString(b []byte) string {
}
// String returns the type as a string.
// CRC32, CRC32C, SHA1, and SHA256 for valid values.
// Empty string for unset and "<invalid>" if not valid.
func (c ChecksumType) String() string {
switch c & checksumMask {
@@ -249,6 +302,16 @@ func (c ChecksumType) String() string {
return ""
case ChecksumCRC64NVME:
return "CRC64NVME"
case ChecksumMD5:
return "MD5"
case ChecksumSHA512:
return "SHA512"
case ChecksumXXHash64:
return "XXHASH64"
case ChecksumXXHash3:
return "XXHASH3"
case ChecksumXXHash128:
return "XXHASH128"
}
return "<invalid>"
}
+4 -4
View File
@@ -94,15 +94,15 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
},
"us-iso-east-1": {
"s3.us-iso-east-1.c2s.ic.gov",
"s3.dualstack.us-iso-east-1.c2s.ic.gov",
"", // dualstack endpoint doesn't exist
},
"us-isob-east-1": {
"s3.us-isob-east-1.sc2s.sgov.gov",
"s3.dualstack.us-isob-east-1.sc2s.sgov.gov",
"", // dualstack endpoint doesn't exist
},
"us-iso-west-1": {
"s3.us-iso-west-1.c2s.ic.gov",
"s3.dualstack.us-iso-west-1.c2s.ic.gov",
"", // dualstack endpoint doesn't exist
},
"us-west-2": {
"s3.us-west-2.amazonaws.com",
@@ -269,7 +269,7 @@ func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
}
return "s3.us-east-1.amazonaws.com"
}
if useDualstack {
if useDualstack && s3Endpoint.dualstackEndpoint != "" {
return s3Endpoint.dualstackEndpoint
}
return s3Endpoint.endpoint
+160 -4
View File
@@ -73,8 +73,25 @@ const (
enableKMS = "ENABLE_KMS"
appVersion = "0.1.0"
skipCERTValidation = "SKIP_CERT_VALIDATION"
// TODO: remove when server supports the 2026 checksum types.
ignore2026Checksums = true
)
func ignore2026ChecksumError(cs minio.ChecksumType, err error) bool {
if !ignore2026Checksums {
return false
}
switch cs.Base() {
case minio.ChecksumMD5, minio.ChecksumSHA512,
minio.ChecksumXXHash64, minio.ChecksumXXHash3, minio.ChecksumXXHash128:
default:
return false
}
var er minio.ErrorResponse
return errors.As(err, &er) && er.Code == "InvalidArgument"
}
func createHTTPTransport() (transport *http.Transport) {
var err error
transport, err = minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
@@ -2096,6 +2113,11 @@ func testPutObjectWithChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
@@ -2160,6 +2182,10 @@ func testPutObjectWithChecksums() {
UserMetadata: meta,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2168,6 +2194,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
@@ -2191,6 +2222,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if st.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
@@ -2238,6 +2274,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
cmpChecksum(st.ChecksumMD5, "")
cmpChecksum(st.ChecksumSHA512, "")
cmpChecksum(st.ChecksumXXHash64, "")
cmpChecksum(st.ChecksumXXHash3, "")
cmpChecksum(st.ChecksumXXHash128, "")
delete(args, "range")
delete(args, "metadata")
@@ -2283,6 +2324,11 @@ func testPutObjectWithTrailingChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumCRC64NVME},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
@@ -2329,6 +2375,10 @@ func testPutObjectWithTrailingChecksums() {
Checksum: test.cs,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2341,6 +2391,11 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@@ -2361,7 +2416,12 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@@ -2407,6 +2467,11 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
cmpChecksum(st.ChecksumMD5, "")
cmpChecksum(st.ChecksumSHA512, "")
cmpChecksum(st.ChecksumXXHash64, "")
cmpChecksum(st.ChecksumXXHash3, "")
cmpChecksum(st.ChecksumXXHash128, "")
function = "GetObjectAttributes(...)"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
@@ -2418,6 +2483,16 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(s.Checksum.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(s.Checksum.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(s.Checksum.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(s.Checksum.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(s.Checksum.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(s.Checksum.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if s.Checksum.ChecksumType != "" && s.Checksum.ChecksumType != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "ChecksumType mismatch in GetObjectAttributes", fmt.Errorf("want %s, got %s", minio.ChecksumFullObjectMode.String(), s.Checksum.ChecksumType))
return
}
delete(args, "range")
delete(args, "metadata")
@@ -2495,6 +2570,11 @@ func testPutMultipartObjectWithChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumFullObjectCRC32},
{cs: minio.ChecksumFullObjectCRC32C},
{cs: minio.ChecksumCRC64NVME},
@@ -2552,6 +2632,10 @@ func testPutMultipartObjectWithChecksums() {
Checksum: cs,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2567,6 +2651,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(resp.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(resp.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(resp.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(resp.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(resp.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(resp.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(resp.ChecksumXXHash128, wantChksm)
}
args["section"] = "HeadObject"
@@ -2586,6 +2680,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(st.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantChksm)
}
// Use the CopyObject API to make a copy, in the case it was a composite checksum,
@@ -2622,6 +2726,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm)
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantFullObjectChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantFullObjectChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantFullObjectChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantFullObjectChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantFullObjectChksm)
}
args["section"] = "GetObjectAttributes"
@@ -2644,6 +2758,35 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(s.Checksum.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(s.Checksum.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(s.Checksum.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(s.Checksum.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(s.Checksum.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(s.Checksum.ChecksumXXHash128, wantChksm)
}
if s.Checksum.ChecksumType != "" {
var wantType string
if test.cs.FullObjectRequested() {
wantType = minio.ChecksumFullObjectMode.String()
} else {
wantType = minio.ChecksumCompositeMode.String()
}
cmpChecksum(s.Checksum.ChecksumType, wantType)
}
for _, part := range s.ObjectParts.Parts {
if test.cs == minio.ChecksumCRC64NVME && part.ChecksumCRC64NVME == "" {
logError(testName, function, args, startTime, "", "Part missing CRC64NVME checksum in GetObjectAttributes", fmt.Errorf("part %d", part.PartNumber))
return
}
}
// Read the data back
@@ -2684,6 +2827,16 @@ func testPutMultipartObjectWithChecksums() {
if st.ChecksumCRC64NVME != "" {
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
}
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantChksm)
}
delete(args, "metadata")
@@ -3475,6 +3628,8 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
checksumFound = true
} else if v.ChecksumCRC32C != "" {
checksumFound = true
} else if v.ChecksumCRC64NVME != "" {
checksumFound = true
}
if !checksumFound {
partsMissingChecksum = true
@@ -3497,6 +3652,7 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" ||
OA.Checksum.ChecksumCRC32C != "" ||
OA.Checksum.ChecksumCRC64NVME != "" ||
OA.Checksum.ChecksumSHA1 != "" ||
OA.Checksum.ChecksumSHA256 != "")
@@ -14789,6 +14945,9 @@ func main() {
// execute tests
if isFullMode() {
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums()
testCopyObjectWithChecksums()
testReplaceObjectWithChecksums()
testCorsSetGetDelete()
@@ -14808,9 +14967,6 @@ func main() {
testComposeObjectErrorCasesV2()
testCompose10KSourcesV2()
testUserMetadataCopyingV2()
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums()
testPutObject0ByteV2()
testPutObjectMetadataNonUSASCIIV2()
testPutObjectNoLengthV2()
+21
View File
@@ -119,6 +119,10 @@ var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`)
// amazonS3HostPrivateLink - regular expression used to determine if an arg is s3 host in AWS PrivateLink interface endpoints style
var amazonS3HostPrivateLink = regexp.MustCompile(`^(?:bucket|accesspoint).vpce-.*?.s3.(.*?).vpce.amazonaws.com$`)
// amazonS3HostOutposts - regular expression used to determine if an arg is S3 on Outposts endpoint.
// Pattern: <something>.s3-outposts.<region>.amazonaws.com
var amazonS3HostOutposts = regexp.MustCompile(`^(.+)\.s3-outposts\.([a-z0-9-]+)\.amazonaws\.com$`)
// GetRegionFromURL - returns a region from url host.
func GetRegionFromURL(endpointURL url.URL) string {
if endpointURL == sentinelURL {
@@ -181,6 +185,11 @@ func GetRegionFromURL(endpointURL url.URL) string {
return parts[1]
}
parts = amazonS3HostOutposts.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 2 {
return parts[2]
}
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
if strings.HasPrefix(parts[1], "xpress-") {
@@ -210,8 +219,20 @@ func IsAmazonExpressZonalEndpoint(endpointURL url.URL) bool {
return amazonS3HostExpress.MatchString(endpointURL.Hostname())
}
// IsAmazonOutpostsEndpoint - Match if the endpoint is S3 on Outposts endpoint.
func IsAmazonOutpostsEndpoint(endpointURL url.URL) bool {
if endpointURL == sentinelURL {
return false
}
return amazonS3HostOutposts.MatchString(endpointURL.Hostname())
}
// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint.
// S3 on Outposts is not treated as Amazon S3 here so that the client keeps path-style and does not replace the host.
func IsAmazonEndpoint(endpointURL url.URL) bool {
if IsAmazonOutpostsEndpoint(endpointURL) {
return false
}
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" || endpointURL.Hostname() == "s3.amazonaws.com" {
return true
}
@@ -90,32 +90,34 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
return streamLen
}
// buildChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
// buildChunkStringToSignWithService - like buildChunkStringToSign but with configurable service type.
func buildChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
if serviceType == "" {
serviceType = ServiceTypeS3
}
stringToSignParts := []string{
streamingPayloadHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
getScope(region, t, serviceType),
previousSig,
emptySHA256,
chunkChecksum,
}
return strings.Join(stringToSignParts, "\n")
}
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
// buildTrailerChunkStringToSignWithService - like buildTrailerChunkStringToSign but with configurable service type.
func buildTrailerChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
if serviceType == "" {
serviceType = ServiceTypeS3
}
stringToSignParts := []string{
streamingTrailerHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
getScope(region, t, serviceType),
previousSig,
chunkChecksum,
}
return strings.Join(stringToSignParts, "\n")
}
@@ -150,36 +152,42 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
// serviceType defaults to ServiceTypeS3 when empty.
func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
previousSignature, secretAccessKey string,
previousSignature, secretAccessKey, serviceType string,
) string {
chunkStringToSign := buildChunkStringToSign(reqTime, region,
previousSignature, chunkCheckSum)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
if serviceType == "" {
serviceType = ServiceTypeS3
}
chunkStringToSign := buildChunkStringToSignWithService(reqTime, region,
previousSignature, chunkCheckSum, serviceType)
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
return getSignature(signingKey, chunkStringToSign)
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
// buildTrailerChunkSignature - returns chunk signature for trailer chunk.
// serviceType defaults to ServiceTypeS3 when empty.
func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
previousSignature, secretAccessKey string,
previousSignature, secretAccessKey, serviceType string,
) string {
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
previousSignature, chunkChecksum)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
if serviceType == "" {
serviceType = ServiceTypeS3
}
chunkStringToSign := buildTrailerChunkStringToSignWithService(reqTime, region,
previousSignature, chunkChecksum, serviceType)
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
return getSignature(signingKey, chunkStringToSign)
}
// getSeedSignature - returns the seed signature for a given request.
func (s *StreamingReader) setSeedSignature(req *http.Request) {
// Get canonical request
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req))
// Get string to sign from canonical request.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3)
// Calculate signature.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, serviceType)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, serviceType)
s.seedSignature = getSignature(signingKey, stringToSign)
}
@@ -190,6 +198,7 @@ type StreamingReader struct {
secretAccessKey string
sessionToken string
region string
serviceType string // e.g. ServiceTypeS3, ServiceTypeS3Outposts; empty means S3
prevSignature string
seedSignature string
contentLen int64 // Content-Length from req header
@@ -214,8 +223,12 @@ func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
s.sh256.Write(s.chunkBuf[:chunkLen])
chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
signature := buildChunkSignature(chunckChecksum, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
s.region, s.prevSignature, s.secretAccessKey, serviceType)
// For next chunk signature computation
s.prevSignature = signature
@@ -249,9 +262,12 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
s.sh256.Reset()
s.sh256.Write(s.chunkBuf)
chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
// Compute chunk signature
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
s.region, s.prevSignature, s.secretAccessKey, serviceType)
// For next chunk signature computation
s.prevSignature = signature
@@ -376,6 +392,40 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
return req
}
// StreamingSignV4Outposts - provides chunked upload signatureV4 support for S3 on Outposts (service name s3-outposts).
func StreamingSignV4Outposts(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
) *http.Request {
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
if req.Body == nil {
req.Body = io.NopCloser(bytes.NewReader([]byte("")))
}
stReader := &StreamingReader{
baseReadCloser: req.Body,
accessKeyID: accessKeyID,
secretAccessKey: secretAccessKey,
sessionToken: sessionToken,
region: region,
serviceType: ServiceTypeS3Outposts,
reqTime: reqTime,
chunkBuf: make([]byte, payloadChunkSize),
contentLen: dataLen,
chunkNum: 1,
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
lastChunkSize: int(dataLen % payloadChunkSize),
sh256: sh256,
}
if len(req.Trailer) > 0 {
stReader.trailer = req.Trailer
req.Trailer = nil
}
stReader.setSeedSignature(req)
stReader.setStreamingAuthHeader(req, ServiceTypeS3Outposts)
stReader.prevSignature = stReader.seedSignature
req.Body = stReader
return req
}
// Read - this method performs chunk upload signature providing a
// io.Reader interface.
func (s *StreamingReader) Read(buf []byte) (int, error) {
+51 -3
View File
@@ -38,9 +38,10 @@ const (
// Different service types
const (
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
ServiceTypeS3Express = "s3express"
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
ServiceTypeS3Express = "s3express"
ServiceTypeS3Outposts = "s3-outposts"
)
// Excerpts from @lsegal -
@@ -256,6 +257,38 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
return &req
}
// PreSignV4Outposts presign the request for S3 on Outposts (service name s3-outposts).
func PreSignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
// Presign is not needed for anonymous credentials.
if accessKeyID == "" || secretAccessKey == "" {
return &req
}
t := time.Now().UTC()
credential := GetCredential(accessKeyID, location, t, ServiceTypeS3Outposts)
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
query := req.URL.Query()
query.Set("X-Amz-Algorithm", signV4Algorithm)
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
query.Set("X-Amz-SignedHeaders", signedHeaders)
query.Set("X-Amz-Credential", credential)
if sessionToken != "" {
if v := req.Header.Get("x-amz-s3session-token"); v != "" {
query.Set("X-Amz-S3session-Token", sessionToken)
} else {
query.Set("X-Amz-Security-Token", sessionToken)
}
}
req.URL.RawQuery = query.Encode()
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))
stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3Outposts)
signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3Outposts)
signature := getSignature(signingKey, stringToSign)
req.URL.RawQuery += "&X-Amz-Signature=" + signature
return &req
}
// PostPresignSignatureV4 - presigned signature for PostPolicy
// requests.
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
@@ -393,3 +426,18 @@ func SignV4TrailerExpress(req http.Request, accessKeyID, secretAccessKey, sessio
func SignV4Trailer(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, trailer)
}
// SignV4Outposts sign the request for S3 on Outposts (service name s3-outposts).
func SignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, nil)
}
// SignV4WithServiceType signs a request with AWS Signature Version 4 using a custom service type.
func SignV4WithServiceType(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, serviceType, nil)
}
// SignV4TrailerOutposts sign the request with trailer for S3 on Outposts (service name s3-outposts).
func SignV4TrailerOutposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, trailer)
}
+21
View File
@@ -18,6 +18,7 @@
package tags
import (
"encoding/json"
"encoding/xml"
"io"
"net/url"
@@ -293,6 +294,26 @@ func (tags Tags) ToMap() map[string]string {
return tags.TagSet.toMap()
}
// MarshalJSON encodes Tags as a flat JSON object {"key":"value",...}.
func (tags Tags) MarshalJSON() ([]byte, error) {
return json.Marshal(tags.ToMap())
}
// UnmarshalJSON decodes a flat JSON object {"key":"value",...} into Tags.
func (tags *Tags) UnmarshalJSON(data []byte) error {
var m map[string]string
if err := json.Unmarshal(data, &m); err != nil {
return err
}
if tags.TagSet == nil {
tags.TagSet = &tagSet{
tagMap: make(map[string]string),
}
}
tags.TagSet.tagMap = m
return nil
}
// MapToObjectTags converts an input map of key and value into
// *Tags data structure with validation.
func MapToObjectTags(tagMap map[string]string) (*Tags, error) {
+27
View File
@@ -179,6 +179,11 @@ func isValidEndpointURL(endpointURL url.URL) error {
return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
}
}
if strings.Contains(host, "s3-outposts") {
if !s3utils.IsAmazonOutpostsEndpoint(endpointURL) {
return errInvalidArgument("S3 Outposts endpoint must match <prefix>.s3-outposts.<region>.amazonaws.com")
}
}
return nil
}
@@ -304,6 +309,21 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Region: h.Get("x-amz-bucket-region"),
}
}
mtimeStr := h.Get("X-Minio-Source-Mtime")
if mtimeStr != "" {
mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
if err != nil {
return ObjectInfo{}, ErrorResponse{
Code: InternalError,
Message: fmt.Sprintf("X-Minio-Source-Mtime is not in supported format: %v", err),
BucketName: bucketName,
Key: objectName,
RequestID: h.Get("x-amz-request-id"),
HostID: h.Get("x-amz-id-2"),
Region: h.Get("x-amz-bucket-region"),
}
}
}
// Fetch content type if any present.
contentType := strings.TrimSpace(h.Get("Content-Type"))
@@ -381,6 +401,7 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Size: size,
LastModified: mtime,
ContentType: contentType,
ContentEncoding: strings.TrimSpace(h.Get("Content-Encoding")),
Expires: expiry,
VersionID: h.Get(amzVersionID),
IsDeleteMarker: deleteMarker,
@@ -402,6 +423,12 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumAlgorithm: h.Get(amzChecksumAlgo),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}