sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Bump reva

Browse Source 
Fixes: #1774
This commit is contained in:
Ralf Haferkamp
2025-11-06 17:24:42 +01:00
committed by Ralf Haferkamp
parent 4ffb79b680
commit 96042752bb
76 changed files with 8001 additions and 639 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+6 -5
View File
@@ -65,7 +65,7 @@ require (
github.com/open-policy-agent/opa v1.9.0
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76
github.com/opencloud-eu/reva/v2 v2.39.2-0.20251030154544-cac8a0257da6
github.com/opencloud-eu/reva/v2 v2.39.2-0.20251106122902-c13e27f55362
github.com/opensearch-project/opensearch-go/v4 v4.5.0
github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1
@@ -161,7 +161,7 @@ require (
github.com/bombsimon/logrusr/v3 v3.1.0 // indirect
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
github.com/cenkalti/backoff/v5 v5.0.3 // indirect
github.com/ceph/go-ceph v0.35.0 // indirect
github.com/ceph/go-ceph v0.36.0 // indirect
github.com/cespare/xxhash/v2 v2.3.0 // indirect
github.com/cevaris/ordered_map v0.0.0-20190319150403-3adeae072e73 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
@@ -234,7 +234,7 @@ require (
github.com/golang-jwt/jwt/v4 v4.5.2 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/gomodule/redigo v1.9.2 // indirect
github.com/gomodule/redigo v1.9.3 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/go-tpm v0.9.6 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
@@ -257,6 +257,7 @@ require (
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.18.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/kovidgoyal/go-parallel v1.0.1 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/lestrrat-go/blackmagic v1.0.4 // indirect
@@ -281,10 +282,10 @@ require (
github.com/mendsley/gojwk v0.0.0-20141217222730-4d5ec6e58103 // indirect
github.com/miekg/dns v1.1.57 // indirect
github.com/mileusna/useragent v1.3.5 // indirect
github.com/minio/crc64nvme v1.0.2 // indirect
github.com/minio/crc64nvme v1.1.0 // indirect
github.com/minio/highwayhash v1.0.3 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.95 // indirect
github.com/minio/minio-go/v7 v7.0.97 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
go.sum
+12 -10
View File
@@ -210,8 +210,8 @@ github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1x
github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw=
github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/ceph/go-ceph v0.35.0 h1:wcDUbsjeNJ7OfbWCE7I5prqUL794uXchopw3IvrGQkk=
github.com/ceph/go-ceph v0.35.0/go.mod h1:ILF8WKhQQ2p2YuX1oWigkmsfT39U8T/HS2NrqxExq2s=
github.com/ceph/go-ceph v0.36.0 h1:IDE4vEF+4fmjve+CPjD1WStgfQ+Lh6vD+9PMUI712KI=
github.com/ceph/go-ceph v0.36.0/go.mod h1:fGCbndVDLuHW7q2954d6y+tgPFOBnRLqJRe2YXyngw4=
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -536,8 +536,8 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y=
github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s=
github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
github.com/gomodule/redigo v1.9.3 h1:dNPSXeXv6HCq2jdyWfjgmhBdqnR6PRO3m/G05nvpPC8=
github.com/gomodule/redigo v1.9.3/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q=
@@ -723,6 +723,8 @@ github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYW
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
github.com/klauspost/crc32 v1.3.0 h1:sSmTt3gUt81RP655XGZPElI0PelVTZ6YwCRnPSupoFM=
github.com/klauspost/crc32 v1.3.0/go.mod h1:D7kQaZhnkX/Y0tstFGf8VUzv2UofNGqCjnC3zdHB0Hw=
github.com/kobergj/gowebdav v0.0.0-20250102091030-aa65266db202 h1:A1xJ2NKgiYFiaHiLl9B5yw/gUBACSs9crDykTS3GuQI=
github.com/kobergj/gowebdav v0.0.0-20250102091030-aa65266db202/go.mod h1:bHA7t77X/QFExdeAnDzK6vKM34kEZAcE1OX4MfiwjkE=
github.com/kolo/xmlrpc v0.0.0-20200310150728-e0350524596b/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ=
@@ -837,14 +839,14 @@ github.com/miekg/dns v1.1.57 h1:Jzi7ApEIzwEPLHWRcafCN9LZSBbqQpxjt/wpgvg7wcM=
github.com/miekg/dns v1.1.57/go.mod h1:uqRjCRUuEAA6qsOiJvDd+CFo/vW+y5WR6SNmHE55hZk=
github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNGDws=
github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
github.com/minio/crc64nvme v1.0.2 h1:6uO1UxGAD+kwqWWp7mBFsi5gAse66C4NXO8cmcVculg=
github.com/minio/crc64nvme v1.0.2/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/crc64nvme v1.1.0 h1:e/tAguZ+4cw32D+IO/8GSf5UVr9y+3eJcxZI2WOO/7Q=
github.com/minio/crc64nvme v1.1.0/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.95 h1:ywOUPg+PebTMTzn9VDsoFJy32ZuARN9zhB+K3IYEvYU=
github.com/minio/minio-go/v7 v7.0.95/go.mod h1:wOOX3uxS334vImCNRVyIDdXX9OsXDm89ToynKgqUKlo=
github.com/minio/minio-go/v7 v7.0.97 h1:lqhREPyfgHTB/ciX8k2r8k0D93WaFqxbJX36UZq5occ=
github.com/minio/minio-go/v7 v7.0.97/go.mod h1:re5VXuo0pwEtoNLsNuSr0RrLfT/MBtohwdaSmPPSRSk=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -948,8 +950,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76 h1:vD/EdfDUrv4omSFjrinT8Mvf+8D7f9g4vgQ2oiDrVUI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
github.com/opencloud-eu/reva/v2 v2.39.2-0.20251030154544-cac8a0257da6 h1:BUrCUrRqBg04MJuhnIK4H1KNK4aebK6H/AYcHjQ0DM4=
github.com/opencloud-eu/reva/v2 v2.39.2-0.20251030154544-cac8a0257da6/go.mod h1:Qm0CibFYrFc096OhWWL14nsGiFoE6g/4oMFHV5CqU+Q=
github.com/opencloud-eu/reva/v2 v2.39.2-0.20251106122902-c13e27f55362 h1:O9oHbqPnC+tAQTbaLD4Tj6I5jmSmTLaQCynTHkFP+cI=
github.com/opencloud-eu/reva/v2 v2.39.2-0.20251106122902-c13e27f55362/go.mod h1:hOCR1OHAhGY8ecpq6sIS5Ru1ZOC/hBgNz+sYf6CrO9Y=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=
vendor/github.com/ceph/go-ceph/cephfs/admin/subvolume_snapshot_path.go
Generated Vendored
+1 -1
View File
@@ -1,4 +1,4 @@
//go:build !(octopus || pacific || quincy || reef || squid) && ceph_preview
//go:build !(octopus || pacific || quincy || reef || squid)
package admin
vendor/github.com/gomodule/redigo/redis/conn.go
Generated Vendored
+5 -2
View File
@@ -332,7 +332,10 @@ func DialURLContext(ctx context.Context, rawurl string, options ...DialOption) (
return nil, err
}
if u.Scheme != "redis" && u.Scheme != "rediss" {
switch u.Scheme {
case "redis", "rediss", "valkey", "valkeys":
// valid scheme
default:
return nil, fmt.Errorf("invalid redis URL scheme: %s", u.Scheme)
}
@@ -386,7 +389,7 @@ func DialURLContext(ctx context.Context, rawurl string, options ...DialOption) (
return nil, fmt.Errorf("invalid database: %s", u.Path[1:])
}
options = append(options, DialUseTLS(u.Scheme == "rediss"))
options = append(options, DialUseTLS(u.Scheme == "rediss" || u.Scheme == "valkeys"))
return DialContext(ctx, "tcp", address, options...)
}
vendor/github.com/gomodule/redigo/redis/scan.go
Generated Vendored
+16 -3
View File
@@ -477,7 +477,7 @@ var errScanStructValue = errors.New("redigo.ScanStruct: value must be non-nil po
// ScanStruct uses exported field names to match values in the response. Use
// 'redis' field tag to override the name:
//
// Field int `redis:"myName"`
// Field int `redis:"myName"`
//
// Fields with the tag redis:"-" are ignored.
//
@@ -513,9 +513,9 @@ func ScanStruct(src []interface{}, dest interface{}) error {
continue
}
name, ok := src[i].([]byte)
name, ok := convertToBulk(src[i])
if !ok {
return fmt.Errorf("redigo.ScanStruct: key %d not a bulk string value", i)
return fmt.Errorf("redigo.ScanStruct: key %d not a bulk string value got type: %T", i, src[i])
}
fs := ss.fieldSpec(name)
@@ -530,6 +530,19 @@ func ScanStruct(src []interface{}, dest interface{}) error {
return nil
}
// convertToBulk converts src to a []byte if src is a string or bulk string
// and returns true. Otherwise nil and false is returned.
func convertToBulk(src interface{}) ([]byte, bool) {
switch v := src.(type) {
case []byte:
return v, true
case string:
return []byte(v), true
default:
return nil, false
}
}
var (
errScanSliceValue = errors.New("redigo.ScanSlice: dest must be non-nil pointer to a struct")
)
vendor/github.com/klauspost/crc32/.gitignore
Generated
+24
View File
@@ -0,0 +1,24 @@
# Compiled Object files, Static and Dynamic libs (Shared Objects)
*.o
*.a
*.so
# Folders
_obj
_test
# Architecture specific extensions/prefixes
*.[568vq]
[568vq].out
*.cgo1.go
*.cgo2.c
_cgo_defun.c
_cgo_gotypes.go
_cgo_export.*
_testmain.go
*.exe
*.test
*.prof
vendor/github.com/klauspost/crc32/LICENSE
Generated Vendored
+27
View File
@@ -0,0 +1,27 @@
Copyright (c) 2012 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
vendor/github.com/klauspost/crc32/README.md
Generated Vendored
+42
View File
@@ -0,0 +1,42 @@
# 2025 revival
For IEEE checksums AVX512 can be used to speed up CRC32 checksums by approximately 2x.
Castagnoli checksums (CRC32C) can also be computer with AVX512,
but the performance gain is not as significant enough for the downsides of using it at this point.
# crc32
This package is a drop-in replacement for the standard library `hash/crc32` package,
that features AVX 512 optimizations on x64 platforms, for a 2x speedup for IEEE CRC32 checksums.
# usage
Install using `go get github.com/klauspost/crc32`. This library is based on Go 1.24
Replace `import "hash/crc32"` with `import "github.com/klauspost/crc32"` and you are good to go.
# changes
* 2025: Revived and updated to Go 1.24, with AVX 512 optimizations.
# performance
AVX512 are enabled above 1KB input size. This rather high limit is due to AVX512 may be slower to ramp up than
the regular SSE4 implementation for smaller inputs. This is not reflected in the benchmarks below.
| Benchmark | Old MB/s | New MB/s | Speedup |
|-----------------------------------------------|----------|----------|---------|
| BenchmarkCRC32/poly=IEEE/size=512/align=0-32 | 17996.39 | 17969.94 | 1.00x |
| BenchmarkCRC32/poly=IEEE/size=512/align=1-32 | 18021.48 | 17945.55 | 1.00x |
| BenchmarkCRC32/poly=IEEE/size=1kB/align=0-32 | 19921.70 | 45613.77 | 2.29x |
| BenchmarkCRC32/poly=IEEE/size=1kB/align=1-32 | 19946.60 | 46819.09 | 2.35x |
| BenchmarkCRC32/poly=IEEE/size=4kB/align=0-32 | 21538.65 | 48600.93 | 2.26x |
| BenchmarkCRC32/poly=IEEE/size=4kB/align=1-32 | 21449.20 | 48477.84 | 2.26x |
| BenchmarkCRC32/poly=IEEE/size=32kB/align=0-32 | 21785.49 | 46013.10 | 2.11x |
| BenchmarkCRC32/poly=IEEE/size=32kB/align=1-32 | 21946.47 | 45954.10 | 2.09x |
cpu: AMD Ryzen 9 9950X 16-Core Processor
# license
Standard Go license. See [LICENSE](LICENSE) for details.
vendor/github.com/klauspost/crc32/crc32.go
Generated Vendored
+253
View File
@@ -0,0 +1,253 @@
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package crc32 implements the 32-bit cyclic redundancy check, or CRC-32,
// checksum. See https://en.wikipedia.org/wiki/Cyclic_redundancy_check for
// information.
//
// Polynomials are represented in LSB-first form also known as reversed representation.
//
// See https://en.wikipedia.org/wiki/Mathematics_of_cyclic_redundancy_checks#Reversed_representations_and_reciprocal_polynomials
// for information.
package crc32
import (
"encoding/binary"
"errors"
"hash"
"sync"
"sync/atomic"
)
// The size of a CRC-32 checksum in bytes.
const Size = 4
// Predefined polynomials.
const (
// IEEE is by far and away the most common CRC-32 polynomial.
// Used by ethernet (IEEE 802.3), v.42, fddi, gzip, zip, png, ...
IEEE = 0xedb88320
// Castagnoli's polynomial, used in iSCSI.
// Has better error detection characteristics than IEEE.
// https://dx.doi.org/10.1109/26.231911
Castagnoli = 0x82f63b78
// Koopman's polynomial.
// Also has better error detection characteristics than IEEE.
// https://dx.doi.org/10.1109/DSN.2002.1028931
Koopman = 0xeb31d82e
)
// Table is a 256-word table representing the polynomial for efficient processing.
type Table [256]uint32
// This file makes use of functions implemented in architecture-specific files.
// The interface that they implement is as follows:
//
// // archAvailableIEEE reports whether an architecture-specific CRC32-IEEE
// // algorithm is available.
// archAvailableIEEE() bool
//
// // archInitIEEE initializes the architecture-specific CRC3-IEEE algorithm.
// // It can only be called if archAvailableIEEE() returns true.
// archInitIEEE()
//
// // archUpdateIEEE updates the given CRC32-IEEE. It can only be called if
// // archInitIEEE() was previously called.
// archUpdateIEEE(crc uint32, p []byte) uint32
//
// // archAvailableCastagnoli reports whether an architecture-specific
// // CRC32-C algorithm is available.
// archAvailableCastagnoli() bool
//
// // archInitCastagnoli initializes the architecture-specific CRC32-C
// // algorithm. It can only be called if archAvailableCastagnoli() returns
// // true.
// archInitCastagnoli()
//
// // archUpdateCastagnoli updates the given CRC32-C. It can only be called
// // if archInitCastagnoli() was previously called.
// archUpdateCastagnoli(crc uint32, p []byte) uint32
// castagnoliTable points to a lazily initialized Table for the Castagnoli
// polynomial. MakeTable will always return this value when asked to make a
// Castagnoli table so we can compare against it to find when the caller is
// using this polynomial.
var castagnoliTable *Table
var castagnoliTable8 *slicing8Table
var updateCastagnoli func(crc uint32, p []byte) uint32
var haveCastagnoli atomic.Bool
var castagnoliInitOnce = sync.OnceFunc(func() {
castagnoliTable = simpleMakeTable(Castagnoli)
if archAvailableCastagnoli() {
archInitCastagnoli()
updateCastagnoli = archUpdateCastagnoli
} else {
// Initialize the slicing-by-8 table.
castagnoliTable8 = slicingMakeTable(Castagnoli)
updateCastagnoli = func(crc uint32, p []byte) uint32 {
return slicingUpdate(crc, castagnoliTable8, p)
}
}
haveCastagnoli.Store(true)
})
// IEEETable is the table for the [IEEE] polynomial.
var IEEETable = simpleMakeTable(IEEE)
// ieeeTable8 is the slicing8Table for IEEE
var ieeeTable8 *slicing8Table
var updateIEEE func(crc uint32, p []byte) uint32
var ieeeInitOnce = sync.OnceFunc(func() {
if archAvailableIEEE() {
archInitIEEE()
updateIEEE = archUpdateIEEE
} else {
// Initialize the slicing-by-8 table.
ieeeTable8 = slicingMakeTable(IEEE)
updateIEEE = func(crc uint32, p []byte) uint32 {
return slicingUpdate(crc, ieeeTable8, p)
}
}
})
// MakeTable returns a [Table] constructed from the specified polynomial.
// The contents of this [Table] must not be modified.
func MakeTable(poly uint32) *Table {
switch poly {
case IEEE:
ieeeInitOnce()
return IEEETable
case Castagnoli:
castagnoliInitOnce()
return castagnoliTable
default:
return simpleMakeTable(poly)
}
}
// digest represents the partial evaluation of a checksum.
type digest struct {
crc uint32
tab *Table
}
// New creates a new [hash.Hash32] computing the CRC-32 checksum using the
// polynomial represented by the [Table]. Its Sum method will lay the
// value out in big-endian byte order. The returned Hash32 also
// implements [encoding.BinaryMarshaler] and [encoding.BinaryUnmarshaler] to
// marshal and unmarshal the internal state of the hash.
func New(tab *Table) hash.Hash32 {
if tab == IEEETable {
ieeeInitOnce()
}
return &digest{0, tab}
}
// NewIEEE creates a new [hash.Hash32] computing the CRC-32 checksum using
// the [IEEE] polynomial. Its Sum method will lay the value out in
// big-endian byte order. The returned Hash32 also implements
// [encoding.BinaryMarshaler] and [encoding.BinaryUnmarshaler] to marshal
// and unmarshal the internal state of the hash.
func NewIEEE() hash.Hash32 { return New(IEEETable) }
func (d *digest) Size() int { return Size }
func (d *digest) BlockSize() int { return 1 }
func (d *digest) Reset() { d.crc = 0 }
const (
magic = "crc\x01"
marshaledSize = len(magic) + 4 + 4
)
func (d *digest) AppendBinary(b []byte) ([]byte, error) {
b = append(b, magic...)
b = binary.BigEndian.AppendUint32(b, tableSum(d.tab))
b = binary.BigEndian.AppendUint32(b, d.crc)
return b, nil
}
func (d *digest) MarshalBinary() ([]byte, error) {
return d.AppendBinary(make([]byte, 0, marshaledSize))
}
func (d *digest) UnmarshalBinary(b []byte) error {
if len(b) < len(magic) || string(b[:len(magic)]) != magic {
return errors.New("hash/crc32: invalid hash state identifier")
}
if len(b) != marshaledSize {
return errors.New("hash/crc32: invalid hash state size")
}
if tableSum(d.tab) != binary.BigEndian.Uint32(b[4:]) {
return errors.New("hash/crc32: tables do not match")
}
d.crc = binary.BigEndian.Uint32(b[8:])
return nil
}
func update(crc uint32, tab *Table, p []byte, checkInitIEEE bool) uint32 {
switch {
case haveCastagnoli.Load() && tab == castagnoliTable:
return updateCastagnoli(crc, p)
case tab == IEEETable:
if checkInitIEEE {
ieeeInitOnce()
}
return updateIEEE(crc, p)
default:
return simpleUpdate(crc, tab, p)
}
}
// Update returns the result of adding the bytes in p to the crc.
func Update(crc uint32, tab *Table, p []byte) uint32 {
// Unfortunately, because IEEETable is exported, IEEE may be used without a
// call to MakeTable. We have to make sure it gets initialized in that case.
return update(crc, tab, p, true)
}
func (d *digest) Write(p []byte) (n int, err error) {
// We only create digest objects through New() which takes care of
// initialization in this case.
d.crc = update(d.crc, d.tab, p, false)
return len(p), nil
}
func (d *digest) Sum32() uint32 { return d.crc }
func (d *digest) Sum(in []byte) []byte {
s := d.Sum32()
return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s))
}
// Checksum returns the CRC-32 checksum of data
// using the polynomial represented by the [Table].
func Checksum(data []byte, tab *Table) uint32 { return Update(0, tab, data) }
// ChecksumIEEE returns the CRC-32 checksum of data
// using the [IEEE] polynomial.
func ChecksumIEEE(data []byte) uint32 {
ieeeInitOnce()
return updateIEEE(0, data)
}
// tableSum returns the IEEE checksum of table t.
func tableSum(t *Table) uint32 {
var a [1024]byte
b := a[:0]
if t != nil {
for _, x := range t {
b = binary.BigEndian.AppendUint32(b, x)
}
}
return ChecksumIEEE(b)
}
vendor/github.com/klauspost/crc32/crc32_amd64.go
Generated Vendored
+253
View File
@@ -0,0 +1,253 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// AMD64-specific hardware-assisted CRC32 algorithms. See crc32.go for a
// description of the interface that each architecture-specific file
// implements.
package crc32
import (
"unsafe"
"golang.org/x/sys/cpu"
)
// This file contains the code to call the SSE 4.2 version of the Castagnoli
// and IEEE CRC.
// castagnoliSSE42 is defined in crc32_amd64.s and uses the SSE 4.2 CRC32
// instruction.
//
//go:noescape
func castagnoliSSE42(crc uint32, p []byte) uint32
// castagnoliSSE42Triple is defined in crc32_amd64.s and uses the SSE 4.2 CRC32
// instruction.
//
//go:noescape
func castagnoliSSE42Triple(
crcA, crcB, crcC uint32,
a, b, c []byte,
rounds uint32,
) (retA uint32, retB uint32, retC uint32)
// ieeeCLMUL is defined in crc_amd64.s and uses the PCLMULQDQ
// instruction as well as SSE 4.1.
//
//go:noescape
func ieeeCLMUL(crc uint32, p []byte) uint32
// castagnoliCLMULAvx512 is defined in crc_amd64.s and uses the PCLMULQDQ
// instruction as well as SSE 4.1.
//
//go:noescape
func castagnoliCLMULAvx512(crc uint32, p []byte) uint32
// ieeeCLMUL is defined in crc_amd64.s and uses the PCLMULQDQ
// instruction as well as SSE 4.1.
//
//go:noescape
func ieeeCLMULAvx512(crc uint32, p []byte) uint32
const castagnoliK1 = 168
const castagnoliK2 = 1344
type sse42Table [4]Table
var castagnoliSSE42TableK1 *sse42Table
var castagnoliSSE42TableK2 *sse42Table
func archAvailableCastagnoli() bool {
return cpu.X86.HasSSE42
}
func archInitCastagnoli() {
if !cpu.X86.HasSSE42 {
panic("arch-specific Castagnoli not available")
}
castagnoliSSE42TableK1 = new(sse42Table)
castagnoliSSE42TableK2 = new(sse42Table)
// See description in updateCastagnoli.
// t[0][i] = CRC(i000, O)
// t[1][i] = CRC(0i00, O)
// t[2][i] = CRC(00i0, O)
// t[3][i] = CRC(000i, O)
// where O is a sequence of K zeros.
var tmp [castagnoliK2]byte
for b := 0; b < 4; b++ {
for i := 0; i < 256; i++ {
val := uint32(i) << uint32(b*8)
castagnoliSSE42TableK1[b][i] = castagnoliSSE42(val, tmp[:castagnoliK1])
castagnoliSSE42TableK2[b][i] = castagnoliSSE42(val, tmp[:])
}
}
}
// castagnoliShift computes the CRC32-C of K1 or K2 zeroes (depending on the
// table given) with the given initial crc value. This corresponds to
// CRC(crc, O) in the description in updateCastagnoli.
func castagnoliShift(table *sse42Table, crc uint32) uint32 {
return table[3][crc>>24] ^
table[2][(crc>>16)&0xFF] ^
table[1][(crc>>8)&0xFF] ^
table[0][crc&0xFF]
}
func archUpdateCastagnoli(crc uint32, p []byte) uint32 {
if !cpu.X86.HasSSE42 {
panic("not available")
}
// This method is inspired from the algorithm in Intel's white paper:
// "Fast CRC Computation for iSCSI Polynomial Using CRC32 Instruction"
// The same strategy of splitting the buffer in three is used but the
// combining calculation is different; the complete derivation is explained
// below.
//
// -- The basic idea --
//
// The CRC32 instruction (available in SSE4.2) can process 8 bytes at a
// time. In recent Intel architectures the instruction takes 3 cycles;
// however the processor can pipeline up to three instructions if they
// don't depend on each other.
//
// Roughly this means that we can process three buffers in about the same
// time we can process one buffer.
//
// The idea is then to split the buffer in three, CRC the three pieces
// separately and then combine the results.
//
// Combining the results requires precomputed tables, so we must choose a
// fixed buffer length to optimize. The longer the length, the faster; but
// only buffers longer than this length will use the optimization. We choose
// two cutoffs and compute tables for both:
// - one around 512: 168*3=504
// - one around 4KB: 1344*3=4032
//
// -- The nitty gritty --
//
// Let CRC(I, X) be the non-inverted CRC32-C of the sequence X (with
// initial non-inverted CRC I). This function has the following properties:
// (a) CRC(I, AB) = CRC(CRC(I, A), B)
// (b) CRC(I, A xor B) = CRC(I, A) xor CRC(0, B)
//
// Say we want to compute CRC(I, ABC) where A, B, C are three sequences of
// K bytes each, where K is a fixed constant. Let O be the sequence of K zero
// bytes.
//
// CRC(I, ABC) = CRC(I, ABO xor C)
// = CRC(I, ABO) xor CRC(0, C)
// = CRC(CRC(I, AB), O) xor CRC(0, C)
// = CRC(CRC(I, AO xor B), O) xor CRC(0, C)
// = CRC(CRC(I, AO) xor CRC(0, B), O) xor CRC(0, C)
// = CRC(CRC(CRC(I, A), O) xor CRC(0, B), O) xor CRC(0, C)
//
// The castagnoliSSE42Triple function can compute CRC(I, A), CRC(0, B),
// and CRC(0, C) efficiently. We just need to find a way to quickly compute
// CRC(uvwx, O) given a 4-byte initial value uvwx. We can precompute these
// values; since we can't have a 32-bit table, we break it up into four
// 8-bit tables:
//
// CRC(uvwx, O) = CRC(u000, O) xor
// CRC(0v00, O) xor
// CRC(00w0, O) xor
// CRC(000x, O)
//
// We can compute tables corresponding to the four terms for all 8-bit
// values.
crc = ^crc
// Disabled, since it is not significantly faster than the SSE 4.2 version, even on Zen 5.
if false && len(p) >= 2048 && cpu.X86.HasAVX512F && cpu.X86.HasAVX512VL && cpu.X86.HasAVX512VPCLMULQDQ && cpu.X86.HasPCLMULQDQ {
left := len(p) & 15
do := len(p) - left
crc = castagnoliCLMULAvx512(crc, p[:do])
return ^castagnoliSSE42(crc, p[do:])
}
// If a buffer is long enough to use the optimization, process the first few
// bytes to align the buffer to an 8 byte boundary (if necessary).
if len(p) >= castagnoliK1*3 {
delta := int(uintptr(unsafe.Pointer(&p[0])) & 7)
if delta != 0 {
delta = 8 - delta
crc = castagnoliSSE42(crc, p[:delta])
p = p[delta:]
}
}
// Process 3*K2 at a time.
for len(p) >= castagnoliK2*3 {
// Compute CRC(I, A), CRC(0, B), and CRC(0, C).
crcA, crcB, crcC := castagnoliSSE42Triple(
crc, 0, 0,
p, p[castagnoliK2:], p[castagnoliK2*2:],
castagnoliK2/24)
// CRC(I, AB) = CRC(CRC(I, A), O) xor CRC(0, B)
crcAB := castagnoliShift(castagnoliSSE42TableK2, crcA) ^ crcB
// CRC(I, ABC) = CRC(CRC(I, AB), O) xor CRC(0, C)
crc = castagnoliShift(castagnoliSSE42TableK2, crcAB) ^ crcC
p = p[castagnoliK2*3:]
}
// Process 3*K1 at a time.
for len(p) >= castagnoliK1*3 {
// Compute CRC(I, A), CRC(0, B), and CRC(0, C).
crcA, crcB, crcC := castagnoliSSE42Triple(
crc, 0, 0,
p, p[castagnoliK1:], p[castagnoliK1*2:],
castagnoliK1/24)
// CRC(I, AB) = CRC(CRC(I, A), O) xor CRC(0, B)
crcAB := castagnoliShift(castagnoliSSE42TableK1, crcA) ^ crcB
// CRC(I, ABC) = CRC(CRC(I, AB), O) xor CRC(0, C)
crc = castagnoliShift(castagnoliSSE42TableK1, crcAB) ^ crcC
p = p[castagnoliK1*3:]
}
// Use the simple implementation for what's left.
crc = castagnoliSSE42(crc, p)
return ^crc
}
func archAvailableIEEE() bool {
return cpu.X86.HasPCLMULQDQ && cpu.X86.HasSSE41
}
var archIeeeTable8 *slicing8Table
func archInitIEEE() {
if !cpu.X86.HasPCLMULQDQ || !cpu.X86.HasSSE41 {
panic("not available")
}
// We still use slicing-by-8 for small buffers.
archIeeeTable8 = slicingMakeTable(IEEE)
}
func archUpdateIEEE(crc uint32, p []byte) uint32 {
if !cpu.X86.HasPCLMULQDQ || !cpu.X86.HasSSE41 {
panic("not available")
}
if len(p) >= 64 {
if len(p) >= 1024 && cpu.X86.HasAVX512F && cpu.X86.HasAVX512VL && cpu.X86.HasAVX512VPCLMULQDQ && cpu.X86.HasPCLMULQDQ {
left := len(p) & 15
do := len(p) - left
crc = ^ieeeCLMULAvx512(^crc, p[:do])
p = p[do:]
} else {
left := len(p) & 15
do := len(p) - left
crc = ^ieeeCLMUL(^crc, p[:do])
p = p[do:]
}
}
if len(p) == 0 {
return crc
}
return slicingUpdate(crc, archIeeeTable8, p)
}
vendor/github.com/klauspost/crc32/crc32_amd64.s
Generated Vendored
+527
View File
@@ -0,0 +1,527 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// castagnoliSSE42 updates the (non-inverted) crc with the given buffer.
//
// func castagnoliSSE42(crc uint32, p []byte) uint32
TEXT ·castagnoliSSE42(SB), NOSPLIT, $0
MOVL crc+0(FP), AX // CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
// If there are fewer than 8 bytes to process, skip alignment.
CMPQ CX, $8
JL less_than_8
MOVQ SI, BX
ANDQ $7, BX
JZ aligned
// Process the first few bytes to 8-byte align the input.
// BX = 8 - BX. We need to process this many bytes to align.
SUBQ $1, BX
XORQ $7, BX
BTQ $0, BX
JNC align_2
CRC32B (SI), AX
DECQ CX
INCQ SI
align_2:
BTQ $1, BX
JNC align_4
CRC32W (SI), AX
SUBQ $2, CX
ADDQ $2, SI
align_4:
BTQ $2, BX
JNC aligned
CRC32L (SI), AX
SUBQ $4, CX
ADDQ $4, SI
aligned:
// The input is now 8-byte aligned and we can process 8-byte chunks.
CMPQ CX, $8
JL less_than_8
CRC32Q (SI), AX
ADDQ $8, SI
SUBQ $8, CX
JMP aligned
less_than_8:
// We may have some bytes left over; process 4 bytes, then 2, then 1.
BTQ $2, CX
JNC less_than_4
CRC32L (SI), AX
ADDQ $4, SI
less_than_4:
BTQ $1, CX
JNC less_than_2
CRC32W (SI), AX
ADDQ $2, SI
less_than_2:
BTQ $0, CX
JNC done
CRC32B (SI), AX
done:
MOVL AX, ret+32(FP)
RET
// castagnoliSSE42Triple updates three (non-inverted) crcs with (24*rounds)
// bytes from each buffer.
//
// func castagnoliSSE42Triple(
// crc1, crc2, crc3 uint32,
// a, b, c []byte,
// rounds uint32,
// ) (retA uint32, retB uint32, retC uint32)
TEXT ·castagnoliSSE42Triple(SB), NOSPLIT, $0
MOVL crcA+0(FP), AX
MOVL crcB+4(FP), CX
MOVL crcC+8(FP), DX
MOVQ a+16(FP), R8 // data pointer
MOVQ b+40(FP), R9 // data pointer
MOVQ c+64(FP), R10 // data pointer
MOVL rounds+88(FP), R11
loop:
CRC32Q (R8), AX
CRC32Q (R9), CX
CRC32Q (R10), DX
CRC32Q 8(R8), AX
CRC32Q 8(R9), CX
CRC32Q 8(R10), DX
CRC32Q 16(R8), AX
CRC32Q 16(R9), CX
CRC32Q 16(R10), DX
ADDQ $24, R8
ADDQ $24, R9
ADDQ $24, R10
DECQ R11
JNZ loop
MOVL AX, retA+96(FP)
MOVL CX, retB+100(FP)
MOVL DX, retC+104(FP)
RET
// CRC32 polynomial data
//
// These constants are lifted from the
// Linux kernel, since they avoid the costly
// PSHUFB 16 byte reversal proposed in the
// original Intel paper.
DATA r2r1<>+0(SB)/8, $0x154442bd4
DATA r2r1<>+8(SB)/8, $0x1c6e41596
DATA r4r3<>+0(SB)/8, $0x1751997d0
DATA r4r3<>+8(SB)/8, $0x0ccaa009e
DATA rupoly<>+0(SB)/8, $0x1db710641
DATA rupoly<>+8(SB)/8, $0x1f7011641
DATA r5<>+0(SB)/8, $0x163cd6124
GLOBL r2r1<>(SB), RODATA, $16
GLOBL r4r3<>(SB), RODATA, $16
GLOBL rupoly<>(SB), RODATA, $16
GLOBL r5<>(SB), RODATA, $8
// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 64, and must be a multiple of 16.
// func ieeeCLMUL(crc uint32, p []byte) uint32
TEXT ·ieeeCLMUL(SB), NOSPLIT, $0
MOVL crc+0(FP), X0 // Initial CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
MOVOU (SI), X1
MOVOU 16(SI), X2
MOVOU 32(SI), X3
MOVOU 48(SI), X4
PXOR X0, X1
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left
JB remain64
MOVOA r2r1<>+0(SB), X0
loopback64:
MOVOA X1, X5
MOVOA X2, X6
MOVOA X3, X7
MOVOA X4, X8
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0, X0, X2
PCLMULQDQ $0, X0, X3
PCLMULQDQ $0, X0, X4
// Load next early
MOVOU (SI), X11
MOVOU 16(SI), X12
MOVOU 32(SI), X13
MOVOU 48(SI), X14
PCLMULQDQ $0x11, X0, X5
PCLMULQDQ $0x11, X0, X6
PCLMULQDQ $0x11, X0, X7
PCLMULQDQ $0x11, X0, X8
PXOR X5, X1
PXOR X6, X2
PXOR X7, X3
PXOR X8, X4
PXOR X11, X1
PXOR X12, X2
PXOR X13, X3
PXOR X14, X4
ADDQ $0x40, DI
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left?
JGE loopback64
// Fold result into a single register (X1)
remain64:
MOVOA r4r3<>+0(SB), X0
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X2, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X3, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X4, X1
// If there is less than 16 bytes left we are done
CMPQ CX, $16
JB finish
// Encode 16 bytes
remain16:
MOVOU (SI), X10
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X10, X1
SUBQ $16, CX
ADDQ $16, SI
CMPQ CX, $16
JGE remain16
finish:
// Fold final result into 32 bits and return it
PCMPEQB X3, X3
PCLMULQDQ $1, X1, X0
PSRLDQ $8, X1
PXOR X0, X1
MOVOA X1, X2
MOVQ r5<>+0(SB), X0
// Creates 32 bit mask. Note that we don't care about upper half.
PSRLQ $32, X3
PSRLDQ $4, X2
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
MOVOA rupoly<>+0(SB), X0
MOVOA X1, X2
PAND X3, X1
PCLMULQDQ $0x10, X0, X1
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
PEXTRD $1, X1, AX
MOVL AX, ret+32(FP)
RET
DATA r2r1X<>+0(SB)/8, $0x154442bd4
DATA r2r1X<>+8(SB)/8, $0x1c6e41596
DATA r2r1X<>+16(SB)/8, $0x154442bd4
DATA r2r1X<>+24(SB)/8, $0x1c6e41596
DATA r2r1X<>+32(SB)/8, $0x154442bd4
DATA r2r1X<>+40(SB)/8, $0x1c6e41596
DATA r2r1X<>+48(SB)/8, $0x154442bd4
DATA r2r1X<>+56(SB)/8, $0x1c6e41596
GLOBL r2r1X<>(SB), RODATA, $64
// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 128, and must be a multiple of 16.
// func ieeeCLMULAvx512(crc uint32, p []byte) uint32
TEXT ·ieeeCLMULAvx512(SB), NOSPLIT, $0
MOVL crc+0(FP), AX // Initial CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
VPXORQ Z0, Z0, Z0
VMOVDQU64 (SI), Z1
VMOVQ AX, X0
VPXORQ Z0, Z1, Z1 // Merge initial CRC value into Z1
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
VMOVDQU64 r2r1X<>+0(SB), Z0
loopback64:
// Load next early
VMOVDQU64 (SI), Z11
VPCLMULQDQ $0x11, Z0, Z1, Z5
VPCLMULQDQ $0, Z0, Z1, Z1
VPTERNLOGD $0x96, Z11, Z5, Z1 // Combine results with xor into Z1
ADDQ $0x40, DI
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left?
JGE loopback64
// Fold result into a single register (X1)
remain64:
VEXTRACTF32X4 $1, Z1, X2 // X2: Second 128-bit lane
VEXTRACTF32X4 $2, Z1, X3 // X3: Third 128-bit lane
VEXTRACTF32X4 $3, Z1, X4 // X4: Fourth 128-bit lane
MOVOA r4r3<>+0(SB), X0
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X2, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X3, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X4, X1
// If there is less than 16 bytes left we are done
CMPQ CX, $16
JB finish
// Encode 16 bytes
remain16:
MOVOU (SI), X10
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X10, X1
SUBQ $16, CX
ADDQ $16, SI
CMPQ CX, $16
JGE remain16
finish:
// Fold final result into 32 bits and return it
PCMPEQB X3, X3
PCLMULQDQ $1, X1, X0
PSRLDQ $8, X1
PXOR X0, X1
MOVOA X1, X2
MOVQ r5<>+0(SB), X0
// Creates 32 bit mask. Note that we don't care about upper half.
PSRLQ $32, X3
PSRLDQ $4, X2
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
MOVOA rupoly<>+0(SB), X0
MOVOA X1, X2
PAND X3, X1
PCLMULQDQ $0x10, X0, X1
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
PEXTRD $1, X1, AX
MOVL AX, ret+32(FP)
VZEROUPPER
RET
// Castagonli Polynomial constants
DATA r2r1C<>+0(SB)/8, $0x0740eef02
DATA r2r1C<>+8(SB)/8, $0x09e4addf8
DATA r2r1C<>+16(SB)/8, $0x0740eef02
DATA r2r1C<>+24(SB)/8, $0x09e4addf8
DATA r2r1C<>+32(SB)/8, $0x0740eef02
DATA r2r1C<>+40(SB)/8, $0x09e4addf8
DATA r2r1C<>+48(SB)/8, $0x0740eef02
DATA r2r1C<>+56(SB)/8, $0x09e4addf8
GLOBL r2r1C<>(SB), RODATA, $64
DATA r4r3C<>+0(SB)/8, $0xf20c0dfe
DATA r4r3C<>+8(SB)/8, $0x14cd00bd6
DATA rupolyC<>+0(SB)/8, $0x105ec76f0
DATA rupolyC<>+8(SB)/8, $0xdea713f1
DATA r5C<>+0(SB)/8, $0xdd45aab8
GLOBL r4r3C<>(SB), RODATA, $16
GLOBL rupolyC<>(SB), RODATA, $16
GLOBL r5C<>(SB), RODATA, $8
// Based on https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf
// len(p) must be at least 128, and must be a multiple of 16.
// func castagnoliCLMULAvx512(crc uint32, p []byte) uint32
TEXT ·castagnoliCLMULAvx512(SB), NOSPLIT, $0
MOVL crc+0(FP), AX // Initial CRC value
MOVQ p+8(FP), SI // data pointer
MOVQ p_len+16(FP), CX // len(p)
VPXORQ Z0, Z0, Z0
VMOVDQU64 (SI), Z1
VMOVQ AX, X0
VPXORQ Z0, Z1, Z1 // Merge initial CRC value into Z1
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
VMOVDQU64 r2r1C<>+0(SB), Z0
loopback64:
// Load next early
VMOVDQU64 (SI), Z11
VPCLMULQDQ $0x11, Z0, Z1, Z5
VPCLMULQDQ $0, Z0, Z1, Z1
VPTERNLOGD $0x96, Z11, Z5, Z1 // Combine results with xor into Z1
ADDQ $0x40, DI
ADDQ $64, SI // buf+=64
SUBQ $64, CX // len-=64
CMPQ CX, $64 // Less than 64 bytes left?
JGE loopback64
// Fold result into a single register (X1)
remain64:
VEXTRACTF32X4 $1, Z1, X2 // X2: Second 128-bit lane
VEXTRACTF32X4 $2, Z1, X3 // X3: Third 128-bit lane
VEXTRACTF32X4 $3, Z1, X4 // X4: Fourth 128-bit lane
MOVOA r4r3C<>+0(SB), X0
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X2, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X3, X1
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X4, X1
// If there is less than 16 bytes left we are done
CMPQ CX, $16
JB finish
// Encode 16 bytes
remain16:
MOVOU (SI), X10
MOVOA X1, X5
PCLMULQDQ $0, X0, X1
PCLMULQDQ $0x11, X0, X5
PXOR X5, X1
PXOR X10, X1
SUBQ $16, CX
ADDQ $16, SI
CMPQ CX, $16
JGE remain16
finish:
// Fold final result into 32 bits and return it
PCMPEQB X3, X3
PCLMULQDQ $1, X1, X0
PSRLDQ $8, X1
PXOR X0, X1
MOVOA X1, X2
MOVQ r5C<>+0(SB), X0
// Creates 32 bit mask. Note that we don't care about upper half.
PSRLQ $32, X3
PSRLDQ $4, X2
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
MOVOA rupolyC<>+0(SB), X0
MOVOA X1, X2
PAND X3, X1
PCLMULQDQ $0x10, X0, X1
PAND X3, X1
PCLMULQDQ $0, X0, X1
PXOR X2, X1
PEXTRD $1, X1, AX
MOVL AX, ret+32(FP)
VZEROUPPER
RET
vendor/github.com/klauspost/crc32/crc32_arm64.go
Generated Vendored
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// ARM64-specific hardware-assisted CRC32 algorithms. See crc32.go for a
// description of the interface that each architecture-specific file
// implements.
package crc32
import "golang.org/x/sys/cpu"
func castagnoliUpdate(crc uint32, p []byte) uint32
func ieeeUpdate(crc uint32, p []byte) uint32
func archAvailableCastagnoli() bool {
return cpu.ARM64.HasCRC32
}
func archInitCastagnoli() {
if !cpu.ARM64.HasCRC32 {
panic("arch-specific crc32 instruction for Castagnoli not available")
}
}
func archUpdateCastagnoli(crc uint32, p []byte) uint32 {
if !cpu.ARM64.HasCRC32 {
panic("arch-specific crc32 instruction for Castagnoli not available")
}
return ^castagnoliUpdate(^crc, p)
}
func archAvailableIEEE() bool {
return cpu.ARM64.HasCRC32
}
func archInitIEEE() {
if !cpu.ARM64.HasCRC32 {
panic("arch-specific crc32 instruction for IEEE not available")
}
}
func archUpdateIEEE(crc uint32, p []byte) uint32 {
if !cpu.ARM64.HasCRC32 {
panic("arch-specific crc32 instruction for IEEE not available")
}
return ^ieeeUpdate(^crc, p)
}
vendor/github.com/klauspost/crc32/crc32_arm64.s
Generated Vendored
+97
View File
@@ -0,0 +1,97 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// castagnoliUpdate updates the non-inverted crc with the given data.
// func castagnoliUpdate(crc uint32, p []byte) uint32
TEXT ·castagnoliUpdate(SB), NOSPLIT, $0-36
MOVWU crc+0(FP), R9 // CRC value
MOVD p+8(FP), R13 // data pointer
MOVD p_len+16(FP), R11 // len(p)
update:
CMP $16, R11
BLT less_than_16
LDP.P 16(R13), (R8, R10)
CRC32CX R8, R9
CRC32CX R10, R9
SUB $16, R11
JMP update
less_than_16:
TBZ $3, R11, less_than_8
MOVD.P 8(R13), R10
CRC32CX R10, R9
less_than_8:
TBZ $2, R11, less_than_4
MOVWU.P 4(R13), R10
CRC32CW R10, R9
less_than_4:
TBZ $1, R11, less_than_2
MOVHU.P 2(R13), R10
CRC32CH R10, R9
less_than_2:
TBZ $0, R11, done
MOVBU (R13), R10
CRC32CB R10, R9
done:
MOVWU R9, ret+32(FP)
RET
// ieeeUpdate updates the non-inverted crc with the given data.
// func ieeeUpdate(crc uint32, p []byte) uint32
TEXT ·ieeeUpdate(SB), NOSPLIT, $0-36
MOVWU crc+0(FP), R9 // CRC value
MOVD p+8(FP), R13 // data pointer
MOVD p_len+16(FP), R11 // len(p)
update:
CMP $16, R11
BLT less_than_16
LDP.P 16(R13), (R8, R10)
CRC32X R8, R9
CRC32X R10, R9
SUB $16, R11
JMP update
less_than_16:
TBZ $3, R11, less_than_8
MOVD.P 8(R13), R10
CRC32X R10, R9
less_than_8:
TBZ $2, R11, less_than_4
MOVWU.P 4(R13), R10
CRC32W R10, R9
less_than_4:
TBZ $1, R11, less_than_2
MOVHU.P 2(R13), R10
CRC32H R10, R9
less_than_2:
TBZ $0, R11, done
MOVBU (R13), R10
CRC32B R10, R9
done:
MOVWU R9, ret+32(FP)
RET
vendor/github.com/klauspost/crc32/crc32_generic.go
Generated Vendored
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// This file contains CRC32 algorithms that are not specific to any architecture
// and don't use hardware acceleration.
//
// The simple (and slow) CRC32 implementation only uses a 256*4 bytes table.
//
// The slicing-by-8 algorithm is a faster implementation that uses a bigger
// table (8*256*4 bytes).
package crc32
import "encoding/binary"
// simpleMakeTable allocates and constructs a Table for the specified
// polynomial. The table is suitable for use with the simple algorithm
// (simpleUpdate).
func simpleMakeTable(poly uint32) *Table {
t := new(Table)
simplePopulateTable(poly, t)
return t
}
// simplePopulateTable constructs a Table for the specified polynomial, suitable
// for use with simpleUpdate.
func simplePopulateTable(poly uint32, t *Table) {
for i := 0; i < 256; i++ {
crc := uint32(i)
for j := 0; j < 8; j++ {
if crc&1 == 1 {
crc = (crc >> 1) ^ poly
} else {
crc >>= 1
}
}
t[i] = crc
}
}
// simpleUpdate uses the simple algorithm to update the CRC, given a table that
// was previously computed using simpleMakeTable.
func simpleUpdate(crc uint32, tab *Table, p []byte) uint32 {
crc = ^crc
for _, v := range p {
crc = tab[byte(crc)^v] ^ (crc >> 8)
}
return ^crc
}
// Use slicing-by-8 when payload >= this value.
const slicing8Cutoff = 16
// slicing8Table is array of 8 Tables, used by the slicing-by-8 algorithm.
type slicing8Table [8]Table
// slicingMakeTable constructs a slicing8Table for the specified polynomial. The
// table is suitable for use with the slicing-by-8 algorithm (slicingUpdate).
func slicingMakeTable(poly uint32) *slicing8Table {
t := new(slicing8Table)
simplePopulateTable(poly, &t[0])
for i := 0; i < 256; i++ {
crc := t[0][i]
for j := 1; j < 8; j++ {
crc = t[0][crc&0xFF] ^ (crc >> 8)
t[j][i] = crc
}
}
return t
}
// slicingUpdate uses the slicing-by-8 algorithm to update the CRC, given a
// table that was previously computed using slicingMakeTable.
func slicingUpdate(crc uint32, tab *slicing8Table, p []byte) uint32 {
if len(p) >= slicing8Cutoff {
crc = ^crc
for len(p) > 8 {
crc ^= binary.LittleEndian.Uint32(p)
crc = tab[0][p[7]] ^ tab[1][p[6]] ^ tab[2][p[5]] ^ tab[3][p[4]] ^
tab[4][crc>>24] ^ tab[5][(crc>>16)&0xFF] ^
tab[6][(crc>>8)&0xFF] ^ tab[7][crc&0xFF]
p = p[8:]
}
crc = ^crc
}
if len(p) == 0 {
return crc
}
return simpleUpdate(crc, &tab[0], p)
}
vendor/github.com/klauspost/crc32/crc32_loong64.go
Generated Vendored
+50
View File
@@ -0,0 +1,50 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// LoongArch64-specific hardware-assisted CRC32 algorithms. See crc32.go for a
// description of the interface that each architecture-specific file
// implements.
package crc32
import "golang.org/x/sys/cpu"
func castagnoliUpdate(crc uint32, p []byte) uint32
func ieeeUpdate(crc uint32, p []byte) uint32
func archAvailableCastagnoli() bool {
return cpu.Loong64.HasCRC32
}
func archInitCastagnoli() {
if !cpu.Loong64.HasCRC32 {
panic("arch-specific crc32 instruction for Castagnoli not available")
}
}
func archUpdateCastagnoli(crc uint32, p []byte) uint32 {
if !cpu.Loong64.HasCRC32 {
panic("arch-specific crc32 instruction for Castagnoli not available")
}
return ^castagnoliUpdate(^crc, p)
}
func archAvailableIEEE() bool {
return cpu.Loong64.HasCRC32
}
func archInitIEEE() {
if !cpu.Loong64.HasCRC32 {
panic("arch-specific crc32 instruction for IEEE not available")
}
}
func archUpdateIEEE(crc uint32, p []byte) uint32 {
if !cpu.Loong64.HasCRC32 {
panic("arch-specific crc32 instruction for IEEE not available")
}
return ^ieeeUpdate(^crc, p)
}
vendor/github.com/klauspost/crc32/crc32_loong64.s
Generated Vendored
+160
View File
@@ -0,0 +1,160 @@
// Copyright 2024 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// castagnoliUpdate updates the non-inverted crc with the given data.
// func castagnoliUpdate(crc uint32, p []byte) uint32
TEXT ·castagnoliUpdate(SB), NOSPLIT, $0-36
MOVWU crc+0(FP), R4 // a0 = CRC value
MOVV p+8(FP), R5 // a1 = data pointer
MOVV p_len+16(FP), R6 // a2 = len(p)
SGT $8, R6, R12
BNE R12, less_than_8
AND $7, R5, R12
BEQ R12, aligned
// Process the first few bytes to 8-byte align the input.
// t0 = 8 - t0. We need to process this many bytes to align.
SUB $1, R12
XOR $7, R12
AND $1, R12, R13
BEQ R13, align_2
MOVB (R5), R13
CRCCWBW R4, R13, R4
ADDV $1, R5
ADDV $-1, R6
align_2:
AND $2, R12, R13
BEQ R13, align_4
MOVH (R5), R13
CRCCWHW R4, R13, R4
ADDV $2, R5
ADDV $-2, R6
align_4:
AND $4, R12, R13
BEQ R13, aligned
MOVW (R5), R13
CRCCWWW R4, R13, R4
ADDV $4, R5
ADDV $-4, R6
aligned:
// The input is now 8-byte aligned and we can process 8-byte chunks.
SGT $8, R6, R12
BNE R12, less_than_8
MOVV (R5), R13
CRCCWVW R4, R13, R4
ADDV $8, R5
ADDV $-8, R6
JMP aligned
less_than_8:
// We may have some bytes left over; process 4 bytes, then 2, then 1.
AND $4, R6, R12
BEQ R12, less_than_4
MOVW (R5), R13
CRCCWWW R4, R13, R4
ADDV $4, R5
ADDV $-4, R6
less_than_4:
AND $2, R6, R12
BEQ R12, less_than_2
MOVH (R5), R13
CRCCWHW R4, R13, R4
ADDV $2, R5
ADDV $-2, R6
less_than_2:
BEQ R6, done
MOVB (R5), R13
CRCCWBW R4, R13, R4
done:
MOVW R4, ret+32(FP)
RET
// ieeeUpdate updates the non-inverted crc with the given data.
// func ieeeUpdate(crc uint32, p []byte) uint32
TEXT ·ieeeUpdate(SB), NOSPLIT, $0-36
MOVWU crc+0(FP), R4 // a0 = CRC value
MOVV p+8(FP), R5 // a1 = data pointer
MOVV p_len+16(FP), R6 // a2 = len(p)
SGT $8, R6, R12
BNE R12, less_than_8
AND $7, R5, R12
BEQ R12, aligned
// Process the first few bytes to 8-byte align the input.
// t0 = 8 - t0. We need to process this many bytes to align.
SUB $1, R12
XOR $7, R12
AND $1, R12, R13
BEQ R13, align_2
MOVB (R5), R13
CRCWBW R4, R13, R4
ADDV $1, R5
ADDV $-1, R6
align_2:
AND $2, R12, R13
BEQ R13, align_4
MOVH (R5), R13
CRCWHW R4, R13, R4
ADDV $2, R5
ADDV $-2, R6
align_4:
AND $4, R12, R13
BEQ R13, aligned
MOVW (R5), R13
CRCWWW R4, R13, R4
ADDV $4, R5
ADDV $-4, R6
aligned:
// The input is now 8-byte aligned and we can process 8-byte chunks.
SGT $8, R6, R12
BNE R12, less_than_8
MOVV (R5), R13
CRCWVW R4, R13, R4
ADDV $8, R5
ADDV $-8, R6
JMP aligned
less_than_8:
// We may have some bytes left over; process 4 bytes, then 2, then 1.
AND $4, R6, R12
BEQ R12, less_than_4
MOVW (R5), R13
CRCWWW R4, R13, R4
ADDV $4, R5
ADDV $-4, R6
less_than_4:
AND $2, R6, R12
BEQ R12, less_than_2
MOVH (R5), R13
CRCWHW R4, R13, R4
ADDV $2, R5
ADDV $-2, R6
less_than_2:
BEQ R6, done
MOVB (R5), R13
CRCWBW R4, R13, R4
done:
MOVW R4, ret+32(FP)
RET
vendor/github.com/klauspost/crc32/crc32_otherarch.go
Generated Vendored
+15
View File
@@ -0,0 +1,15 @@
// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !amd64 && !s390x && !ppc64le && !arm64 && !loong64
package crc32
func archAvailableIEEE() bool { return false }
func archInitIEEE() { panic("not available") }
func archUpdateIEEE(crc uint32, p []byte) uint32 { panic("not available") }
func archAvailableCastagnoli() bool { return false }
func archInitCastagnoli() { panic("not available") }
func archUpdateCastagnoli(crc uint32, p []byte) uint32 { panic("not available") }
vendor/github.com/klauspost/crc32/crc32_ppc64le.go
Generated Vendored
+88
View File
@@ -0,0 +1,88 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package crc32
import (
"unsafe"
)
const (
vecMinLen = 16
vecAlignMask = 15 // align to 16 bytes
crcIEEE = 1
crcCast = 2
)
//go:noescape
func ppc64SlicingUpdateBy8(crc uint32, table8 *slicing8Table, p []byte) uint32
// this function requires the buffer to be 16 byte aligned and > 16 bytes long.
//
//go:noescape
func vectorCrc32(crc uint32, poly uint32, p []byte) uint32
var archCastagnoliTable8 *slicing8Table
func archInitCastagnoli() {
archCastagnoliTable8 = slicingMakeTable(Castagnoli)
}
func archUpdateCastagnoli(crc uint32, p []byte) uint32 {
if len(p) >= 4*vecMinLen {
// If not aligned then process the initial unaligned bytes
if uint64(uintptr(unsafe.Pointer(&p[0])))&uint64(vecAlignMask) != 0 {
align := uint64(uintptr(unsafe.Pointer(&p[0]))) & uint64(vecAlignMask)
newlen := vecMinLen - align
crc = ppc64SlicingUpdateBy8(crc, archCastagnoliTable8, p[:newlen])
p = p[newlen:]
}
// p should be aligned now
aligned := len(p) & ^vecAlignMask
crc = vectorCrc32(crc, crcCast, p[:aligned])
p = p[aligned:]
}
if len(p) == 0 {
return crc
}
return ppc64SlicingUpdateBy8(crc, archCastagnoliTable8, p)
}
func archAvailableIEEE() bool {
return true
}
func archAvailableCastagnoli() bool {
return true
}
var archIeeeTable8 *slicing8Table
func archInitIEEE() {
// We still use slicing-by-8 for small buffers.
archIeeeTable8 = slicingMakeTable(IEEE)
}
// archUpdateIEEE calculates the checksum of p using vectorizedIEEE.
func archUpdateIEEE(crc uint32, p []byte) uint32 {
// Check if vector code should be used. If not aligned, then handle those
// first up to the aligned bytes.
if len(p) >= 4*vecMinLen {
if uint64(uintptr(unsafe.Pointer(&p[0])))&uint64(vecAlignMask) != 0 {
align := uint64(uintptr(unsafe.Pointer(&p[0]))) & uint64(vecAlignMask)
newlen := vecMinLen - align
crc = ppc64SlicingUpdateBy8(crc, archIeeeTable8, p[:newlen])
p = p[newlen:]
}
aligned := len(p) & ^vecAlignMask
crc = vectorCrc32(crc, crcIEEE, p[:aligned])
p = p[aligned:]
}
if len(p) == 0 {
return crc
}
return ppc64SlicingUpdateBy8(crc, archIeeeTable8, p)
}
vendor/github.com/klauspost/crc32/crc32_ppc64le.s
Generated Vendored
+736
View File
@@ -0,0 +1,736 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// The vectorized implementation found below is a derived work
// from code written by Anton Blanchard <anton@au.ibm.com> found
// at https://github.com/antonblanchard/crc32-vpmsum. The original
// is dual licensed under GPL and Apache 2. As the copyright holder
// for the work, IBM has contributed this new work under
// the golang license.
// Changes include porting to Go assembler with modifications for
// the Go ABI for ppc64le.
#include "textflag.h"
#define POWER8_OFFSET 132
#define off16 R16
#define off32 R17
#define off48 R18
#define off64 R19
#define off80 R20
#define off96 R21
#define off112 R22
#define const1 V24
#define const2 V25
#define byteswap V26
#define mask_32bit V27
#define mask_64bit V28
#define zeroes V29
#define MAX_SIZE 32*1024
#define REFLECT
TEXT ·ppc64SlicingUpdateBy8(SB), NOSPLIT|NOFRAME, $0-44
MOVWZ crc+0(FP), R3 // incoming crc
MOVD table8+8(FP), R4 // *Table
MOVD p+16(FP), R5
MOVD p_len+24(FP), R6 // p len
CMP $0, R6 // len == 0?
BNE start
MOVW R3, ret+40(FP) // return crc
RET
start:
NOR R3, R3, R7 // ^crc
MOVWZ R7, R7 // 32 bits
CMP R6, $16
MOVD R6, CTR
BLT short
SRAD $3, R6, R8 // 8 byte chunks
MOVD R8, CTR
loop:
MOVWZ 0(R5), R8 // 0-3 bytes of p ?Endian?
MOVWZ 4(R5), R9 // 4-7 bytes of p
MOVD R4, R10 // &tab[0]
XOR R7, R8, R7 // crc ^= byte[0:3]
RLDICL $40, R9, $56, R17 // p[7]
SLD $2, R17, R17 // p[7]*4
RLDICL $40, R7, $56, R8 // crc>>24
SLD $2, R8, R8 // crc>>24*4
RLDICL $48, R9, $56, R18 // p[6]
SLD $2, R18, R18 // p[6]*4
MOVWZ (R10)(R17), R21 // tab[0][p[7]]
ADD $1024, R10, R10 // tab[1]
RLDICL $56, R9, $56, R19 // p[5]
SLD $2, R19, R19 // p[5]*4:1
MOVWZ (R10)(R18), R22 // tab[1][p[6]]
ADD $1024, R10, R10 // tab[2]
XOR R21, R22, R21 // xor done R22
CLRLSLDI $56, R9, $2, R20
MOVWZ (R10)(R19), R23 // tab[2][p[5]]
ADD $1024, R10, R10 // &tab[3]
XOR R21, R23, R21 // xor done R23
MOVWZ (R10)(R20), R24 // tab[3][p[4]]
ADD $1024, R10, R10 // &tab[4]
XOR R21, R24, R21 // xor done R24
MOVWZ (R10)(R8), R25 // tab[4][crc>>24]
RLDICL $48, R7, $56, R24 // crc>>16&0xFF
XOR R21, R25, R21 // xor done R25
ADD $1024, R10, R10 // &tab[5]
SLD $2, R24, R24 // crc>>16&0xFF*4
MOVWZ (R10)(R24), R26 // tab[5][crc>>16&0xFF]
XOR R21, R26, R21 // xor done R26
RLDICL $56, R7, $56, R25 // crc>>8
ADD $1024, R10, R10 // &tab[6]
SLD $2, R25, R25 // crc>>8&FF*2
MOVBZ R7, R26 // crc&0xFF
MOVWZ (R10)(R25), R27 // tab[6][crc>>8&0xFF]
ADD $1024, R10, R10 // &tab[7]
SLD $2, R26, R26 // crc&0xFF*2
XOR R21, R27, R21 // xor done R27
ADD $8, R5 // p = p[8:]
MOVWZ (R10)(R26), R28 // tab[7][crc&0xFF]
XOR R21, R28, R21 // xor done R28
MOVWZ R21, R7 // crc for next round
BDNZ loop
ANDCC $7, R6, R8 // any leftover bytes
BEQ done // none --> done
MOVD R8, CTR // byte count
PCALIGN $16 // align short loop
short:
MOVBZ 0(R5), R8 // get v
XOR R8, R7, R8 // byte(crc)^v -> R8
RLDIC $2, R8, $54, R8 // rldicl r8,r8,2,22
SRD $8, R7, R14 // crc>>8
MOVWZ (R4)(R8), R10
ADD $1, R5
XOR R10, R14, R7 // loop crc in R7
BDNZ short
done:
NOR R7, R7, R7 // ^crc
MOVW R7, ret+40(FP) // return crc
RET
#ifdef BYTESWAP_DATA
DATA ·byteswapcons+0(SB)/8, $0x0706050403020100
DATA ·byteswapcons+8(SB)/8, $0x0f0e0d0c0b0a0908
GLOBL ·byteswapcons+0(SB), RODATA, $16
#endif
TEXT ·vectorCrc32(SB), NOSPLIT|NOFRAME, $0-36
MOVWZ crc+0(FP), R3 // incoming crc
MOVWZ ctab+4(FP), R14 // crc poly id
MOVD p+8(FP), R4
MOVD p_len+16(FP), R5 // p len
// R3 = incoming crc
// R14 = constant table identifier
// R5 = address of bytes
// R6 = length of bytes
// defines for index loads
MOVD $16, off16
MOVD $32, off32
MOVD $48, off48
MOVD $64, off64
MOVD $80, off80
MOVD $96, off96
MOVD $112, off112
MOVD $0, R15
MOVD R3, R10 // save initial crc
NOR R3, R3, R3 // ^crc
MOVWZ R3, R3 // 32 bits
VXOR zeroes, zeroes, zeroes // clear the V reg
VSPLTISW $-1, V0
VSLDOI $4, V29, V0, mask_32bit
VSLDOI $8, V29, V0, mask_64bit
VXOR V8, V8, V8
MTVSRD R3, VS40 // crc initial value VS40 = V8
#ifdef REFLECT
VSLDOI $8, zeroes, V8, V8 // or: VSLDOI V29,V8,V27,4 for top 32 bits?
#else
VSLDOI $4, V8, zeroes, V8
#endif
#ifdef BYTESWAP_DATA
MOVD $·byteswapcons(SB), R3
LVX (R3), byteswap
#endif
CMPU R5, $256 // length of bytes
BLT short
RLDICR $0, R5, $56, R6 // chunk to process
// First step for larger sizes
l1:
MOVD $32768, R7
MOVD R7, R9
CMP R6, R7 // compare R6, R7 (MAX SIZE)
BGT top // less than MAX, just do remainder
MOVD R6, R7
top:
SUB R7, R6, R6
// mainloop does 128 bytes at a time
SRD $7, R7
// determine the offset into the constants table to start with.
// Each constant is 128 bytes, used against 16 bytes of data.
SLD $4, R7, R8
SRD $3, R9, R9
SUB R8, R9, R8
// The last iteration is reduced in a separate step
ADD $-1, R7
MOVD R7, CTR
// Determine which constant table (depends on poly)
CMP R14, $1
BNE castTable
MOVD $·IEEEConst(SB), R3
BR startConst
castTable:
MOVD $·CastConst(SB), R3
startConst:
ADD R3, R8, R3 // starting point in constants table
VXOR V0, V0, V0 // clear the V regs
VXOR V1, V1, V1
VXOR V2, V2, V2
VXOR V3, V3, V3
VXOR V4, V4, V4
VXOR V5, V5, V5
VXOR V6, V6, V6
VXOR V7, V7, V7
LVX (R3), const1 // loading constant values
CMP R15, $1 // Identify warm up pass
BEQ next
// First warm up pass: load the bytes to process
LVX (R4), V16
LVX (R4+off16), V17
LVX (R4+off32), V18
LVX (R4+off48), V19
LVX (R4+off64), V20
LVX (R4+off80), V21
LVX (R4+off96), V22
LVX (R4+off112), V23
ADD $128, R4 // bump up to next 128 bytes in buffer
VXOR V16, V8, V16 // xor in initial CRC in V8
next:
BC 18, 0, first_warm_up_done
ADD $16, R3 // bump up to next constants
LVX (R3), const2 // table values
VPMSUMD V16, const1, V8 // second warm up pass
LVX (R4), V16 // load from buffer
OR $0, R2, R2
VPMSUMD V17, const1, V9 // vpmsumd with constants
LVX (R4+off16), V17 // load next from buffer
OR $0, R2, R2
VPMSUMD V18, const1, V10 // vpmsumd with constants
LVX (R4+off32), V18 // load next from buffer
OR $0, R2, R2
VPMSUMD V19, const1, V11 // vpmsumd with constants
LVX (R4+off48), V19 // load next from buffer
OR $0, R2, R2
VPMSUMD V20, const1, V12 // vpmsumd with constants
LVX (R4+off64), V20 // load next from buffer
OR $0, R2, R2
VPMSUMD V21, const1, V13 // vpmsumd with constants
LVX (R4+off80), V21 // load next from buffer
OR $0, R2, R2
VPMSUMD V22, const1, V14 // vpmsumd with constants
LVX (R4+off96), V22 // load next from buffer
OR $0, R2, R2
VPMSUMD V23, const1, V15 // vpmsumd with constants
LVX (R4+off112), V23 // load next from buffer
ADD $128, R4 // bump up to next 128 bytes in buffer
BC 18, 0, first_cool_down
cool_top:
LVX (R3), const1 // constants
ADD $16, R3 // inc to next constants
OR $0, R2, R2
VXOR V0, V8, V0 // xor in previous vpmsumd
VPMSUMD V16, const2, V8 // vpmsumd with constants
LVX (R4), V16 // buffer
OR $0, R2, R2
VXOR V1, V9, V1 // xor in previous
VPMSUMD V17, const2, V9 // vpmsumd with constants
LVX (R4+off16), V17 // next in buffer
OR $0, R2, R2
VXOR V2, V10, V2 // xor in previous
VPMSUMD V18, const2, V10 // vpmsumd with constants
LVX (R4+off32), V18 // next in buffer
OR $0, R2, R2
VXOR V3, V11, V3 // xor in previous
VPMSUMD V19, const2, V11 // vpmsumd with constants
LVX (R4+off48), V19 // next in buffer
LVX (R3), const2 // get next constant
OR $0, R2, R2
VXOR V4, V12, V4 // xor in previous
VPMSUMD V20, const1, V12 // vpmsumd with constants
LVX (R4+off64), V20 // next in buffer
OR $0, R2, R2
VXOR V5, V13, V5 // xor in previous
VPMSUMD V21, const1, V13 // vpmsumd with constants
LVX (R4+off80), V21 // next in buffer
OR $0, R2, R2
VXOR V6, V14, V6 // xor in previous
VPMSUMD V22, const1, V14 // vpmsumd with constants
LVX (R4+off96), V22 // next in buffer
OR $0, R2, R2
VXOR V7, V15, V7 // xor in previous
VPMSUMD V23, const1, V15 // vpmsumd with constants
LVX (R4+off112), V23 // next in buffer
ADD $128, R4 // bump up buffer pointer
BDNZ cool_top // are we done?
first_cool_down:
// load the constants
// xor in the previous value
// vpmsumd the result with constants
LVX (R3), const1
ADD $16, R3
VXOR V0, V8, V0
VPMSUMD V16, const1, V8
OR $0, R2, R2
VXOR V1, V9, V1
VPMSUMD V17, const1, V9
OR $0, R2, R2
VXOR V2, V10, V2
VPMSUMD V18, const1, V10
OR $0, R2, R2
VXOR V3, V11, V3
VPMSUMD V19, const1, V11
OR $0, R2, R2
VXOR V4, V12, V4
VPMSUMD V20, const1, V12
OR $0, R2, R2
VXOR V5, V13, V5
VPMSUMD V21, const1, V13
OR $0, R2, R2
VXOR V6, V14, V6
VPMSUMD V22, const1, V14
OR $0, R2, R2
VXOR V7, V15, V7
VPMSUMD V23, const1, V15
OR $0, R2, R2
second_cool_down:
VXOR V0, V8, V0
VXOR V1, V9, V1
VXOR V2, V10, V2
VXOR V3, V11, V3
VXOR V4, V12, V4
VXOR V5, V13, V5
VXOR V6, V14, V6
VXOR V7, V15, V7
#ifdef REFLECT
VSLDOI $4, V0, zeroes, V0
VSLDOI $4, V1, zeroes, V1
VSLDOI $4, V2, zeroes, V2
VSLDOI $4, V3, zeroes, V3
VSLDOI $4, V4, zeroes, V4
VSLDOI $4, V5, zeroes, V5
VSLDOI $4, V6, zeroes, V6
VSLDOI $4, V7, zeroes, V7
#endif
LVX (R4), V8
LVX (R4+off16), V9
LVX (R4+off32), V10
LVX (R4+off48), V11
LVX (R4+off64), V12
LVX (R4+off80), V13
LVX (R4+off96), V14
LVX (R4+off112), V15
ADD $128, R4
VXOR V0, V8, V16
VXOR V1, V9, V17
VXOR V2, V10, V18
VXOR V3, V11, V19
VXOR V4, V12, V20
VXOR V5, V13, V21
VXOR V6, V14, V22
VXOR V7, V15, V23
MOVD $1, R15
CMP $0, R6
ADD $128, R6
BNE l1
ANDCC $127, R5
SUBC R5, $128, R6
ADD R3, R6, R3
SRD $4, R5, R7
MOVD R7, CTR
LVX (R3), V0
LVX (R3+off16), V1
LVX (R3+off32), V2
LVX (R3+off48), V3
LVX (R3+off64), V4
LVX (R3+off80), V5
LVX (R3+off96), V6
LVX (R3+off112), V7
ADD $128, R3
VPMSUMW V16, V0, V0
VPMSUMW V17, V1, V1
VPMSUMW V18, V2, V2
VPMSUMW V19, V3, V3
VPMSUMW V20, V4, V4
VPMSUMW V21, V5, V5
VPMSUMW V22, V6, V6
VPMSUMW V23, V7, V7
// now reduce the tail
CMP $0, R7
BEQ next1
LVX (R4), V16
LVX (R3), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off16), V16
LVX (R3+off16), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off32), V16
LVX (R3+off32), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off48), V16
LVX (R3+off48), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off64), V16
LVX (R3+off64), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off80), V16
LVX (R3+off80), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
BC 18, 0, next1
LVX (R4+off96), V16
LVX (R3+off96), V17
VPMSUMW V16, V17, V16
VXOR V0, V16, V0
next1:
VXOR V0, V1, V0
VXOR V2, V3, V2
VXOR V4, V5, V4
VXOR V6, V7, V6
VXOR V0, V2, V0
VXOR V4, V6, V4
VXOR V0, V4, V0
barrett_reduction:
CMP R14, $1
BNE barcstTable
MOVD $·IEEEBarConst(SB), R3
BR startbarConst
barcstTable:
MOVD $·CastBarConst(SB), R3
startbarConst:
LVX (R3), const1
LVX (R3+off16), const2
VSLDOI $8, V0, V0, V1
VXOR V0, V1, V0
#ifdef REFLECT
VSPLTISB $1, V1
VSL V0, V1, V0
#endif
VAND V0, mask_64bit, V0
#ifndef REFLECT
VPMSUMD V0, const1, V1
VSLDOI $8, zeroes, V1, V1
VPMSUMD V1, const2, V1
VXOR V0, V1, V0
VSLDOI $8, V0, zeroes, V0
#else
VAND V0, mask_32bit, V1
VPMSUMD V1, const1, V1
VAND V1, mask_32bit, V1
VPMSUMD V1, const2, V1
VXOR V0, V1, V0
VSLDOI $4, V0, zeroes, V0
#endif
MFVSRD VS32, R3 // VS32 = V0
NOR R3, R3, R3 // return ^crc
MOVW R3, ret+32(FP)
RET
first_warm_up_done:
LVX (R3), const1
ADD $16, R3
VPMSUMD V16, const1, V8
VPMSUMD V17, const1, V9
VPMSUMD V18, const1, V10
VPMSUMD V19, const1, V11
VPMSUMD V20, const1, V12
VPMSUMD V21, const1, V13
VPMSUMD V22, const1, V14
VPMSUMD V23, const1, V15
BR second_cool_down
short:
CMP $0, R5
BEQ zero
// compute short constants
CMP R14, $1
BNE castshTable
MOVD $·IEEEConst(SB), R3
ADD $4080, R3
BR startshConst
castshTable:
MOVD $·CastConst(SB), R3
ADD $4080, R3
startshConst:
SUBC R5, $256, R6 // sub from 256
ADD R3, R6, R3
// calculate where to start
SRD $4, R5, R7
MOVD R7, CTR
VXOR V19, V19, V19
VXOR V20, V20, V20
LVX (R4), V0
LVX (R3), V16
VXOR V0, V8, V0
VPMSUMW V0, V16, V0
BC 18, 0, v0
LVX (R4+off16), V1
LVX (R3+off16), V17
VPMSUMW V1, V17, V1
BC 18, 0, v1
LVX (R4+off32), V2
LVX (R3+off32), V16
VPMSUMW V2, V16, V2
BC 18, 0, v2
LVX (R4+off48), V3
LVX (R3+off48), V17
VPMSUMW V3, V17, V3
BC 18, 0, v3
LVX (R4+off64), V4
LVX (R3+off64), V16
VPMSUMW V4, V16, V4
BC 18, 0, v4
LVX (R4+off80), V5
LVX (R3+off80), V17
VPMSUMW V5, V17, V5
BC 18, 0, v5
LVX (R4+off96), V6
LVX (R3+off96), V16
VPMSUMW V6, V16, V6
BC 18, 0, v6
LVX (R4+off112), V7
LVX (R3+off112), V17
VPMSUMW V7, V17, V7
BC 18, 0, v7
ADD $128, R3
ADD $128, R4
LVX (R4), V8
LVX (R3), V16
VPMSUMW V8, V16, V8
BC 18, 0, v8
LVX (R4+off16), V9
LVX (R3+off16), V17
VPMSUMW V9, V17, V9
BC 18, 0, v9
LVX (R4+off32), V10
LVX (R3+off32), V16
VPMSUMW V10, V16, V10
BC 18, 0, v10
LVX (R4+off48), V11
LVX (R3+off48), V17
VPMSUMW V11, V17, V11
BC 18, 0, v11
LVX (R4+off64), V12
LVX (R3+off64), V16
VPMSUMW V12, V16, V12
BC 18, 0, v12
LVX (R4+off80), V13
LVX (R3+off80), V17
VPMSUMW V13, V17, V13
BC 18, 0, v13
LVX (R4+off96), V14
LVX (R3+off96), V16
VPMSUMW V14, V16, V14
BC 18, 0, v14
LVX (R4+off112), V15
LVX (R3+off112), V17
VPMSUMW V15, V17, V15
VXOR V19, V15, V19
v14:
VXOR V20, V14, V20
v13:
VXOR V19, V13, V19
v12:
VXOR V20, V12, V20
v11:
VXOR V19, V11, V19
v10:
VXOR V20, V10, V20
v9:
VXOR V19, V9, V19
v8:
VXOR V20, V8, V20
v7:
VXOR V19, V7, V19
v6:
VXOR V20, V6, V20
v5:
VXOR V19, V5, V19
v4:
VXOR V20, V4, V20
v3:
VXOR V19, V3, V19
v2:
VXOR V20, V2, V20
v1:
VXOR V19, V1, V19
v0:
VXOR V20, V0, V20
VXOR V19, V20, V0
BR barrett_reduction
zero:
// This case is the original crc, so just return it
MOVW R10, ret+32(FP)
RET
vendor/github.com/klauspost/crc32/crc32_s390x.go
Generated Vendored
+91
View File
@@ -0,0 +1,91 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package crc32
import "golang.org/x/sys/cpu"
const (
vxMinLen = 64
vxAlignMask = 15 // align to 16 bytes
)
// hasVX reports whether the machine has the z/Architecture
// vector facility installed and enabled.
var hasVX = cpu.S390X.HasVX
// vectorizedCastagnoli implements CRC32 using vector instructions.
// It is defined in crc32_s390x.s.
//
//go:noescape
func vectorizedCastagnoli(crc uint32, p []byte) uint32
// vectorizedIEEE implements CRC32 using vector instructions.
// It is defined in crc32_s390x.s.
//
//go:noescape
func vectorizedIEEE(crc uint32, p []byte) uint32
func archAvailableCastagnoli() bool {
return hasVX
}
var archCastagnoliTable8 *slicing8Table
func archInitCastagnoli() {
if !hasVX {
panic("not available")
}
// We still use slicing-by-8 for small buffers.
archCastagnoliTable8 = slicingMakeTable(Castagnoli)
}
// archUpdateCastagnoli calculates the checksum of p using
// vectorizedCastagnoli.
func archUpdateCastagnoli(crc uint32, p []byte) uint32 {
if !hasVX {
panic("not available")
}
// Use vectorized function if data length is above threshold.
if len(p) >= vxMinLen {
aligned := len(p) & ^vxAlignMask
crc = vectorizedCastagnoli(crc, p[:aligned])
p = p[aligned:]
}
if len(p) == 0 {
return crc
}
return slicingUpdate(crc, archCastagnoliTable8, p)
}
func archAvailableIEEE() bool {
return hasVX
}
var archIeeeTable8 *slicing8Table
func archInitIEEE() {
if !hasVX {
panic("not available")
}
// We still use slicing-by-8 for small buffers.
archIeeeTable8 = slicingMakeTable(IEEE)
}
// archUpdateIEEE calculates the checksum of p using vectorizedIEEE.
func archUpdateIEEE(crc uint32, p []byte) uint32 {
if !hasVX {
panic("not available")
}
// Use vectorized function if data length is above threshold.
if len(p) >= vxMinLen {
aligned := len(p) & ^vxAlignMask
crc = vectorizedIEEE(crc, p[:aligned])
p = p[aligned:]
}
if len(p) == 0 {
return crc
}
return slicingUpdate(crc, archIeeeTable8, p)
}
vendor/github.com/klauspost/crc32/crc32_s390x.s
Generated Vendored
+225
View File
@@ -0,0 +1,225 @@
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
#include "textflag.h"
// Vector register range containing CRC-32 constants
#define CONST_PERM_LE2BE V9
#define CONST_R2R1 V10
#define CONST_R4R3 V11
#define CONST_R5 V12
#define CONST_RU_POLY V13
#define CONST_CRC_POLY V14
// The CRC-32 constant block contains reduction constants to fold and
// process particular chunks of the input data stream in parallel.
//
// Note that the constant definitions below are extended in order to compute
// intermediate results with a single VECTOR GALOIS FIELD MULTIPLY instruction.
// The rightmost doubleword can be 0 to prevent contribution to the result or
// can be multiplied by 1 to perform an XOR without the need for a separate
// VECTOR EXCLUSIVE OR instruction.
//
// The polynomials used are bit-reflected:
//
// IEEE: P'(x) = 0x0edb88320
// Castagnoli: P'(x) = 0x082f63b78
// IEEE polynomial constants
DATA ·crclecons+0(SB)/8, $0x0F0E0D0C0B0A0908 // LE-to-BE mask
DATA ·crclecons+8(SB)/8, $0x0706050403020100
DATA ·crclecons+16(SB)/8, $0x00000001c6e41596 // R2
DATA ·crclecons+24(SB)/8, $0x0000000154442bd4 // R1
DATA ·crclecons+32(SB)/8, $0x00000000ccaa009e // R4
DATA ·crclecons+40(SB)/8, $0x00000001751997d0 // R3
DATA ·crclecons+48(SB)/8, $0x0000000000000000
DATA ·crclecons+56(SB)/8, $0x0000000163cd6124 // R5
DATA ·crclecons+64(SB)/8, $0x0000000000000000
DATA ·crclecons+72(SB)/8, $0x00000001F7011641 // u'
DATA ·crclecons+80(SB)/8, $0x0000000000000000
DATA ·crclecons+88(SB)/8, $0x00000001DB710641 // P'(x) << 1
GLOBL ·crclecons(SB), RODATA, $144
// Castagonli Polynomial constants
DATA ·crcclecons+0(SB)/8, $0x0F0E0D0C0B0A0908 // LE-to-BE mask
DATA ·crcclecons+8(SB)/8, $0x0706050403020100
DATA ·crcclecons+16(SB)/8, $0x000000009e4addf8 // R2
DATA ·crcclecons+24(SB)/8, $0x00000000740eef02 // R1
DATA ·crcclecons+32(SB)/8, $0x000000014cd00bd6 // R4
DATA ·crcclecons+40(SB)/8, $0x00000000f20c0dfe // R3
DATA ·crcclecons+48(SB)/8, $0x0000000000000000
DATA ·crcclecons+56(SB)/8, $0x00000000dd45aab8 // R5
DATA ·crcclecons+64(SB)/8, $0x0000000000000000
DATA ·crcclecons+72(SB)/8, $0x00000000dea713f1 // u'
DATA ·crcclecons+80(SB)/8, $0x0000000000000000
DATA ·crcclecons+88(SB)/8, $0x0000000105ec76f0 // P'(x) << 1
GLOBL ·crcclecons(SB), RODATA, $144
// The CRC-32 function(s) use these calling conventions:
//
// Parameters:
//
// R2: Initial CRC value, typically ~0; and final CRC (return) value.
// R3: Input buffer pointer, performance might be improved if the
// buffer is on a doubleword boundary.
// R4: Length of the buffer, must be 64 bytes or greater.
//
// Register usage:
//
// R5: CRC-32 constant pool base pointer.
// V0: Initial CRC value and intermediate constants and results.
// V1..V4: Data for CRC computation.
// V5..V8: Next data chunks that are fetched from the input buffer.
//
// V9..V14: CRC-32 constants.
// func vectorizedIEEE(crc uint32, p []byte) uint32
TEXT ·vectorizedIEEE(SB), NOSPLIT, $0
MOVWZ crc+0(FP), R2 // R2 stores the CRC value
MOVD p+8(FP), R3 // data pointer
MOVD p_len+16(FP), R4 // len(p)
MOVD $·crclecons(SB), R5
BR vectorizedBody<>(SB)
// func vectorizedCastagnoli(crc uint32, p []byte) uint32
TEXT ·vectorizedCastagnoli(SB), NOSPLIT, $0
MOVWZ crc+0(FP), R2 // R2 stores the CRC value
MOVD p+8(FP), R3 // data pointer
MOVD p_len+16(FP), R4 // len(p)
// R5: crc-32 constant pool base pointer, constant is used to reduce crc
MOVD $·crcclecons(SB), R5
BR vectorizedBody<>(SB)
TEXT vectorizedBody<>(SB), NOSPLIT, $0
XOR $0xffffffff, R2 // NOTW R2
VLM 0(R5), CONST_PERM_LE2BE, CONST_CRC_POLY
// Load the initial CRC value into the rightmost word of V0
VZERO V0
VLVGF $3, R2, V0
// Crash if the input size is less than 64-bytes.
CMP R4, $64
BLT crash
// Load a 64-byte data chunk and XOR with CRC
VLM 0(R3), V1, V4 // 64-bytes into V1..V4
// Reflect the data if the CRC operation is in the bit-reflected domain
VPERM V1, V1, CONST_PERM_LE2BE, V1
VPERM V2, V2, CONST_PERM_LE2BE, V2
VPERM V3, V3, CONST_PERM_LE2BE, V3
VPERM V4, V4, CONST_PERM_LE2BE, V4
VX V0, V1, V1 // V1 ^= CRC
ADD $64, R3 // BUF = BUF + 64
ADD $(-64), R4
// Check remaining buffer size and jump to proper folding method
CMP R4, $64
BLT less_than_64bytes
fold_64bytes_loop:
// Load the next 64-byte data chunk into V5 to V8
VLM 0(R3), V5, V8
VPERM V5, V5, CONST_PERM_LE2BE, V5
VPERM V6, V6, CONST_PERM_LE2BE, V6
VPERM V7, V7, CONST_PERM_LE2BE, V7
VPERM V8, V8, CONST_PERM_LE2BE, V8
// Perform a GF(2) multiplication of the doublewords in V1 with
// the reduction constants in V0. The intermediate result is
// then folded (accumulated) with the next data chunk in V5 and
// stored in V1. Repeat this step for the register contents
// in V2, V3, and V4 respectively.
VGFMAG CONST_R2R1, V1, V5, V1
VGFMAG CONST_R2R1, V2, V6, V2
VGFMAG CONST_R2R1, V3, V7, V3
VGFMAG CONST_R2R1, V4, V8, V4
// Adjust buffer pointer and length for next loop
ADD $64, R3 // BUF = BUF + 64
ADD $(-64), R4 // LEN = LEN - 64
CMP R4, $64
BGE fold_64bytes_loop
less_than_64bytes:
// Fold V1 to V4 into a single 128-bit value in V1
VGFMAG CONST_R4R3, V1, V2, V1
VGFMAG CONST_R4R3, V1, V3, V1
VGFMAG CONST_R4R3, V1, V4, V1
// Check whether to continue with 64-bit folding
CMP R4, $16
BLT final_fold
fold_16bytes_loop:
VL 0(R3), V2 // Load next data chunk
VPERM V2, V2, CONST_PERM_LE2BE, V2
VGFMAG CONST_R4R3, V1, V2, V1 // Fold next data chunk
// Adjust buffer pointer and size for folding next data chunk
ADD $16, R3
ADD $-16, R4
// Process remaining data chunks
CMP R4, $16
BGE fold_16bytes_loop
final_fold:
VLEIB $7, $0x40, V9
VSRLB V9, CONST_R4R3, V0
VLEIG $0, $1, V0
VGFMG V0, V1, V1
VLEIB $7, $0x20, V9 // Shift by words
VSRLB V9, V1, V2 // Store remaining bits in V2
VUPLLF V1, V1 // Split rightmost doubleword
VGFMAG CONST_R5, V1, V2, V1 // V1 = (V1 * R5) XOR V2
// The input values to the Barret reduction are the degree-63 polynomial
// in V1 (R(x)), degree-32 generator polynomial, and the reduction
// constant u. The Barret reduction result is the CRC value of R(x) mod
// P(x).
//
// The Barret reduction algorithm is defined as:
//
// 1. T1(x) = floor( R(x) / x^32 ) GF2MUL u
// 2. T2(x) = floor( T1(x) / x^32 ) GF2MUL P(x)
// 3. C(x) = R(x) XOR T2(x) mod x^32
//
// Note: To compensate the division by x^32, use the vector unpack
// instruction to move the leftmost word into the leftmost doubleword
// of the vector register. The rightmost doubleword is multiplied
// with zero to not contribute to the intermediate results.
// T1(x) = floor( R(x) / x^32 ) GF2MUL u
VUPLLF V1, V2
VGFMG CONST_RU_POLY, V2, V2
// Compute the GF(2) product of the CRC polynomial in VO with T1(x) in
// V2 and XOR the intermediate result, T2(x), with the value in V1.
// The final result is in the rightmost word of V2.
VUPLLF V2, V2
VGFMAG CONST_CRC_POLY, V2, V1, V2
done:
VLGVF $2, V2, R2
XOR $0xffffffff, R2 // NOTW R2
MOVWZ R2, ret + 32(FP)
RET
crash:
MOVD $0, (R0) // input size is less than 64-bytes
vendor/github.com/klauspost/crc32/crc32_table_ppc64le.s
Generated Vendored
+3285
View File
File diff suppressed because it is too large Load Diff
vendor/github.com/klauspost/crc32/gen.go
Generated Vendored
+7
View File
@@ -0,0 +1,7 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:generate go run gen_const_ppc64le.go
package crc32
vendor/github.com/minio/crc64nvme/crc64.go
Generated Vendored
+7 -2
View File
@@ -125,14 +125,19 @@ func update(crc uint64, p []byte) uint64 {
p = p[align:]
}
runs := len(p) / 128
crc = updateAsm(crc, p[:128*runs])
if hasAsm512 && runs >= 8 {
// Use 512-bit wide instructions for >= 1KB.
crc = updateAsm512(crc, p[:128*runs])
} else {
crc = updateAsm(crc, p[:128*runs])
}
return update(crc, p[128*runs:])
}
buildSlicing8TablesOnce()
crc = ^crc
// table comparison is somewhat expensive, so avoid it for small sizes
for len(p) >= 64 {
if len(p) >= 64 {
var helperTable = slicing8TableNVME
// Update using slicing-by-8
for len(p) > 8 {
vendor/github.com/minio/crc64nvme/crc64_amd64.go
Generated Vendored
+2
View File
@@ -11,5 +11,7 @@ import (
)
var hasAsm = cpuid.CPU.Supports(cpuid.SSE2, cpuid.CLMUL, cpuid.SSE4)
var hasAsm512 = cpuid.CPU.Supports(cpuid.AVX512F, cpuid.VPCLMULQDQ, cpuid.AVX512VL, cpuid.CLMUL)
func updateAsm(crc uint64, p []byte) (checksum uint64)
func updateAsm512(crc uint64, p []byte) (checksum uint64)
vendor/github.com/minio/crc64nvme/crc64_amd64.s
Generated Vendored
+150
View File
@@ -155,3 +155,153 @@ skip128:
NOTQ AX
MOVQ AX, checksum+32(FP)
RET
// Constants, pre-splatted.
DATA ·asmConstantsPoly<>+0x00(SB)/8, $0xa1ca681e733f9c40
DATA ·asmConstantsPoly<>+0x08(SB)/8, $0
DATA ·asmConstantsPoly<>+0x10(SB)/8, $0xa1ca681e733f9c40
DATA ·asmConstantsPoly<>+0x18(SB)/8, $0
DATA ·asmConstantsPoly<>+0x20(SB)/8, $0xa1ca681e733f9c40
DATA ·asmConstantsPoly<>+0x28(SB)/8, $0
DATA ·asmConstantsPoly<>+0x30(SB)/8, $0xa1ca681e733f9c40
DATA ·asmConstantsPoly<>+0x38(SB)/8, $0
// Upper
DATA ·asmConstantsPoly<>+0x40(SB)/8, $0
DATA ·asmConstantsPoly<>+0x48(SB)/8, $0x5f852fb61e8d92dc
DATA ·asmConstantsPoly<>+0x50(SB)/8, $0
DATA ·asmConstantsPoly<>+0x58(SB)/8, $0x5f852fb61e8d92dc
DATA ·asmConstantsPoly<>+0x60(SB)/8, $0
DATA ·asmConstantsPoly<>+0x68(SB)/8, $0x5f852fb61e8d92dc
DATA ·asmConstantsPoly<>+0x70(SB)/8, $0
DATA ·asmConstantsPoly<>+0x78(SB)/8, $0x5f852fb61e8d92dc
GLOBL ·asmConstantsPoly<>(SB), (NOPTR+RODATA), $128
TEXT ·updateAsm512(SB), $0-40
MOVQ crc+0(FP), AX // checksum
MOVQ p_base+8(FP), SI // start pointer
MOVQ p_len+16(FP), CX // length of buffer
NOTQ AX
SHRQ $7, CX
CMPQ CX, $1
VPXORQ Z8, Z8, Z8 // Initialize ZMM8 to zero
JLT skip128
VMOVDQU64 0x00(SI), Z0
VMOVDQU64 0x40(SI), Z4
MOVQ $·asmConstantsPoly<>(SB), BX
VMOVQ AX, X8
// XOR initialization value into lower 64 bits of ZMM0
VPXORQ Z8, Z0, Z0
CMPQ CX, $1
JE tail128
VMOVDQU64 0(BX), Z8
VMOVDQU64 64(BX), Z9
PCALIGN $16
loop128:
VMOVDQU64 0x80(SI), Z1
VMOVDQU64 0xc0(SI), Z5
ADDQ $128, SI
SUBQ $1, CX
VPCLMULQDQ $0x00, Z8, Z0, Z10
VPCLMULQDQ $0x11, Z9, Z0, Z0
VPTERNLOGD $0x96, Z1, Z10, Z0 // Combine results with xor into Z0
VPCLMULQDQ $0x00, Z8, Z4, Z10
VPCLMULQDQ $0x11, Z9, Z4, Z4
VPTERNLOGD $0x96, Z5, Z10, Z4 // Combine results with xor into Z4
CMPQ CX, $1
JGT loop128
tail128:
// Extract X0 to X3 from ZMM0
VEXTRACTF32X4 $1, Z0, X1 // X1: Second 128-bit lane
VEXTRACTF32X4 $2, Z0, X2 // X2: Third 128-bit lane
VEXTRACTF32X4 $3, Z0, X3 // X3: Fourth 128-bit lane
// Extract X4 to X7 from ZMM4
VEXTRACTF32X4 $1, Z4, X5 // X5: Second 128-bit lane
VEXTRACTF32X4 $2, Z4, X6 // X6: Third 128-bit lane
VEXTRACTF32X4 $3, Z4, X7 // X7: Fourth 128-bit lane
MOVQ $0xd083dd594d96319d, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X0, X11
MOVQ $0x946588403d4adcbc, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X0
PXOR X11, X7
PXOR X0, X7
MOVQ $0x3c255f5ebc414423, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X1, X11
MOVQ $0x34f5a24e22d66e90, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X1
PXOR X11, X1
PXOR X7, X1
MOVQ $0x7b0ab10dd0f809fe, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X2, X11
MOVQ $0x03363823e6e791e5, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X2
PXOR X11, X2
PXOR X1, X2
MOVQ $0x0c32cdb31e18a84a, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X3, X11
MOVQ $0x62242240ace5045a, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X3
PXOR X11, X3
PXOR X2, X3
MOVQ $0xbdd7ac0ee1a4a0f0, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X4, X11
MOVQ $0xa3ffdc1fe8e82a8b, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X4
PXOR X11, X4
PXOR X3, X4
MOVQ $0xb0bc2e589204f500, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X5, X11
MOVQ $0xe1e0bb9d45d7a44c, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X5
PXOR X11, X5
PXOR X4, X5
MOVQ $0xeadc41fd2ba3d420, AX
MOVQ AX, X11
PCLMULQDQ $0x00, X6, X11
MOVQ $0x21e9761e252621ac, AX
PINSRQ $0x1, AX, X12
PCLMULQDQ $0x11, X12, X6
PXOR X11, X6
PXOR X5, X6
MOVQ AX, X5
PCLMULQDQ $0x00, X6, X5
PSHUFD $0xee, X6, X6
PXOR X5, X6
MOVQ $0x27ecfa329aef9f77, AX
MOVQ AX, X4
PCLMULQDQ $0x00, X4, X6
PEXTRQ $0, X6, BX
MOVQ $0x34d926535897936b, AX
MOVQ AX, X4
PCLMULQDQ $0x00, X4, X6
PXOR X5, X6
PEXTRQ $1, X6, AX
XORQ BX, AX
skip128:
NOTQ AX
MOVQ AX, checksum+32(FP)
VZEROUPPER
RET
vendor/github.com/minio/crc64nvme/crc64_arm64.go
Generated Vendored
+2
View File
@@ -11,5 +11,7 @@ import (
)
var hasAsm = cpuid.CPU.Supports(cpuid.ASIMD, cpuid.PMULL, cpuid.SHA3)
var hasAsm512 = false
func updateAsm(crc uint64, p []byte) (checksum uint64)
func updateAsm512(crc uint64, p []byte) (checksum uint64) { panic("should not be reached") }
vendor/github.com/minio/crc64nvme/crc64_other.go
Generated Vendored
+3 -1
View File
@@ -7,5 +7,7 @@
package crc64nvme
var hasAsm = false
var hasAsm512 = false
func updateAsm(crc uint64, p []byte) (checksum uint64) { panic("should not be reached") }
func updateAsm(crc uint64, p []byte) (checksum uint64) { panic("should not be reached") }
func updateAsm512(crc uint64, p []byte) (checksum uint64) { panic("should not be reached") }
vendor/github.com/minio/minio-go/v7/CLAUDE.md
Generated Vendored
+125
View File
@@ -0,0 +1,125 @@
CLAUDE.md
=========
This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
Commands
--------
### Testing
```bash
# Run all tests with race detection (requires MinIO server at localhost:9000)
SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full go test -race -v ./...
# Run tests without race detection
go test ./...
# Run short tests only (no functional tests)
go test -short -race ./...
# Run functional tests
go build -race functional_tests.go
SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=1 MINT_MODE=full ./functional_tests
# Run functional tests without TLS
SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minioadmin SECRET_KEY=minioadmin ENABLE_HTTPS=0 MINT_MODE=full ./functional_tests
```
### Linting and Code Quality
```bash
# Run all checks (lint, vet, test, examples, functional tests)
make checks
# Run linter only
make lint
# Run vet and staticcheck
make vet
# Alternative: run golangci-lint directly
golangci-lint run --timeout=5m --config ./.golangci.yml
```
### Building Examples
```bash
# Build all examples
make examples
# Build a specific example
cd examples/s3 && go build -mod=mod putobject.go
```
Architecture
------------
### Core Client Structure
The MinIO Go SDK is organized around a central `Client` struct (api.go:52) that implements Amazon S3 compatible methods. Key architectural patterns:
1. **Modular API Organization**: API methods are split into logical files:
- `api-bucket-*.go`: Bucket operations (lifecycle, encryption, versioning, etc.)
- `api-object-*.go`: Object operations (legal hold, retention, tagging, etc.)
- `api-get-*.go`, `api-put-*.go`: GET and PUT operations
- `api-list.go`: Listing operations
- `api-stat.go`: Status/info operations
2. **Credential Management**: The `pkg/credentials/` package provides various credential providers:
- Static credentials
- Environment variables (AWS/MinIO)
- IAM roles
- STS (Security Token Service) variants
- File-based credentials
- Chain provider for fallback mechanisms
3. **Request Signing**: The `pkg/signer/` package handles AWS signature versions:
- V2 signatures (legacy)
- V4 signatures (standard)
- Streaming signatures for large uploads
4. **Transport Layer**: Custom HTTP transport with:
- Retry logic with configurable max retries
- Health status monitoring
- Tracing support via httptrace
- Bucket location caching (`bucketLocCache`\)
- Session caching for credentials
5. **Helper Packages**:
- `pkg/encrypt/`: Server-side encryption utilities
- `pkg/notification/`: Event notification handling
- `pkg/policy/`: Bucket policy management
- `pkg/lifecycle/`: Object lifecycle rules
- `pkg/tags/`: Object and bucket tagging
- `pkg/s3utils/`: S3 utility functions
- `pkg/kvcache/`: Key-value caching
- `pkg/singleflight/`: Deduplication of concurrent requests
### Testing Strategy
- Unit tests alongside implementation files (`*_test.go`\)
- Comprehensive functional tests in `functional_tests.go` requiring a live MinIO server
- Example programs in `examples/` directory demonstrating API usage
- Build tag `//go:build mint` for integration tests
### Error Handling
- Custom error types in `api-error-response.go`
- HTTP status code mapping
- Retry logic for transient failures
- Detailed error context preservation
Important Patterns
------------------
1. **Context Usage**: All API methods accept `context.Context` for cancellation and timeout control
2. **Options Pattern**: Methods use Options structs for optional parameters (e.g., `PutObjectOptions`, `GetObjectOptions`\)
3. **Streaming Support**: Large file operations use io.Reader/Writer interfaces for memory efficiency
4. **Bucket Lookup Types**: Supports both path-style and virtual-host-style S3 URLs
5. **MD5/SHA256 Hashing**: Configurable hash functions for integrity checks via `md5Hasher` and `sha256Hasher`
vendor/github.com/minio/minio-go/v7/CONTRIBUTING.md
Generated Vendored
+19 -18
View File
@@ -1,22 +1,23 @@
### Developer Guidelines
### Developer Guidelines
``minio-go`` welcomes your contribution. To make the process as seamless as possible, we ask for the following:
`minio-go` welcomes your contribution. To make the process as seamless as possible, we ask for the following:
* Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
- Fork it
- Create your feature branch (git checkout -b my-new-feature)
- Commit your changes (git commit -am 'Add some feature')
- Push to the branch (git push origin my-new-feature)
- Create new Pull Request
- Go ahead and fork the project and make your changes. We encourage pull requests to discuss code changes.
* When you're ready to create a pull request, be sure to:
- Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
- Run `go fmt`
- Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
- Make sure `go test -race ./...` and `go build` completes.
NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables
``ACCESS_KEY`` and ``SECRET_KEY``. To run shorter version of the tests please use ``go test -short -race ./...``
- Fork it
- Create your feature branch (git checkout -b my-new-feature)
- Commit your changes (git commit -am 'Add some feature')
- Push to the branch (git push origin my-new-feature)
- Create new Pull Request
* Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
- `minio-go` project is strictly conformant with Golang style
- if you happen to observe offending code, please feel free to send a pull request
- When you're ready to create a pull request, be sure to:
- Have test cases for the new code. If you have questions about how to do it, please ask in your pull request.
- Run `go fmt`
- Squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request.
- Make sure `go test -race ./...` and `go build` completes. NOTE: go test runs functional tests and requires you to have a AWS S3 account. Set them as environment variables`ACCESS_KEY` and `SECRET_KEY`. To run shorter version of the tests please use `go test -short -race ./...`
- Read [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project
- `minio-go` project is strictly conformant with Golang style
- if you happen to observe offending code, please feel free to send a pull request
vendor/github.com/minio/minio-go/v7/MAINTAINERS.md
Generated Vendored
+10 -2
View File
@@ -1,11 +1,15 @@
# For maintainers only
For maintainers only
====================
## Responsibilities
Responsibilities
----------------
Please go through this link [Maintainer Responsibility](https://gist.github.com/abperiasamy/f4d9b31d3186bbd26522)
### Making new releases
Tag and sign your release commit, additionally this step requires you to have access to MinIO's trusted private key.
```sh
$ export GNUPGHOME=/media/${USER}/minio/trusted
$ git tag -s 4.0.0
@@ -14,6 +18,7 @@ $ git push --tags
```
### Update version
Once release has been made update `libraryVersion` constant in `api.go` to next to be released version.
```sh
@@ -22,14 +27,17 @@ $ grep libraryVersion api.go
```
Commit your changes
```
$ git commit -a -m "Update version for next release" --author "MinIO Trusted <trusted@min.io>"
```
### Announce
Announce new release by adding release notes at https://github.com/minio/minio-go/releases from `trusted@min.io` account. Release notes requires two sections `highlights` and `changelog`. Highlights is a bulleted list of salient features in this release and Changelog contains list of all commits since the last release.
To generate `changelog`
```sh
$ git log --no-color --pretty=format:'-%d %s (%cr) <%an>' <last_release_tag>..<latest_release_tag>
```
vendor/github.com/minio/minio-go/v7/README.md
Generated Vendored
+153 -147
View File
@@ -1,13 +1,14 @@
# MinIO Go Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-go/blob/master/LICENSE)
MinIO Go Client SDK for Amazon S3 Compatible Cloud Storage [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Sourcegraph](https://sourcegraph.com/github.com/minio/minio-go/-/badge.svg)](https://sourcegraph.com/github.com/minio/minio-go?badge) [![Apache V2 License](https://img.shields.io/badge/license-Apache%20V2-blue.svg)](https://github.com/minio/minio-go/blob/master/LICENSE)
==================================================================================================================================================================================================================================================================================================================================================================================================================
The MinIO Go Client SDK provides straightforward APIs to access any Amazon S3 compatible object storage.
This Quickstart Guide covers how to install the MinIO client SDK, connect to MinIO, and create a sample file uploader.
For a complete list of APIs and examples, see the [godoc documentation](https://pkg.go.dev/github.com/minio/minio-go/v7) or [Go Client API Reference](https://min.io/docs/minio/linux/developers/go/API.html).
This Quickstart Guide covers how to install the MinIO client SDK, connect to MinIO, and create a sample file uploader. For a complete list of APIs and examples, see the [godoc documentation](https://pkg.go.dev/github.com/minio/minio-go/v7) or [Go Client API Reference](https://min.io/docs/minio/linux/developers/go/API.html).
These examples presume a working [Go development environment](https://golang.org/doc/install) and the [MinIO `mc` command line tool](https://min.io/docs/minio/linux/reference/minio-mc.html).
## Download from Github
Download from Github
--------------------
From your project directory:
@@ -15,12 +16,13 @@ From your project directory:
go get github.com/minio/minio-go/v7
```
## Initialize a MinIO Client Object
Initialize a MinIO Client Object
--------------------------------
The MinIO client requires the following parameters to connect to an Amazon S3 compatible object storage:
| Parameter | Description |
| ----------------- | ---------------------------------------------------------- |
|-------------------|------------------------------------------------------------|
| `endpoint` | URL to object storage service. |
| `_minio.Options_` | All the options such as credentials, custom transport etc. |
@@ -53,83 +55,82 @@ func main() {
}
```
## Example - File Uploader
Example - File Uploader
-----------------------
This sample code connects to an object storage server, creates a bucket, and uploads a file to the bucket.
It uses the MinIO `play` server, a public MinIO cluster located at [https://play.min.io](https://play.min.io).
This sample code connects to an object storage server, creates a bucket, and uploads a file to the bucket. It uses the MinIO `play` server, a public MinIO cluster located at [https://play.min.io](https://play.min.io).
The `play` server runs the latest stable version of MinIO and may be used for testing and development.
The access credentials shown in this example are open to the public and all data uploaded to `play` should be considered public and non-protected.
The `play` server runs the latest stable version of MinIO and may be used for testing and development. The access credentials shown in this example are open to the public and all data uploaded to `play` should be considered public and non-protected.
### FileUploader.go
This example does the following:
- Connects to the MinIO `play` server using the provided credentials.
- Creates a bucket named `testbucket`.
- Uploads a file named `testdata` from `/tmp`.
- Verifies the file was created using `mc ls`.
- Connects to the MinIO `play` server using the provided credentials.
- Creates a bucket named `testbucket`.
- Uploads a file named `testdata` from `/tmp`.
- Verifies the file was created using `mc ls`.
```go
// FileUploader.go MinIO example
package main
```go
// FileUploader.go MinIO example
package main
import (
"context"
"log"
import (
"context"
"log"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
)
func main() {
ctx := context.Background()
endpoint := "play.min.io"
accessKeyID := "Q3AM3UQ867SPQQA43P2F"
secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
useSSL := true
func main() {
ctx := context.Background()
endpoint := "play.min.io"
accessKeyID := "Q3AM3UQ867SPQQA43P2F"
secretAccessKey := "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG"
useSSL := true
// Initialize minio client object.
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
})
if err != nil {
log.Fatalln(err)
}
// Make a new bucket called testbucket.
bucketName := "testbucket"
location := "us-east-1"
err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
if err != nil {
// Check to see if we already own this bucket (which happens if you run this twice)
exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
if errBucketExists == nil && exists {
log.Printf("We already own %s\n", bucketName)
} else {
// Initialize minio client object.
minioClient, err := minio.New(endpoint, &minio.Options{
Creds: credentials.NewStaticV4(accessKeyID, secretAccessKey, ""),
Secure: useSSL,
})
if err != nil {
log.Fatalln(err)
}
} else {
log.Printf("Successfully created %s\n", bucketName)
// Make a new bucket called testbucket.
bucketName := "testbucket"
location := "us-east-1"
err = minioClient.MakeBucket(ctx, bucketName, minio.MakeBucketOptions{Region: location})
if err != nil {
// Check to see if we already own this bucket (which happens if you run this twice)
exists, errBucketExists := minioClient.BucketExists(ctx, bucketName)
if errBucketExists == nil && exists {
log.Printf("We already own %s\n", bucketName)
} else {
log.Fatalln(err)
}
} else {
log.Printf("Successfully created %s\n", bucketName)
}
// Upload the test file
// Change the value of filePath if the file is in another location
objectName := "testdata"
filePath := "/tmp/testdata"
contentType := "application/octet-stream"
// Upload the test file with FPutObject
info, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType})
if err != nil {
log.Fatalln(err)
}
log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
}
// Upload the test file
// Change the value of filePath if the file is in another location
objectName := "testdata"
filePath := "/tmp/testdata"
contentType := "application/octet-stream"
// Upload the test file with FPutObject
info, err := minioClient.FPutObject(ctx, bucketName, objectName, filePath, minio.PutObjectOptions{ContentType: contentType})
if err != nil {
log.Fatalln(err)
}
log.Printf("Successfully uploaded %s of size %d\n", objectName, info.Size)
}
```
```
**1. Create a test file containing data:**
@@ -168,145 +169,150 @@ mc ls play/testbucket
[2023-11-01 14:27:55 UTC] 20KiB STANDARD TestDataFile
```
## API Reference
API Reference
-------------
The full API Reference is available here.
* [Complete API Reference](https://min.io/docs/minio/linux/developers/go/API.html)
- [Complete API Reference](https://min.io/docs/minio/linux/developers/go/API.html)
### API Reference : Bucket Operations
* [`MakeBucket`](https://min.io/docs/minio/linux/developers/go/API.html#MakeBucket)
* [`ListBuckets`](https://min.io/docs/minio/linux/developers/go/API.html#ListBuckets)
* [`BucketExists`](https://min.io/docs/minio/linux/developers/go/API.html#BucketExists)
* [`RemoveBucket`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveBucket)
* [`ListObjects`](https://min.io/docs/minio/linux/developers/go/API.html#ListObjects)
* [`ListIncompleteUploads`](https://min.io/docs/minio/linux/developers/go/API.html#ListIncompleteUploads)
- [`MakeBucket`](https://min.io/docs/minio/linux/developers/go/API.html#MakeBucket)
- [`ListBuckets`](https://min.io/docs/minio/linux/developers/go/API.html#ListBuckets)
- [`BucketExists`](https://min.io/docs/minio/linux/developers/go/API.html#BucketExists)
- [`RemoveBucket`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveBucket)
- [`ListObjects`](https://min.io/docs/minio/linux/developers/go/API.html#ListObjects)
- [`ListIncompleteUploads`](https://min.io/docs/minio/linux/developers/go/API.html#ListIncompleteUploads)
### API Reference : Bucket policy Operations
* [`SetBucketPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#SetBucketPolicy)
* [`GetBucketPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#GetBucketPolicy)
- [`SetBucketPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#SetBucketPolicy)
- [`GetBucketPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#GetBucketPolicy)
### API Reference : Bucket notification Operations
* [`SetBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#SetBucketNotification)
* [`GetBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#GetBucketNotification)
* [`RemoveAllBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveAllBucketNotification)
* [`ListenBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#ListenBucketNotification) (MinIO Extension)
* [`ListenNotification`](https://min.io/docs/minio/linux/developers/go/API.html#ListenNotification) (MinIO Extension)
- [`SetBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#SetBucketNotification)
- [`GetBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#GetBucketNotification)
- [`RemoveAllBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveAllBucketNotification)
- [`ListenBucketNotification`](https://min.io/docs/minio/linux/developers/go/API.html#ListenBucketNotification) (MinIO Extension)
- [`ListenNotification`](https://min.io/docs/minio/linux/developers/go/API.html#ListenNotification) (MinIO Extension)
### API Reference : File Object Operations
* [`FPutObject`](https://min.io/docs/minio/linux/developers/go/API.html#FPutObject)
* [`FGetObject`](https://min.io/docs/minio/linux/developers/go/API.html#FGetObject)
- [`FPutObject`](https://min.io/docs/minio/linux/developers/go/API.html#FPutObject)
- [`FGetObject`](https://min.io/docs/minio/linux/developers/go/API.html#FGetObject)
### API Reference : Object Operations
* [`GetObject`](https://min.io/docs/minio/linux/developers/go/API.html#GetObject)
* [`PutObject`](https://min.io/docs/minio/linux/developers/go/API.html#PutObject)
* [`PutObjectStreaming`](https://min.io/docs/minio/linux/developers/go/API.html#PutObjectStreaming)
* [`StatObject`](https://min.io/docs/minio/linux/developers/go/API.html#StatObject)
* [`CopyObject`](https://min.io/docs/minio/linux/developers/go/API.html#CopyObject)
* [`RemoveObject`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObject)
* [`RemoveObjects`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObjects)
* [`RemoveIncompleteUpload`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveIncompleteUpload)
* [`SelectObjectContent`](https://min.io/docs/minio/linux/developers/go/API.html#SelectObjectContent)
- [`GetObject`](https://min.io/docs/minio/linux/developers/go/API.html#GetObject)
- [`PutObject`](https://min.io/docs/minio/linux/developers/go/API.html#PutObject)
- [`PutObjectStreaming`](https://min.io/docs/minio/linux/developers/go/API.html#PutObjectStreaming)
- [`StatObject`](https://min.io/docs/minio/linux/developers/go/API.html#StatObject)
- [`CopyObject`](https://min.io/docs/minio/linux/developers/go/API.html#CopyObject)
- [`RemoveObject`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObject)
- [`RemoveObjects`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveObjects)
- [`RemoveIncompleteUpload`](https://min.io/docs/minio/linux/developers/go/API.html#RemoveIncompleteUpload)
- [`SelectObjectContent`](https://min.io/docs/minio/linux/developers/go/API.html#SelectObjectContent)
### API Reference : Presigned Operations
* [`PresignedGetObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedGetObject)
* [`PresignedPutObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedPutObject)
* [`PresignedHeadObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedHeadObject)
* [`PresignedPostPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedPostPolicy)
- [`PresignedGetObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedGetObject)
- [`PresignedPutObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedPutObject)
- [`PresignedHeadObject`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedHeadObject)
- [`PresignedPostPolicy`](https://min.io/docs/minio/linux/developers/go/API.html#PresignedPostPolicy)
### API Reference : Client custom settings
* [`SetAppInfo`](https://min.io/docs/minio/linux/developers/go/API.html#SetAppInfo)
* [`TraceOn`](https://min.io/docs/minio/linux/developers/go/API.html#TraceOn)
* [`TraceOff`](https://min.io/docs/minio/linux/developers/go/API.html#TraceOff)
- [`SetAppInfo`](https://min.io/docs/minio/linux/developers/go/API.html#SetAppInfo)
- [`TraceOn`](https://min.io/docs/minio/linux/developers/go/API.html#TraceOn)
- [`TraceOff`](https://min.io/docs/minio/linux/developers/go/API.html#TraceOff)
## Full Examples
Full Examples
-------------
### Full Examples : Bucket Operations
* [makebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/makebucket.go)
* [listbuckets.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbuckets.go)
* [bucketexists.go](https://github.com/minio/minio-go/blob/master/examples/s3/bucketexists.go)
* [removebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucket.go)
* [listobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjects.go)
* [listobjectsV2.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjectsV2.go)
* [listincompleteuploads.go](https://github.com/minio/minio-go/blob/master/examples/s3/listincompleteuploads.go)
- [makebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/makebucket.go)
- [listbuckets.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbuckets.go)
- [bucketexists.go](https://github.com/minio/minio-go/blob/master/examples/s3/bucketexists.go)
- [removebucket.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucket.go)
- [listobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjects.go)
- [listobjectsV2.go](https://github.com/minio/minio-go/blob/master/examples/s3/listobjectsV2.go)
- [listincompleteuploads.go](https://github.com/minio/minio-go/blob/master/examples/s3/listincompleteuploads.go)
### Full Examples : Bucket policy Operations
* [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go)
* [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go)
* [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go)
- [setbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketpolicy.go)
- [getbucketpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketpolicy.go)
- [listbucketpolicies.go](https://github.com/minio/minio-go/blob/master/examples/s3/listbucketpolicies.go)
### Full Examples : Bucket lifecycle Operations
* [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go)
* [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go)
- [setbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketlifecycle.go)
- [getbucketlifecycle.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketlifecycle.go)
### Full Examples : Bucket encryption Operations
* [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go)
* [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go)
* [removebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketencryption.go)
- [setbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketencryption.go)
- [getbucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketencryption.go)
- [removebucketencryption.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketencryption.go)
### Full Examples : Bucket replication Operations
* [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go)
* [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go)
* [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go)
- [setbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketreplication.go)
- [getbucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketreplication.go)
- [removebucketreplication.go](https://github.com/minio/minio-go/blob/master/examples/s3/removebucketreplication.go)
### Full Examples : Bucket notification Operations
* [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go)
* [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go)
* [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go)
* [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension)
* [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO Extension)
- [setbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/setbucketnotification.go)
- [getbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/getbucketnotification.go)
- [removeallbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeallbucketnotification.go)
- [listenbucketnotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listenbucketnotification.go) (MinIO Extension)
- [listennotification.go](https://github.com/minio/minio-go/blob/master/examples/minio/listen-notification.go) (MinIO Extension)
### Full Examples : File Object Operations
* [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go)
* [fgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject.go)
- [fputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputobject.go)
- [fgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/fgetobject.go)
### Full Examples : Object Operations
* [putobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject.go)
* [getobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject.go)
* [statobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/statobject.go)
* [copyobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/copyobject.go)
* [removeobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobject.go)
* [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go)
* [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go)
- [putobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/putobject.go)
- [getobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/getobject.go)
- [statobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/statobject.go)
- [copyobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/copyobject.go)
- [removeobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobject.go)
- [removeincompleteupload.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeincompleteupload.go)
- [removeobjects.go](https://github.com/minio/minio-go/blob/master/examples/s3/removeobjects.go)
### Full Examples : Encrypted Object Operations
* [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go)
* [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go)
* [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go)
- [put-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/put-encrypted-object.go)
- [get-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/get-encrypted-object.go)
- [fput-encrypted-object.go](https://github.com/minio/minio-go/blob/master/examples/s3/fputencrypted-object.go)
### Full Examples : Presigned Operations
* [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go)
* [presignedputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedputobject.go)
* [presignedheadobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedheadobject.go)
* [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go)
- [presignedgetobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedgetobject.go)
- [presignedputobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedputobject.go)
- [presignedheadobject.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedheadobject.go)
- [presignedpostpolicy.go](https://github.com/minio/minio-go/blob/master/examples/s3/presignedpostpolicy.go)
## Explore Further
Explore Further
---------------
* [Godoc Documentation](https://pkg.go.dev/github.com/minio/minio-go/v7)
* [Complete Documentation](https://min.io/docs/minio/kubernetes/upstream/index.html)
* [MinIO Go Client SDK API Reference](https://min.io/docs/minio/linux/developers/go/API.html)
- [Godoc Documentation](https://pkg.go.dev/github.com/minio/minio-go/v7)
- [Complete Documentation](https://min.io/docs/minio/kubernetes/upstream/index.html)
- [MinIO Go Client SDK API Reference](https://min.io/docs/minio/linux/developers/go/API.html)
## Contribute
Contribute
----------
[Contributors Guide](https://github.com/minio/minio-go/blob/master/CONTRIBUTING.md)
## License
License
-------
This SDK is distributed under the [Apache License, Version 2.0](https://www.apache.org/licenses/LICENSE-2.0), see [LICENSE](https://github.com/minio/minio-go/blob/master/LICENSE) and [NOTICE](https://github.com/minio/minio-go/blob/master/NOTICE) for more information.
vendor/github.com/minio/minio-go/v7/api-bucket-cors.go
Generated Vendored
+17 -2
View File
@@ -26,7 +26,15 @@ import (
"github.com/minio/minio-go/v7/pkg/s3utils"
)
// SetBucketCors sets the cors configuration for the bucket
// SetBucketCors sets the Cross-Origin Resource Sharing (CORS) configuration for the bucket.
// If corsConfig is nil, the existing CORS configuration will be removed.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - corsConfig: CORS configuration to apply (nil to remove existing configuration)
//
// Returns an error if the operation fails.
func (c *Client) SetBucketCors(ctx context.Context, bucketName string, corsConfig *cors.Config) error {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
@@ -90,7 +98,14 @@ func (c *Client) removeBucketCors(ctx context.Context, bucketName string) error
return nil
}
// GetBucketCors returns the current cors
// GetBucketCors retrieves the Cross-Origin Resource Sharing (CORS) configuration from the bucket.
// If no CORS configuration exists, returns nil with no error.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the CORS configuration or an error if the operation fails.
func (c *Client) GetBucketCors(ctx context.Context, bucketName string) (*cors.Config, error) {
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return nil, err
vendor/github.com/minio/minio-go/v7/api-bucket-encryption.go
Generated Vendored
+25 -3
View File
@@ -28,6 +28,14 @@ import (
)
// SetBucketEncryption sets the default encryption configuration on an existing bucket.
// The encryption configuration specifies the default encryption behavior for objects uploaded to the bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - config: Server-side encryption configuration to apply
//
// Returns an error if the operation fails or if config is nil.
func (c *Client) SetBucketEncryption(ctx context.Context, bucketName string, config *sse.Configuration) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -69,7 +77,15 @@ func (c *Client) SetBucketEncryption(ctx context.Context, bucketName string, con
return nil
}
// RemoveBucketEncryption removes the default encryption configuration on a bucket with a context to control cancellations and timeouts.
// RemoveBucketEncryption removes the default encryption configuration from a bucket.
// After removal, the bucket will no longer apply default encryption to new objects.
// It uses the provided context to control cancellations and timeouts.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns an error if the operation fails.
func (c *Client) RemoveBucketEncryption(ctx context.Context, bucketName string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -97,8 +113,14 @@ func (c *Client) RemoveBucketEncryption(ctx context.Context, bucketName string)
return nil
}
// GetBucketEncryption gets the default encryption configuration
// on an existing bucket with a context to control cancellations and timeouts.
// GetBucketEncryption retrieves the default encryption configuration from a bucket.
// It uses the provided context to control cancellations and timeouts.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the bucket's encryption configuration or an error if the operation fails.
func (c *Client) GetBucketEncryption(ctx context.Context, bucketName string) (*sse.Configuration, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
vendor/github.com/minio/minio-go/v7/api-bucket-notification.go
Generated Vendored
+1 -1
View File
@@ -21,12 +21,12 @@ import (
"bufio"
"bytes"
"context"
"encoding/json"
"encoding/xml"
"net/http"
"net/url"
"time"
"github.com/minio/minio-go/v7/internal/json"
"github.com/minio/minio-go/v7/pkg/notification"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
vendor/github.com/minio/minio-go/v7/api-bucket-policy.go
Generated Vendored
+18 -2
View File
@@ -26,7 +26,16 @@ import (
"github.com/minio/minio-go/v7/pkg/s3utils"
)
// SetBucketPolicy sets the access permissions on an existing bucket.
// SetBucketPolicy sets the access permissions policy on an existing bucket.
// The policy should be a valid JSON string that conforms to the IAM policy format.
// If policy is an empty string, the existing bucket policy will be removed.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - policy: JSON policy string (empty string to remove existing policy)
//
// Returns an error if the operation fails.
func (c *Client) SetBucketPolicy(ctx context.Context, bucketName, policy string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -95,7 +104,14 @@ func (c *Client) removeBucketPolicy(ctx context.Context, bucketName string) erro
return nil
}
// GetBucketPolicy returns the current policy
// GetBucketPolicy retrieves the access permissions policy for the bucket.
// If no bucket policy exists, returns an empty string with no error.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the policy as a JSON string or an error if the operation fails.
func (c *Client) GetBucketPolicy(ctx context.Context, bucketName string) (string, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
vendor/github.com/minio/minio-go/v7/api-bucket-qos.go
Generated Vendored
+212
View File
@@ -0,0 +1,212 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2025 MinIO, Inc.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"encoding/json"
"io"
"net/http"
"net/url"
"strings"
"github.com/minio/minio-go/v7/pkg/s3utils"
"gopkg.in/yaml.v3"
)
// QOSConfigVersionCurrent is the current version of the QoS configuration.
const QOSConfigVersionCurrent = "v1"
// QOSConfig represents the QoS configuration for a bucket.
type QOSConfig struct {
Version string `yaml:"version"`
Rules []QOSRule `yaml:"rules"`
}
// QOSRule represents a single QoS rule.
type QOSRule struct {
ID string `yaml:"id"`
Label string `yaml:"label,omitempty"`
Priority int `yaml:"priority"`
ObjectPrefix string `yaml:"objectPrefix"`
API string `yaml:"api"`
Rate int64 `yaml:"rate"`
Burst int64 `yaml:"burst"` // not required for concurrency limit
Limit string `yaml:"limit"` // "concurrency" or "rps"
}
// NewQOSConfig creates a new empty QoS configuration.
func NewQOSConfig() *QOSConfig {
return &QOSConfig{
Version: "v1",
Rules: []QOSRule{},
}
}
// GetBucketQOS retrieves the Quality of Service (QoS) configuration for the bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
//
// Returns the QoS configuration or an error if the operation fails.
func (c *Client) GetBucketQOS(ctx context.Context, bucket string) (*QOSConfig, error) {
var qosCfg QOSConfig
// Input validation.
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return nil, err
}
urlValues := make(url.Values)
urlValues.Set("qos", "")
// Execute GET on bucket to fetch qos.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucket,
queryValues: urlValues,
contentSHA256Hex: emptySHA256Hex,
})
defer closeResponse(resp)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucket, "")
}
b, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if err = yaml.Unmarshal(b, &qosCfg); err != nil {
return nil, err
}
return &qosCfg, nil
}
// SetBucketQOS sets the Quality of Service (QoS) configuration for a bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - qosCfg: QoS configuration to apply
//
// Returns an error if the operation fails.
func (c *Client) SetBucketQOS(ctx context.Context, bucket string, qosCfg *QOSConfig) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return err
}
data, err := yaml.Marshal(qosCfg)
if err != nil {
return err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("qos", "")
reqMetadata := requestMetadata{
bucketName: bucket,
queryValues: urlValues,
contentBody: strings.NewReader(string(data)),
contentLength: int64(len(data)),
}
// Execute PUT to upload a new bucket QoS configuration.
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
}
if resp != nil {
if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucket, "")
}
}
return nil
}
// CounterMetric returns stats for a counter
type CounterMetric struct {
Last1m uint64 `json:"last1m"`
Last1hr uint64 `json:"last1hr"`
Total uint64 `json:"total"`
}
// QOSMetric - metric for a qos rule per bucket
type QOSMetric struct {
APIName string `json:"apiName"`
Rule QOSRule `json:"rule"`
Totals CounterMetric `json:"totals"`
Throttled CounterMetric `json:"throttleCount"`
ExceededRateLimit CounterMetric `json:"exceededRateLimitCount"`
ClientDisconnCount CounterMetric `json:"clientDisconnectCount"`
ReqTimeoutCount CounterMetric `json:"reqTimeoutCount"`
}
// QOSNodeStats represents stats for a bucket on a single node
type QOSNodeStats struct {
Stats []QOSMetric `json:"stats"`
NodeName string `json:"node"`
}
// GetBucketQOSMetrics retrieves Quality of Service (QoS) metrics for a bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - nodeName: Name of the node (empty string for all nodes)
//
// Returns QoS metrics per node or an error if the operation fails.
func (c *Client) GetBucketQOSMetrics(ctx context.Context, bucketName, nodeName string) (qs []QOSNodeStats, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return qs, err
}
// Get resources properly escaped and lined up before
// using them in http request.
urlValues := make(url.Values)
urlValues.Set("qos-metrics", "")
if nodeName != "" {
urlValues.Set("node", nodeName)
}
// Execute GET on bucket to get qos metrics.
resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{
bucketName: bucketName,
queryValues: urlValues,
})
defer closeResponse(resp)
if err != nil {
return qs, err
}
if resp.StatusCode != http.StatusOK {
return qs, httpRespToErrorResponse(resp, bucketName, "")
}
respBytes, err := io.ReadAll(resp.Body)
if err != nil {
return qs, err
}
if err := json.Unmarshal(respBytes, &qs); err != nil {
return qs, err
}
return qs, nil
}
vendor/github.com/minio/minio-go/v7/api-bucket-replication.go
Generated Vendored
+83 -16
View File
@@ -20,6 +20,7 @@ package minio
import (
"bytes"
"context"
"encoding/json"
"encoding/xml"
"io"
"net/http"
@@ -27,17 +28,30 @@ import (
"time"
"github.com/google/uuid"
"github.com/minio/minio-go/v7/internal/json"
"github.com/minio/minio-go/v7/pkg/replication"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
// RemoveBucketReplication removes a replication config on an existing bucket.
// RemoveBucketReplication removes the replication configuration from an existing bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns an error if the operation fails.
func (c *Client) RemoveBucketReplication(ctx context.Context, bucketName string) error {
return c.removeBucketReplication(ctx, bucketName)
}
// SetBucketReplication sets a replication config on an existing bucket.
// SetBucketReplication sets the replication configuration on an existing bucket.
// If the provided configuration is empty, this method removes the existing replication configuration.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - cfg: Replication configuration to apply
//
// Returns an error if the operation fails.
func (c *Client) SetBucketReplication(ctx context.Context, bucketName string, cfg replication.Config) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -108,8 +122,14 @@ func (c *Client) removeBucketReplication(ctx context.Context, bucketName string)
return nil
}
// GetBucketReplication fetches bucket replication configuration.If config is not
// found, returns empty config with nil error.
// GetBucketReplication retrieves the bucket replication configuration.
// If no replication configuration is found, returns an empty config with nil error.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the replication configuration or an error if the operation fails.
func (c *Client) GetBucketReplication(ctx context.Context, bucketName string) (cfg replication.Config, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -155,7 +175,13 @@ func (c *Client) getBucketReplication(ctx context.Context, bucketName string) (c
return cfg, nil
}
// GetBucketReplicationMetrics fetches bucket replication status metrics
// GetBucketReplicationMetrics retrieves bucket replication status metrics.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the replication metrics or an error if the operation fails.
func (c *Client) GetBucketReplicationMetrics(ctx context.Context, bucketName string) (s replication.Metrics, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -200,8 +226,15 @@ func mustGetUUID() string {
return u.String()
}
// ResetBucketReplication kicks off replication of previously replicated objects if ExistingObjectReplication
// is enabled in the replication config
// ResetBucketReplication initiates replication of previously replicated objects.
// This requires ExistingObjectReplication to be enabled in the replication configuration.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - olderThan: Only replicate objects older than this duration (0 for all objects)
//
// Returns a reset ID that can be used to track the operation, or an error if the operation fails.
func (c *Client) ResetBucketReplication(ctx context.Context, bucketName string, olderThan time.Duration) (rID string, err error) {
rID = mustGetUUID()
_, err = c.resetBucketReplicationOnTarget(ctx, bucketName, olderThan, "", rID)
@@ -211,8 +244,16 @@ func (c *Client) ResetBucketReplication(ctx context.Context, bucketName string,
return rID, nil
}
// ResetBucketReplicationOnTarget kicks off replication of previously replicated objects if
// ExistingObjectReplication is enabled in the replication config
// ResetBucketReplicationOnTarget initiates replication of previously replicated objects to a specific target.
// This requires ExistingObjectReplication to be enabled in the replication configuration.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - olderThan: Only replicate objects older than this duration (0 for all objects)
// - tgtArn: ARN of the target to reset replication for
//
// Returns resync target information or an error if the operation fails.
func (c *Client) ResetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn string) (replication.ResyncTargetsInfo, error) {
return c.resetBucketReplicationOnTarget(ctx, bucketName, olderThan, tgtArn, mustGetUUID())
}
@@ -222,7 +263,7 @@ func (c *Client) ResetBucketReplicationOnTarget(ctx context.Context, bucketName
func (c *Client) resetBucketReplicationOnTarget(ctx context.Context, bucketName string, olderThan time.Duration, tgtArn, resetID string) (rinfo replication.ResyncTargetsInfo, err error) {
// Input validation.
if err = s3utils.CheckValidBucketName(bucketName); err != nil {
return
return rinfo, err
}
// Get resources properly escaped and lined up before
// using them in http request.
@@ -256,7 +297,14 @@ func (c *Client) resetBucketReplicationOnTarget(ctx context.Context, bucketName
return rinfo, nil
}
// GetBucketReplicationResyncStatus gets the status of replication resync
// GetBucketReplicationResyncStatus retrieves the status of a replication resync operation.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - arn: ARN of the replication target (empty string for all targets)
//
// Returns resync status information or an error if the operation fails.
func (c *Client) GetBucketReplicationResyncStatus(ctx context.Context, bucketName, arn string) (rinfo replication.ResyncTargetsInfo, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -290,11 +338,18 @@ func (c *Client) GetBucketReplicationResyncStatus(ctx context.Context, bucketNam
return rinfo, nil
}
// CancelBucketReplicationResync cancels in progress replication resync
// CancelBucketReplicationResync cancels an in-progress replication resync operation.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - tgtArn: ARN of the replication target (empty string for all targets)
//
// Returns the ID of the canceled resync operation or an error if the operation fails.
func (c *Client) CancelBucketReplicationResync(ctx context.Context, bucketName string, tgtArn string) (id string, err error) {
// Input validation.
if err = s3utils.CheckValidBucketName(bucketName); err != nil {
return
return id, err
}
// Get resources properly escaped and lined up before
// using them in http request.
@@ -326,7 +381,13 @@ func (c *Client) CancelBucketReplicationResync(ctx context.Context, bucketName s
return id, nil
}
// GetBucketReplicationMetricsV2 fetches bucket replication status metrics
// GetBucketReplicationMetricsV2 retrieves bucket replication status metrics using the V2 API.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the V2 replication metrics or an error if the operation fails.
func (c *Client) GetBucketReplicationMetricsV2(ctx context.Context, bucketName string) (s replication.MetricsV2, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -362,7 +423,13 @@ func (c *Client) GetBucketReplicationMetricsV2(ctx context.Context, bucketName s
return s, nil
}
// CheckBucketReplication validates if replication is set up properly for a bucket
// CheckBucketReplication validates whether replication is properly configured for a bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns nil if replication is valid, or an error describing the validation failure.
func (c *Client) CheckBucketReplication(ctx context.Context, bucketName string) (err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
vendor/github.com/minio/minio-go/v7/api-bucket-tagging.go
Generated Vendored
+25 -6
View File
@@ -29,8 +29,14 @@ import (
"github.com/minio/minio-go/v7/pkg/tags"
)
// GetBucketTagging fetch tagging configuration for a bucket with a
// context to control cancellations and timeouts.
// GetBucketTagging fetches the tagging configuration for a bucket.
// It uses the provided context to control cancellations and timeouts.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns the bucket's tags or an error if the operation fails.
func (c *Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags.Tags, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -61,8 +67,15 @@ func (c *Client) GetBucketTagging(ctx context.Context, bucketName string) (*tags
return tags.ParseBucketXML(resp.Body)
}
// SetBucketTagging sets tagging configuration for a bucket
// with a context to control cancellations and timeouts.
// SetBucketTagging sets the tagging configuration for a bucket.
// It uses the provided context to control cancellations and timeouts.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - tags: Tag set to apply to the bucket
//
// Returns an error if the operation fails or if tags is nil.
func (c *Client) SetBucketTagging(ctx context.Context, bucketName string, tags *tags.Tags) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -104,8 +117,14 @@ func (c *Client) SetBucketTagging(ctx context.Context, bucketName string, tags *
return nil
}
// RemoveBucketTagging removes tagging configuration for a
// bucket with a context to control cancellations and timeouts.
// RemoveBucketTagging removes the tagging configuration from a bucket.
// It uses the provided context to control cancellations and timeouts.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
//
// Returns an error if the operation fails.
func (c *Client) RemoveBucketTagging(ctx context.Context, bucketName string) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
vendor/github.com/minio/minio-go/v7/api-compose-object.go
Generated Vendored
+9 -4
View File
@@ -42,13 +42,15 @@ type CopyDestOptions struct {
// provided key. If it is nil, no encryption is performed.
Encryption encrypt.ServerSide
ChecksumType ChecksumType
// `userMeta` is the user-metadata key-value pairs to be set on the
// destination. The keys are automatically prefixed with `x-amz-meta-`
// if needed. If nil is passed, and if only a single source (of any
// size) is provided in the ComposeObject call, then metadata from the
// source is copied to the destination.
// if no user-metadata is provided, it is copied from source
// (when there is only once source object in the compose
// (when there is only one source object in the compose
// request)
UserMetadata map[string]string
// UserMetadata is only set to destination if ReplaceMetadata is true
@@ -140,6 +142,9 @@ func (opts CopyDestOptions) Marshal(header http.Header) {
if !opts.Expires.IsZero() {
header.Set("Expires", opts.Expires.UTC().Format(http.TimeFormat))
}
if opts.ChecksumType.IsSet() {
header.Set(amzChecksumAlgo, opts.ChecksumType.String())
}
if opts.ReplaceMetadata {
header.Set("x-amz-metadata-directive", replaceDirective)
@@ -345,7 +350,7 @@ func (c *Client) copyObjectPartDo(ctx context.Context, srcBucket, srcObject, des
})
defer closeResponse(resp)
if err != nil {
return
return p, err
}
// Check if we got an error response.
@@ -580,7 +585,7 @@ func partsRequired(size int64) int64 {
// it is not the last part.
func calculateEvenSplits(size int64, src CopySrcOptions) (startIndex, endIndex []int64) {
if size == 0 {
return
return startIndex, endIndex
}
reqParts := partsRequired(size)
@@ -617,5 +622,5 @@ func calculateEvenSplits(size int64, src CopySrcOptions) (startIndex, endIndex [
startIndex[j], endIndex[j] = cStart, cEnd
}
return
return startIndex, endIndex
}
vendor/github.com/minio/minio-go/v7/api-error-response.go
Generated Vendored
+10
View File
@@ -20,6 +20,7 @@ package minio
import (
"bytes"
"encoding/xml"
"errors"
"fmt"
"io"
"net/http"
@@ -128,9 +129,18 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string)
Server: resp.Header.Get("Server"),
}
_, success := successStatus[resp.StatusCode]
errBody, err := xmlDecodeAndBody(resp.Body, &errResp)
// Xml decoding failed with no body, fall back to HTTP headers.
if err != nil {
var unmarshalErr xml.UnmarshalError
if success && errors.As(err, &unmarshalErr) {
// This is a successful message so not an error response
// return nil,
return nil
}
switch resp.StatusCode {
case http.StatusNotFound:
if objectName == "" {
vendor/github.com/minio/minio-go/v7/api-get-object-attributes.go
Generated Vendored
+1 -1
View File
@@ -127,7 +127,7 @@ func (o *ObjectAttributes) parseResponse(resp *http.Response) (err error) {
}
o.ObjectAttributesResponse = *response
return
return err
}
// GetObjectAttributes API combines HeadObject and ListParts.
vendor/github.com/minio/minio-go/v7/api-get-object-file.go
Generated Vendored
+1 -1
View File
@@ -69,7 +69,7 @@ func (c *Client) FGetObject(ctx context.Context, bucketName, objectName, filePat
}
// Write to a temporary file "fileName.part.minio" before saving.
filePartPath := filePath + sum256Hex([]byte(objectStat.ETag)) + ".part.minio"
filePartPath := filepath.Join(filepath.Dir(filePath), sum256Hex([]byte(filepath.Base(filePath)+objectStat.ETag))+".part.minio")
// If exists, open in append mode. If not create it as a part file.
filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
vendor/github.com/minio/minio-go/v7/api-inventory-ext.go
Generated Vendored
+332
View File
@@ -0,0 +1,332 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2025 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package minio
import (
"context"
"encoding/json"
"io"
"iter"
"net/http"
"net/url"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
// This file contains the inventory API extension for MinIO server. It is not
// compatible with AWS S3.
func makeInventoryReqMetadata(bucket string, urlParams ...string) requestMetadata {
urlValues := make(url.Values)
urlValues.Set("minio-inventory", "")
// If an odd number of parameters is given, we skip the last pair to avoid
// an out of bounds access.
for i := 0; i+1 < len(urlParams); i += 2 {
urlValues.Set(urlParams[i], urlParams[i+1])
}
return requestMetadata{
bucketName: bucket,
queryValues: urlValues,
}
}
// GenerateInventoryConfigYAML generates a YAML template for an inventory configuration.
// This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - id: Unique identifier for the inventory configuration
//
// Returns a YAML template string that can be customized and used with PutBucketInventoryConfiguration.
func (c *Client) GenerateInventoryConfigYAML(ctx context.Context, bucket, id string) (string, error) {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return "", err
}
if id == "" {
return "", errInvalidArgument("inventory ID cannot be empty")
}
reqMeta := makeInventoryReqMetadata(bucket, "generate", "", "id", id)
resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta)
defer closeResponse(resp)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", httpRespToErrorResponse(resp, bucket, "")
}
buf := new(strings.Builder)
_, err = io.Copy(buf, resp.Body)
return buf.String(), err
}
// inventoryPutConfigOpts is a placeholder for future options that may be added.
type inventoryPutConfigOpts struct{}
// InventoryPutConfigOption is to allow for functional options for
// PutBucketInventoryConfiguration. It may be used in the future to customize
// the PutBucketInventoryConfiguration request, but currently does not do
// anything.
type InventoryPutConfigOption func(*inventoryPutConfigOpts)
// PutBucketInventoryConfiguration creates or updates an inventory configuration for a bucket.
// This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - id: Unique identifier for the inventory configuration
// - yamlDef: YAML definition of the inventory configuration
//
// Returns an error if the operation fails, or if bucket name, id, or yamlDef is empty.
func (c *Client) PutBucketInventoryConfiguration(ctx context.Context, bucket string, id string, yamlDef string, _ ...InventoryPutConfigOption) error {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return err
}
if id == "" {
return errInvalidArgument("inventory ID cannot be empty")
}
if yamlDef == "" {
return errInvalidArgument("YAML definition cannot be empty")
}
reqMeta := makeInventoryReqMetadata(bucket, "id", id)
reqMeta.contentBody = strings.NewReader(yamlDef)
reqMeta.contentLength = int64(len(yamlDef))
reqMeta.contentMD5Base64 = sumMD5Base64([]byte(yamlDef))
resp, err := c.executeMethod(ctx, http.MethodPut, reqMeta)
defer closeResponse(resp)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucket, "")
}
return nil
}
// GetBucketInventoryConfiguration retrieves the inventory configuration for a bucket.
// This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - id: Unique identifier for the inventory configuration
//
// Returns the inventory configuration or an error if the operation fails or if the configuration doesn't exist.
func (c *Client) GetBucketInventoryConfiguration(ctx context.Context, bucket, id string) (*InventoryConfiguration, error) {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return nil, err
}
if id == "" {
return nil, errInvalidArgument("inventory ID cannot be empty")
}
reqMeta := makeInventoryReqMetadata(bucket, "id", id)
resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta)
defer closeResponse(resp)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucket, "")
}
decoder := json.NewDecoder(resp.Body)
var ic InventoryConfiguration
err = decoder.Decode(&ic)
if err != nil {
return nil, err
}
return &ic, nil
}
// DeleteBucketInventoryConfiguration deletes an inventory configuration from a bucket.
// This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - id: Unique identifier for the inventory configuration to delete
//
// Returns an error if the operation fails or if the configuration doesn't exist.
func (c *Client) DeleteBucketInventoryConfiguration(ctx context.Context, bucket, id string) error {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return err
}
if id == "" {
return errInvalidArgument("inventory ID cannot be empty")
}
reqMeta := makeInventoryReqMetadata(bucket, "id", id)
resp, err := c.executeMethod(ctx, http.MethodDelete, reqMeta)
defer closeResponse(resp)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucket, "")
}
return nil
}
// InventoryConfiguration represents the inventory configuration
type InventoryConfiguration struct {
Bucket string `json:"bucket"`
ID string `json:"id"`
User string `json:"user"`
YamlDef string `json:"yamlDef,omitempty"`
}
// InventoryListResult represents the result of listing inventory
// configurations.
type InventoryListResult struct {
Items []InventoryConfiguration `json:"items"`
NextContinuationToken string `json:"nextContinuationToken,omitempty"`
}
// ListBucketInventoryConfigurations lists up to 100 inventory configurations for a bucket.
// This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - continuationToken: Token for pagination (empty string for first request)
//
// Returns a list result with configurations and a continuation token for the next page, or an error.
func (c *Client) ListBucketInventoryConfigurations(ctx context.Context, bucket, continuationToken string) (lr *InventoryListResult, err error) {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return nil, err
}
reqMeta := makeInventoryReqMetadata(bucket, "continuation-token", continuationToken)
resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta)
defer closeResponse(resp)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucket, "")
}
decoder := json.NewDecoder(resp.Body)
err = decoder.Decode(&lr)
if err != nil {
return nil, err
}
return lr, nil
}
// ListBucketInventoryConfigurationsIterator returns an iterator that lists all inventory configurations
// for a bucket. This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
//
// Returns an iterator that yields InventoryConfiguration values and errors. The iterator automatically
// handles pagination and fetches all configurations.
func (c *Client) ListBucketInventoryConfigurationsIterator(ctx context.Context, bucket string) iter.Seq2[InventoryConfiguration, error] {
return func(yield func(InventoryConfiguration, error) bool) {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
yield(InventoryConfiguration{}, err)
return
}
var continuationToken string
for {
listResult, err := c.ListBucketInventoryConfigurations(ctx, bucket, continuationToken)
if err != nil {
yield(InventoryConfiguration{}, err)
return
}
for _, item := range listResult.Items {
if !yield(item, nil) {
return
}
}
if listResult.NextContinuationToken == "" {
return
}
continuationToken = listResult.NextContinuationToken
}
}
}
// InventoryJobStatus represents the status of an inventory job.
type InventoryJobStatus struct {
Bucket string `json:"bucket"`
ID string `json:"id"`
User string `json:"user"`
AccessKey string `json:"accessKey"`
Schedule string `json:"schedule"`
State string `json:"state"`
NextScheduledTime time.Time `json:"nextScheduledTime,omitempty"`
StartTime time.Time `json:"startTime,omitempty"`
EndTime time.Time `json:"endTime,omitempty"`
LastUpdate time.Time `json:"lastUpdate,omitempty"`
Scanned string `json:"scanned,omitempty"`
Matched string `json:"matched,omitempty"`
ScannedCount uint64 `json:"scannedCount,omitempty"`
MatchedCount uint64 `json:"matchedCount,omitempty"`
RecordsWritten uint64 `json:"recordsWritten,omitempty"`
OutputFilesCount uint64 `json:"outputFilesCount,omitempty"`
ExecutionTime string `json:"executionTime,omitempty"`
NumStarts uint64 `json:"numStarts,omitempty"`
NumErrors uint64 `json:"numErrors,omitempty"`
NumLockLosses uint64 `json:"numLockLosses,omitempty"`
ManifestPath string `json:"manifestPath,omitempty"`
RetryAttempts uint64 `json:"retryAttempts,omitempty"`
LastFailTime time.Time `json:"lastFailTime,omitempty"`
LastFailErrors []string `json:"lastFailErrors,omitempty"`
}
// GetBucketInventoryJobStatus retrieves the status of an inventory job for a bucket.
// This is a MinIO-specific API and is not compatible with AWS S3.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucket: Name of the bucket
// - id: Unique identifier for the inventory job
//
// Returns the inventory job status including execution state, progress, and error information, or an error if the operation fails.
func (c *Client) GetBucketInventoryJobStatus(ctx context.Context, bucket, id string) (*InventoryJobStatus, error) {
if err := s3utils.CheckValidBucketName(bucket); err != nil {
return nil, err
}
if id == "" {
return nil, errInvalidArgument("inventory ID cannot be empty")
}
reqMeta := makeInventoryReqMetadata(bucket, "id", id, "status", "")
resp, err := c.executeMethod(ctx, http.MethodGet, reqMeta)
defer closeResponse(resp)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, httpRespToErrorResponse(resp, bucket, "")
}
decoder := json.NewDecoder(resp.Body)
var jStatus InventoryJobStatus
err = decoder.Decode(&jStatus)
if err != nil {
return nil, err
}
return &jStatus, nil
}
vendor/github.com/minio/minio-go/v7/api-object-legal-hold.go
Generated Vendored
+19 -2
View File
@@ -80,7 +80,16 @@ func newObjectLegalHold(status *LegalHoldStatus) (*objectLegalHold, error) {
return legalHold, nil
}
// PutObjectLegalHold : sets object legal hold for a given object and versionID.
// PutObjectLegalHold sets the legal hold status for an object and specific version.
// Legal hold prevents an object version from being overwritten or deleted, regardless of retention settings.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - opts: Options including Status (LegalHoldEnabled or LegalHoldDisabled) and optional VersionID
//
// Returns an error if the operation fails or if the status is invalid.
func (c *Client) PutObjectLegalHold(ctx context.Context, bucketName, objectName string, opts PutObjectLegalHoldOptions) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -134,7 +143,15 @@ func (c *Client) PutObjectLegalHold(ctx context.Context, bucketName, objectName
return nil
}
// GetObjectLegalHold gets legal-hold status of given object.
// GetObjectLegalHold retrieves the legal hold status for an object and specific version.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - opts: Options including optional VersionID to target a specific version
//
// Returns the legal hold status (LegalHoldEnabled or LegalHoldDisabled) or an error if the operation fails.
func (c *Client) GetObjectLegalHold(ctx context.Context, bucketName, objectName string, opts GetObjectLegalHoldOptions) (status *LegalHoldStatus, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
vendor/github.com/minio/minio-go/v7/api-object-retention.go
Generated Vendored
+19 -2
View File
@@ -62,7 +62,16 @@ type PutObjectRetentionOptions struct {
VersionID string
}
// PutObjectRetention sets object retention for a given object and versionID.
// PutObjectRetention sets the retention configuration for an object and specific version.
// Object retention prevents an object version from being deleted or overwritten for a specified period.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - opts: Options including Mode (GOVERNANCE or COMPLIANCE), RetainUntilDate, optional VersionID, and GovernanceBypass
//
// Returns an error if the operation fails or if the retention settings are invalid.
func (c *Client) PutObjectRetention(ctx context.Context, bucketName, objectName string, opts PutObjectRetentionOptions) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -125,7 +134,15 @@ func (c *Client) PutObjectRetention(ctx context.Context, bucketName, objectName
return nil
}
// GetObjectRetention gets retention of given object.
// GetObjectRetention retrieves the retention configuration for an object and specific version.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - versionID: Optional version ID to target a specific version (empty string for current version)
//
// Returns the retention mode (GOVERNANCE or COMPLIANCE), retain-until date, and any error.
func (c *Client) GetObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *RetentionMode, retainUntilDate *time.Time, err error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
vendor/github.com/minio/minio-go/v7/api-object-tagging.go
Generated Vendored
+31 -6
View File
@@ -40,8 +40,17 @@ type AdvancedObjectTaggingOptions struct {
ReplicationProxyRequest string
}
// PutObjectTagging replaces or creates object tag(s) and can target
// a specific object version in a versioned bucket.
// PutObjectTagging replaces or creates object tag(s) and can target a specific object version
// in a versioned bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - otags: Tags to apply to the object
// - opts: Options including VersionID to target a specific version
//
// Returns an error if the operation fails.
func (c *Client) PutObjectTagging(ctx context.Context, bucketName, objectName string, otags *tags.Tags, opts PutObjectTaggingOptions) error {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
@@ -96,8 +105,16 @@ type GetObjectTaggingOptions struct {
Internal AdvancedObjectTaggingOptions
}
// GetObjectTagging fetches object tag(s) with options to target
// a specific object version in a versioned bucket.
// GetObjectTagging retrieves object tag(s) with options to target a specific object version
// in a versioned bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - opts: Options including VersionID to target a specific version
//
// Returns the object's tags or an error if the operation fails.
func (c *Client) GetObjectTagging(ctx context.Context, bucketName, objectName string, opts GetObjectTaggingOptions) (*tags.Tags, error) {
// Get resources properly escaped and lined up before
// using them in http request.
@@ -139,8 +156,16 @@ type RemoveObjectTaggingOptions struct {
Internal AdvancedObjectTaggingOptions
}
// RemoveObjectTagging removes object tag(s) with options to control a specific object
// version in a versioned bucket
// RemoveObjectTagging removes object tag(s) with options to target a specific object version
// in a versioned bucket.
//
// Parameters:
// - ctx: Context for request cancellation and timeout
// - bucketName: Name of the bucket
// - objectName: Name of the object
// - opts: Options including VersionID to target a specific version
//
// Returns an error if the operation fails.
func (c *Client) RemoveObjectTagging(ctx context.Context, bucketName, objectName string, opts RemoveObjectTaggingOptions) error {
// Get resources properly escaped and lined up before
// using them in http request.
vendor/github.com/minio/minio-go/v7/api-prompt-object.go
Generated Vendored
+1 -1
View File
@@ -20,10 +20,10 @@ package minio
import (
"bytes"
"context"
"encoding/json"
"io"
"net/http"
"github.com/minio/minio-go/v7/internal/json"
"github.com/minio/minio-go/v7/pkg/s3utils"
)
vendor/github.com/minio/minio-go/v7/api-put-object-common.go
Generated Vendored
+7 -7
View File
@@ -31,7 +31,7 @@ const nullVersionID = "null"
// Verify if reader is *minio.Object
func isObject(reader io.Reader) (ok bool) {
_, ok = reader.(*Object)
return
return ok
}
// Verify if reader is a generic ReaderAt
@@ -56,7 +56,7 @@ func isReadAt(reader io.Reader) (ok bool) {
} else {
_, ok = reader.(io.ReaderAt)
}
return
return ok
}
// OptimalPartInfo - calculate the optimal part info for a given
@@ -79,31 +79,31 @@ func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCou
// object size is larger than supported maximum.
if objectSize > maxMultipartPutObjectSize {
err = errEntityTooLarge(objectSize, maxMultipartPutObjectSize, "", "")
return
return totalPartsCount, partSize, lastPartSize, err
}
var partSizeFlt float64
if configuredPartSize > 0 {
if int64(configuredPartSize) > objectSize {
err = errEntityTooLarge(int64(configuredPartSize), objectSize, "", "")
return
return totalPartsCount, partSize, lastPartSize, err
}
if !unknownSize {
if objectSize > (int64(configuredPartSize) * maxPartsCount) {
err = errInvalidArgument("Part size * max_parts(10000) is lesser than input objectSize.")
return
return totalPartsCount, partSize, lastPartSize, err
}
}
if configuredPartSize < absMinPartSize {
err = errInvalidArgument("Input part size is smaller than allowed minimum of 5MiB.")
return
return totalPartsCount, partSize, lastPartSize, err
}
if configuredPartSize > maxPartSize {
err = errInvalidArgument("Input part size is bigger than allowed maximum of 5GiB.")
return
return totalPartsCount, partSize, lastPartSize, err
}
partSizeFlt = float64(configuredPartSize)
vendor/github.com/minio/minio-go/v7/api-put-object-fan-out.go
Generated Vendored
+1 -1
View File
@@ -19,6 +19,7 @@ package minio
import (
"context"
"encoding/json"
"errors"
"io"
"mime/multipart"
@@ -27,7 +28,6 @@ import (
"strings"
"time"
"github.com/minio/minio-go/v7/internal/json"
"github.com/minio/minio-go/v7/pkg/encrypt"
)
vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go
Generated Vendored
+9 -2
View File
@@ -210,9 +210,13 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
}
objPart, err := c.uploadPart(ctx, p)
if err != nil {
uploadedPartsCh <- uploadedPartRes{
select {
case <-ctx.Done():
case uploadedPartsCh <- uploadedPartRes{
Error: err,
}:
}
// Exit the goroutine.
return
}
@@ -221,10 +225,13 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
uploadReq.Part = objPart
// Send successful part info through the channel.
uploadedPartsCh <- uploadedPartRes{
select {
case <-ctx.Done():
case uploadedPartsCh <- uploadedPartRes{
Size: objPart.Size,
PartNum: uploadReq.PartNum,
Part: uploadReq.Part,
}:
}
}
}(partSize)
vendor/github.com/minio/minio-go/v7/api-put-object.go
Generated Vendored
+2 -2
View File
@@ -150,7 +150,7 @@ func (opts PutObjectOptions) getNumThreads() (numThreads int) {
} else {
numThreads = totalWorkers
}
return
return numThreads
}
// Header - constructs the headers from metadata entered by user in
@@ -249,7 +249,7 @@ func (opts PutObjectOptions) Header() (header http.Header) {
header[k] = v
}
return
return header
}
// validate() checks if the UserMetadata map has standard headers or and raises an error if so.
vendor/github.com/minio/minio-go/v7/api-remove.go
Generated Vendored
+4 -5
View File
@@ -617,12 +617,11 @@ func (c *Client) removeObjects(ctx context.Context, bucketName string, objectsCh
customHeader: headers,
expect200OKWithError: true,
})
if resp != nil {
if resp.StatusCode != http.StatusOK {
e := httpRespToErrorResponse(resp, bucketName, "")
resultCh <- RemoveObjectResult{ObjectName: "", Err: e}
}
if resp != nil && resp.StatusCode != http.StatusOK {
err = httpRespToErrorResponse(resp, bucketName, "")
}
if err != nil {
for _, b := range batch {
resultCh <- RemoveObjectResult{
vendor/github.com/minio/minio-go/v7/api.go
Generated Vendored
+51 -66
View File
@@ -21,7 +21,6 @@ import (
"bytes"
"context"
"encoding/base64"
"encoding/xml"
"errors"
"fmt"
"io"
@@ -43,7 +42,6 @@ import (
md5simd "github.com/minio/md5-simd"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/kvcache"
"github.com/minio/minio-go/v7/pkg/peeker"
"github.com/minio/minio-go/v7/pkg/s3utils"
"github.com/minio/minio-go/v7/pkg/signer"
"github.com/minio/minio-go/v7/pkg/singleflight"
@@ -162,7 +160,7 @@ type Options struct {
// Global constants.
const (
libraryName = "minio-go"
libraryVersion = "v7.0.95"
libraryVersion = "v7.0.96"
)
// User Agent should always following the below style.
@@ -204,7 +202,9 @@ func New(endpoint string, opts *Options) (*Client, error) {
return clnt, nil
}
// EndpointURL returns the URL of the S3 endpoint.
// EndpointURL returns the URL of the S3-compatible endpoint that this client connects to.
//
// Returns a copy of the endpoint URL to prevent modification of internal state.
func (c *Client) EndpointURL() *url.URL {
endpoint := *c.endpointURL // copy to prevent callers from modifying internal state
return &endpoint
@@ -221,7 +221,7 @@ func (r *lockedRandSource) Int63() (n int64) {
r.lk.Lock()
n = r.src.Int63()
r.lk.Unlock()
return
return n
}
// Seed uses the provided seed value to initialize the generator to a
@@ -325,7 +325,14 @@ func privateNew(endpoint string, opts *Options) (*Client, error) {
return clnt, nil
}
// SetAppInfo - add application details to user agent.
// SetAppInfo adds custom application name and version to the User-Agent header for all requests.
// This helps identify your application in server logs and metrics.
//
// Parameters:
// - appName: Name of the application
// - appVersion: Version of the application
//
// Both parameters must be non-empty for the custom User-Agent to be set.
func (c *Client) SetAppInfo(appName, appVersion string) {
// if app name and version not set, we do not set a new user agent.
if appName != "" && appVersion != "" {
@@ -334,7 +341,11 @@ func (c *Client) SetAppInfo(appName, appVersion string) {
}
}
// TraceOn - enable HTTP tracing.
// TraceOn enables HTTP request and response tracing for debugging purposes.
// All HTTP traffic will be written to the provided output stream.
//
// Parameters:
// - outputStream: Writer where trace output will be written (defaults to os.Stdout if nil)
func (c *Client) TraceOn(outputStream io.Writer) {
// if outputStream is nil then default to os.Stdout.
if outputStream == nil {
@@ -347,19 +358,23 @@ func (c *Client) TraceOn(outputStream io.Writer) {
c.isTraceEnabled = true
}
// TraceErrorsOnlyOn - same as TraceOn, but only errors will be traced.
// TraceErrorsOnlyOn enables HTTP tracing but only for requests that result in errors.
// This is useful for debugging without the overhead of tracing all requests.
//
// Parameters:
// - outputStream: Writer where trace output will be written (defaults to os.Stdout if nil)
func (c *Client) TraceErrorsOnlyOn(outputStream io.Writer) {
c.TraceOn(outputStream)
c.traceErrorsOnly = true
}
// TraceErrorsOnlyOff - Turns off the errors only tracing and everything will be traced after this call.
// If all tracing needs to be turned off, call TraceOff().
// TraceErrorsOnlyOff disables errors-only mode and traces all requests.
// To disable all tracing, call TraceOff() instead.
func (c *Client) TraceErrorsOnlyOff() {
c.traceErrorsOnly = false
}
// TraceOff - disable HTTP tracing.
// TraceOff disables all HTTP tracing (both normal and errors-only modes).
func (c *Client) TraceOff() {
// Disable tracing.
c.isTraceEnabled = false
@@ -620,33 +635,11 @@ func (c *Client) do(req *http.Request) (resp *http.Response, err error) {
return resp, nil
}
// Peek resp.Body looking for S3 XMl error response:
// - Return the error XML bytes if an error is found
// - Make sure to always restablish the whole http response stream before returning
func tryParseErrRespFromBody(resp *http.Response) ([]byte, error) {
peeker := peeker.NewPeekReadCloser(resp.Body, 5*humanize.MiByte)
defer func() {
peeker.ReplayFromStart()
resp.Body = peeker
}()
errResp := ErrorResponse{}
errBytes, err := xmlDecodeAndBody(peeker, &errResp)
if err != nil {
var unmarshalErr xml.UnmarshalError
if errors.As(err, &unmarshalErr) {
return nil, nil
}
return nil, err
}
return errBytes, nil
}
// List of success status.
var successStatus = []int{
http.StatusOK,
http.StatusNoContent,
http.StatusPartialContent,
var successStatus = map[int]struct{}{
http.StatusOK: {},
http.StatusNoContent: {},
http.StatusPartialContent: {},
}
// executeMethod - instantiates a given method, and retries the
@@ -729,29 +722,15 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
return nil, err
}
var success bool
var errBodyBytes []byte
for _, httpStatus := range successStatus {
if httpStatus == res.StatusCode {
success = true
break
}
}
if success {
if !metadata.expect200OKWithError {
return res, nil
}
errBodyBytes, err = tryParseErrRespFromBody(res)
if err == nil && len(errBodyBytes) == 0 {
// No S3 XML error is found
return res, nil
}
} else {
errBodyBytes, err = io.ReadAll(res.Body)
}
_, success := successStatus[res.StatusCode]
if success && !metadata.expect200OKWithError {
// We do not expect 2xx to return an error return.
return res, nil
} // in all other situations we must first parse the body as ErrorResponse
// 5MiB is sufficiently large enough to hold any error or regular XML response.
var bodyBytes []byte
bodyBytes, err = io.ReadAll(io.LimitReader(res.Body, 5*humanize.MiByte))
// By now, res.Body should be closed
closeResponse(res)
if err != nil {
@@ -759,16 +738,22 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ
}
// Save the body.
errBodySeeker := bytes.NewReader(errBodyBytes)
res.Body = io.NopCloser(errBodySeeker)
bodySeeker := bytes.NewReader(bodyBytes)
res.Body = io.NopCloser(bodySeeker)
// For errors verify if its retryable otherwise fail quickly.
errResponse := ToErrorResponse(httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName))
err = errResponse
apiErr := httpRespToErrorResponse(res, metadata.bucketName, metadata.objectName)
// Save the body back again.
errBodySeeker.Seek(0, 0) // Seek back to starting point.
res.Body = io.NopCloser(errBodySeeker)
bodySeeker.Seek(0, 0) // Seek back to starting point.
res.Body = io.NopCloser(bodySeeker)
if apiErr == nil {
return res, nil
}
// For errors verify if its retryable otherwise fail quickly.
errResponse := ToErrorResponse(apiErr)
err = errResponse
// Bucket region if set in error response and the error
// code dictates invalid region, we can retry the request
vendor/github.com/minio/minio-go/v7/bucket-cache.go
Generated Vendored
+1 -1
View File
@@ -142,7 +142,7 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
if h, p, err := net.SplitHostPort(targetURL.Host); err == nil {
if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" {
targetURL.Host = h
if ip := net.ParseIP(h); ip != nil && ip.To16() != nil {
if ip := net.ParseIP(h); ip != nil && ip.To4() == nil {
targetURL.Host = "[" + h + "]"
}
}
vendor/github.com/minio/minio-go/v7/checksum.go
Generated Vendored
+1 -1
View File
@@ -24,13 +24,13 @@ import (
"encoding/binary"
"errors"
"hash"
"hash/crc32"
"io"
"math/bits"
"net/http"
"sort"
"strings"
"github.com/klauspost/crc32"
"github.com/minio/crc64nvme"
)
vendor/github.com/minio/minio-go/v7/code_of_conduct.md
Generated Vendored
+33 -61
View File
@@ -1,80 +1,52 @@
# Contributor Covenant Code of Conduct
Contributor Covenant Code of Conduct
====================================
## Our Pledge
Our Pledge
----------
In the interest of fostering an open and welcoming environment, we as
contributors and maintainers pledge to making participation in our project and
our community a harassment-free experience for everyone, regardless of age, body
size, disability, ethnicity, gender identity and expression, level of experience,
nationality, personal appearance, race, religion, or sexual identity and
orientation.
In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, nationality, personal appearance, race, religion, or sexual identity and orientation.
## Our Standards
Our Standards
-------------
Examples of behavior that contributes to creating a positive environment
include:
Examples of behavior that contributes to creating a positive environment include:
* Using welcoming and inclusive language
* Being respectful of differing viewpoints and experiences
* Gracefully accepting constructive criticism
* Focusing on what is best for the community
* Showing empathy towards other community members
- Using welcoming and inclusive language
- Being respectful of differing viewpoints and experiences
- Gracefully accepting constructive criticism
- Focusing on what is best for the community
- Showing empathy towards other community members
Examples of unacceptable behavior by participants include:
* The use of sexualized language or imagery and unwelcome sexual attention or
advances
* Trolling, insulting/derogatory comments, and personal or political attacks
* Public or private harassment
* Publishing others' private information, such as a physical or electronic
address, without explicit permission
* Other conduct which could reasonably be considered inappropriate in a
professional setting
- The use of sexualized language or imagery and unwelcome sexual attention or advances
- Trolling, insulting/derogatory comments, and personal or political attacks
- Public or private harassment
- Publishing others' private information, such as a physical or electronic address, without explicit permission
- Other conduct which could reasonably be considered inappropriate in a professional setting
## Our Responsibilities
Our Responsibilities
--------------------
Project maintainers are responsible for clarifying the standards of acceptable
behavior and are expected to take appropriate and fair corrective action in
response to any instances of unacceptable behavior, in compliance with the
licensing terms applying to the Project developments.
Project maintainers are responsible for clarifying the standards of acceptable behavior and are expected to take appropriate and fair corrective action in response to any instances of unacceptable behavior, in compliance with the licensing terms applying to the Project developments.
Project maintainers have the right and responsibility to remove, edit, or
reject comments, commits, code, wiki edits, issues, and other contributions
that are not aligned to this Code of Conduct, or to ban temporarily or
permanently any contributor for other behaviors that they deem inappropriate,
threatening, offensive, or harmful. However, these actions shall respect the
licensing terms of the Project Developments that will always supersede such
Code of Conduct.
Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct, or to ban temporarily or permanently any contributor for other behaviors that they deem inappropriate, threatening, offensive, or harmful. However, these actions shall respect the licensing terms of the Project Developments that will always supersede such Code of Conduct.
## Scope
Scope
-----
This Code of Conduct applies both within project spaces and in public spaces
when an individual is representing the project or its community. Examples of
representing a project or community include using an official project e-mail
address, posting via an official social media account, or acting as an appointed
representative at an online or offline event. Representation of a project may be
further defined and clarified by project maintainers.
This Code of Conduct applies both within project spaces and in public spaces when an individual is representing the project or its community. Examples of representing a project or community include using an official project e-mail address, posting via an official social media account, or acting as an appointed representative at an online or offline event. Representation of a project may be further defined and clarified by project maintainers.
## Enforcement
Enforcement
-----------
Instances of abusive, harassing, or otherwise unacceptable behavior may be
reported by contacting the project team at dev@min.io. The project team
will review and investigate all complaints, and will respond in a way that it deems
appropriate to the circumstances. The project team is obligated to maintain
confidentiality with regard to the reporter of an incident.
Further details of specific enforcement policies may be posted separately.
Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by contacting the project team at dev@min.io. The project team will review and investigate all complaints, and will respond in a way that it deems appropriate to the circumstances. The project team is obligated to maintain confidentiality with regard to the reporter of an incident. Further details of specific enforcement policies may be posted separately.
Project maintainers who do not follow or enforce the Code of Conduct in good
faith may face temporary or permanent repercussions as determined by other
members of the project's leadership.
Project maintainers who do not follow or enforce the Code of Conduct in good faith may face temporary or permanent repercussions as determined by other members of the project's leadership.
## Attribution
Attribution
-----------
This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4,
available at [http://contributor-covenant.org/version/1/4][version]
This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.4, available at [http://contributor-covenant.org/version/1/4](http://contributor-covenant.org/version/1/4/)
This version includes a clarification to ensure that the code of conduct is in
compliance with the free software licensing terms of the project.
[homepage]: http://contributor-covenant.org
[version]: http://contributor-covenant.org/version/1/4/
This version includes a clarification to ensure that the code of conduct is in compliance with the free software licensing terms of the project.
vendor/github.com/minio/minio-go/v7/create-session.go
Generated Vendored
+1 -1
View File
@@ -114,7 +114,7 @@ func (c *Client) createSessionRequest(ctx context.Context, bucketName string, se
if h, p, err := net.SplitHostPort(host); err == nil {
if targetURL.Scheme == "http" && p == "80" || targetURL.Scheme == "https" && p == "443" {
host = h
if ip := net.ParseIP(h); ip != nil && ip.To16() != nil {
if ip := net.ParseIP(h); ip != nil && ip.To4() == nil {
host = "[" + h + "]"
}
}
vendor/github.com/minio/minio-go/v7/endpoints.go
Generated Vendored
+4
View File
@@ -240,6 +240,10 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
"s3.mx-central-1.amazonaws.com",
"s3.dualstack.mx-central-1.amazonaws.com",
},
"ap-east-2": {
"s3.ap-east-2.amazonaws.com",
"s3.dualstack.ap-east-2.amazonaws.com",
},
}
// getS3ExpressEndpoint get Amazon S3 Express endpoing based on the region
vendor/github.com/minio/minio-go/v7/functional_tests.go
Generated Vendored
+502 -58
View File
@@ -88,7 +88,7 @@ func createHTTPTransport() (transport *http.Transport) {
transport.TLSClientConfig.InsecureSkipVerify = true
}
return
return transport
}
var readFull = func(r io.Reader, buf []byte) (n int, err error) {
@@ -123,7 +123,7 @@ var readFull = func(r io.Reader, buf []byte) (n int, err error) {
} else if n > 0 && err == io.EOF {
err = io.ErrUnexpectedEOF
}
return
return n, err
}
func baseLogger(testName, function string, args map[string]interface{}, startTime time.Time) *slog.Logger {
@@ -282,7 +282,7 @@ var mintDataDir = os.Getenv("MINT_DATA_DIR")
func getMintDataDirFilePath(filename string) (fp string) {
if mintDataDir == "" {
return
return fp
}
return filepath.Join(mintDataDir, filename)
}
@@ -2075,6 +2075,9 @@ func testPutObjectWithChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@@ -2095,6 +2098,9 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
if st.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@@ -2435,7 +2441,10 @@ func testPutMultipartObjectWithChecksums() {
reader.Close()
h := test.cs.Hasher()
h.Reset()
want := hashMultiPart(b, partSize, test.cs)
// wantChksm might be the full object checksum or the multipart checksum, depending on the test.cs type.
wantChksm := hashMultiPart(b, partSize, test.cs)
// wantFullObjectChksm is always the full object checksum that is returned after CopyObject.
wantFullObjectChksm := hashMultiPart(b, len(b), test.cs)
rd := bytes.NewReader(b)
cs := test.cs
@@ -2456,15 +2465,15 @@ func testPutMultipartObjectWithChecksums() {
switch test.cs.Base() {
case minio.ChecksumCRC32C:
cmpChecksum(resp.ChecksumCRC32C, want)
cmpChecksum(resp.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
cmpChecksum(resp.ChecksumCRC32, want)
cmpChecksum(resp.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
cmpChecksum(resp.ChecksumSHA1, want)
cmpChecksum(resp.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(resp.ChecksumSHA256, want)
cmpChecksum(resp.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(resp.ChecksumCRC64NVME, want)
cmpChecksum(resp.ChecksumCRC64NVME, wantChksm)
}
args["section"] = "HeadObject"
@@ -2475,15 +2484,51 @@ func testPutMultipartObjectWithChecksums() {
}
switch test.cs.Base() {
case minio.ChecksumCRC32C:
cmpChecksum(st.ChecksumCRC32C, want)
cmpChecksum(st.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
cmpChecksum(st.ChecksumCRC32, want)
cmpChecksum(st.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
cmpChecksum(st.ChecksumSHA1, want)
cmpChecksum(st.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(st.ChecksumSHA256, want)
cmpChecksum(st.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, want)
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
}
// Use the CopyObject API to make a copy, in the case it was a composite checksum,
// it will change because the copy is no longer a multipart object. S3 returns the checksum
// of the full object when HeadObject is called on the copy.
args["section"] = "CopyObject"
objectCopyName := objectName + "-copy"
_, err = c.CopyObject(context.Background(), minio.CopyDestOptions{
Bucket: bucketName,
Object: objectCopyName,
}, minio.CopySrcOptions{
Bucket: bucketName,
Object: objectName,
})
if err != nil {
logError(testName, function, args, startTime, "", "CopyObject failed", err)
return
}
args["section"] = "HeadObject-Copy"
st, err = c.StatObject(context.Background(), bucketName, objectCopyName, minio.StatObjectOptions{Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "StatObject failed", err)
return
}
switch test.cs.Base() {
case minio.ChecksumCRC32C:
cmpChecksum(st.ChecksumCRC32C, wantFullObjectChksm)
case minio.ChecksumCRC32:
cmpChecksum(st.ChecksumCRC32, wantFullObjectChksm)
case minio.ChecksumSHA1:
cmpChecksum(st.ChecksumSHA1, wantFullObjectChksm)
case minio.ChecksumSHA256:
cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm)
}
args["section"] = "GetObjectAttributes"
@@ -2493,19 +2538,19 @@ func testPutMultipartObjectWithChecksums() {
return
}
if strings.ContainsRune(want, '-') {
want = want[:strings.IndexByte(want, '-')]
if strings.ContainsRune(wantChksm, '-') {
wantChksm = wantChksm[:strings.IndexByte(wantChksm, '-')]
}
switch test.cs {
// Full Object CRC does not return anything with GetObjectAttributes
case minio.ChecksumCRC32C:
cmpChecksum(s.Checksum.ChecksumCRC32C, want)
cmpChecksum(s.Checksum.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
cmpChecksum(s.Checksum.ChecksumCRC32, want)
cmpChecksum(s.Checksum.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
cmpChecksum(s.Checksum.ChecksumSHA1, want)
cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(s.Checksum.ChecksumSHA256, want)
cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm)
}
// Read the data back
@@ -2529,22 +2574,22 @@ func testPutMultipartObjectWithChecksums() {
// Test part 2 checksum...
h.Reset()
h.Write(b[partSize : 2*partSize])
want = base64.StdEncoding.EncodeToString(h.Sum(nil))
wantChksm = base64.StdEncoding.EncodeToString(h.Sum(nil))
switch test.cs {
// Full Object CRC does not return any part CRC for whatever reason.
case minio.ChecksumCRC32C:
cmpChecksum(st.ChecksumCRC32C, want)
cmpChecksum(st.ChecksumCRC32C, wantChksm)
case minio.ChecksumCRC32:
cmpChecksum(st.ChecksumCRC32, want)
cmpChecksum(st.ChecksumCRC32, wantChksm)
case minio.ChecksumSHA1:
cmpChecksum(st.ChecksumSHA1, want)
cmpChecksum(st.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(st.ChecksumSHA256, want)
cmpChecksum(st.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
// AWS doesn't return part checksum, but may in the future.
if st.ChecksumCRC64NVME != "" {
cmpChecksum(st.ChecksumCRC64NVME, want)
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
}
}
@@ -3321,7 +3366,7 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if opts.VersionID != "" {
if OA.VersionID != opts.VersionID {
err = fmt.Errorf("Expected versionId %s but got versionId %s", opts.VersionID, OA.VersionID)
return
return err
}
}
@@ -3348,12 +3393,12 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if test.HasPartChecksums {
if partsMissingChecksum {
err = fmt.Errorf("One or all parts were missing a checksum")
return
return err
}
} else {
if foundPartChecksum {
err = fmt.Errorf("Did not expect ObjectParts to have checksums but found one")
return
return err
}
}
@@ -3365,52 +3410,52 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if test.HasFullChecksum {
if !hasFullObjectChecksum {
err = fmt.Errorf("Full object checksum not found")
return
return err
}
} else {
if hasFullObjectChecksum {
err = fmt.Errorf("Did not expect a full object checksum but we got one")
return
return err
}
}
if OA.ETag != test.ETag {
err = fmt.Errorf("Etags do not match, got %s but expected %s", OA.ETag, test.ETag)
return
return err
}
if test.HasParts {
if len(OA.ObjectParts.Parts) < 1 {
err = fmt.Errorf("Was expecting ObjectParts but none were present")
return
return err
}
}
if OA.StorageClass == "" {
err = fmt.Errorf("Was expecting a StorageClass but got none")
return
return err
}
if OA.ObjectSize != test.ObjectSize {
err = fmt.Errorf("Was expecting a ObjectSize but got none")
return
return err
}
if test.HasParts {
if opts.MaxParts == 0 {
if len(OA.ObjectParts.Parts) != OA.ObjectParts.PartsCount {
err = fmt.Errorf("expected %s parts but got %d", OA.ObjectParts.PartsCount, len(OA.ObjectParts.Parts))
return
return err
}
} else if (opts.MaxParts + opts.PartNumberMarker) > OA.ObjectParts.PartsCount {
if len(OA.ObjectParts.Parts) != (OA.ObjectParts.PartsCount - opts.PartNumberMarker) {
err = fmt.Errorf("expected %d parts but got %d", (OA.ObjectParts.PartsCount - opts.PartNumberMarker), len(OA.ObjectParts.Parts))
return
return err
}
} else if opts.MaxParts != 0 {
if opts.MaxParts != len(OA.ObjectParts.Parts) {
err = fmt.Errorf("expected %d parts but got %d", opts.MaxParts, len(OA.ObjectParts.Parts))
return
return err
}
}
}
@@ -3418,18 +3463,18 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
if OA.ObjectParts.NextPartNumberMarker == OA.ObjectParts.PartsCount {
if OA.ObjectParts.IsTruncated {
err = fmt.Errorf("Expected ObjectParts to NOT be truncated, but it was")
return
return err
}
}
if OA.ObjectParts.NextPartNumberMarker != OA.ObjectParts.PartsCount {
if !OA.ObjectParts.IsTruncated {
err = fmt.Errorf("Expected ObjectParts to be truncated, but it was NOT")
return
return err
}
}
return
return err
}
// Test PutObject using a large data to trigger multipart readat
@@ -3645,6 +3690,93 @@ func testPutObjectStreaming() {
logSuccess(testName, function, args, startTime)
}
// Test PutObject with preconditions on non-existent objects
func testPutObjectPreconditionOnNonExistent() {
startTime := time.Now()
testName := getFuncName()
function := "PutObject(bucketName, objectName, reader, size, opts) with preconditions"
args := map[string]interface{}{
"bucketName": "",
"objectName": "",
"opts": "minio.PutObjectOptions{SetMatchETag/SetMatchETagExcept}",
}
c, err := NewClient(ClientConfig{})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO client object creation failed", err)
return
}
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
args["bucketName"] = bucketName
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
// Test 1: PutObject with SetMatchETag on non-existent object should fail
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object-")
args["objectName"] = objectName
data := bytes.NewReader([]byte("test data"))
opts := minio.PutObjectOptions{}
opts.SetMatchETag("some-etag")
_, err = c.PutObject(context.Background(), bucketName, objectName, data, int64(data.Len()), opts)
if err == nil {
logError(testName, function, args, startTime, "", "PutObject with SetMatchETag on non-existent object should have failed", nil)
return
}
errResp := minio.ToErrorResponse(err)
if errResp.Code != "NoSuchKey" {
logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects), got %s", errResp.Code), err)
return
}
// Test 2: PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should succeed
objectName2 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-object2-")
args["objectName"] = objectName2
data2 := bytes.NewReader([]byte("test data 2"))
opts2 := minio.PutObjectOptions{}
opts2.SetMatchETagExcept("some-etag")
_, err = c.PutObject(context.Background(), bucketName, objectName2, data2, int64(data2.Len()), opts2)
if err != nil {
logError(testName, function, args, startTime, "", "PutObject with SetMatchETagExcept (If-None-Match) on non-existent object should have succeeded", err)
return
}
// Test 3: CompleteMultipartUpload with preconditions on non-existent object should fail
objectName3 := randString(60, rand.NewSource(time.Now().UnixNano()), "test-multipart-")
args["objectName"] = objectName3
data3 := bytes.Repeat([]byte("a"), 5*1024*1024+1)
reader3 := bytes.NewReader(data3)
opts3 := minio.PutObjectOptions{}
opts3.SetMatchETag("non-existent-etag")
_, err = c.PutObject(context.Background(), bucketName, objectName3, reader3, int64(len(data3)), opts3)
if err == nil {
logError(testName, function, args, startTime, "", "CompleteMultipartUpload with SetMatchETag on non-existent object should have failed", nil)
return
}
errResp = minio.ToErrorResponse(err)
if errResp.Code != "NoSuchKey" {
logError(testName, function, args, startTime, "", fmt.Sprintf("Expected NoSuchKey error (AWS standard for non-existent objects) for multipart, got %s", errResp.Code), err)
return
}
logSuccess(testName, function, args, startTime)
}
// Test get object seeker from the end, using whence set to '2'.
func testGetObjectSeekEnd() {
// initialize logging params
@@ -8653,6 +8785,7 @@ func testCopyObjectV2() {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
// Check the various fields of source object against destination object.
objInfo, err = r.Stat()
if err != nil {
@@ -8691,6 +8824,260 @@ func testCopyObjectV2() {
logSuccess(testName, function, args, startTime)
}
// Tests copy object with various checksum scenarios, tries to not repeat CopyObjectV2 test and
// instead just focus on Checksum.
func testCopyObjectWithChecksums() {
startTime := time.Now()
testName := getFuncName()
function := "CopyObjectWithChecksums(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{CredsV2: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
}
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
// Make a new bucket in 'us-east-1' (source bucket).
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
// Make a new bucket in 'us-east-1' (destination bucket).
err = c.MakeBucket(context.Background(), bucketName+"-copy", minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
defer cleanupBucket(bucketName+"-copy", c)
// Generate 33K of data.
bufSize := dataFileMap["datafile-33-kB"]
reader := getDataReader("datafile-33-kB")
defer reader.Close()
// PutObject to upload the object to the bucket, this object will have a Crc64NVME checksum applied
// by default since nothing was explicitly specified.
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
// GetObject to obtain the eTag
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
objInfo, err := r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
}
r.Close()
// Copy source options
src := minio.CopySrcOptions{
Bucket: bucketName,
Object: objectName,
MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
MatchETag: objInfo.ETag,
}
tests := []struct {
csType minio.ChecksumType
cs wantChecksums
}{
{csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
{csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}},
{csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}},
{csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}},
{csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}},
// In S3, all copied objects without checksums and specified destination checksum algorithms
// automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case.
{csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
}
for _, test := range tests {
args := map[string]interface{}{}
args["srcOpts"] = src
args["section"] = "setup"
args["checksum"] = test.csType.String()
// Copy destination options
bucketCopyName := bucketName + "-copy"
objectCopyName := objectName + "-copy-" + test.csType.String()
dst := minio.CopyDestOptions{
Bucket: bucketCopyName,
Object: objectCopyName,
ReplaceMetadata: true,
}
if test.csType != minio.ChecksumNone {
// Request the server-side checksum on the copy.
// ChecksumNone is a flag to leave off the header
dst.ChecksumType = test.csType
}
args["destOpts"] = dst
// Perform the Copy
args["section"] = "CopyObject"
_, err = c.CopyObject(context.Background(), dst, src)
if err != nil {
logError(testName, function, args, startTime, "", "CopyObject failed", err)
return
}
// Checksum verification
args["section"] = "HeadObject"
st, err := c.StatObject(context.Background(), bucketCopyName, objectCopyName, minio.StatObjectOptions{Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "StatObject failed", err)
return
}
if st.ChecksumMode != "FULL_OBJECT" {
logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil)
return
}
err = cmpChecksum(st, test.cs)
if err != nil {
logError(testName, function, args, startTime, "", "Checksum mismatch", err)
return
}
logSuccess(testName, function, args, startTime)
}
}
// Tests replacing an object with CopyObject and a new Checksum type
func testReplaceObjectWithChecksums() {
startTime := time.Now()
testName := getFuncName()
function := "CopyObjectWithChecksums(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{CredsV2: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
}
// Generate a new random bucket name.
bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-")
// Make a new bucket in 'us-east-1' (source bucket).
err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"})
if err != nil {
logError(testName, function, args, startTime, "", "MakeBucket failed", err)
return
}
defer cleanupBucket(bucketName, c)
tests := []struct {
csType minio.ChecksumType
cs wantChecksums
}{
{csType: minio.ChecksumCRC64NVME, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
{csType: minio.ChecksumCRC32C, cs: wantChecksums{minio.ChecksumCRC32C: "aHnJMw=="}},
{csType: minio.ChecksumCRC32, cs: wantChecksums{minio.ChecksumCRC32: "tIZ8hA=="}},
{csType: minio.ChecksumSHA1, cs: wantChecksums{minio.ChecksumSHA1: "6YIIbcWH1iLaCFqs5vwq5Rwvm+o="}},
{csType: minio.ChecksumSHA256, cs: wantChecksums{minio.ChecksumSHA256: "GKeJTopbMGPs3h4fAw4oe0R2QnnmFVJeIWkqCkp28Yo="}},
// In S3, all copied objects without checksums and specified destination checksum algorithms
// automatically gain a CRC-64NVME checksum algorithm. Use ChecksumNone for this case.
{csType: minio.ChecksumNone, cs: wantChecksums{minio.ChecksumCRC64NVME: "iRtfQH3xflQ="}},
}
for _, test := range tests {
args := map[string]interface{}{}
args["section"] = "setup"
args["destOpts"] = ""
args["checksum"] = test.csType.String()
bufSize := dataFileMap["datafile-33-kB"]
reader := getDataReader("datafile-33-kB")
defer reader.Close()
// PutObject to upload the object to the bucket
objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "")
_, err = c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ContentType: "binary/octet-stream"})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
// GetObject to obtain the eTag
r, err := c.GetObject(context.Background(), bucketName, objectName, minio.GetObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
}
objInfo, err := r.Stat()
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
}
r.Close()
// Copy source options
src := minio.CopySrcOptions{
Bucket: bucketName,
Object: objectName,
MatchModifiedSince: time.Date(2014, time.April, 0, 0, 0, 0, 0, time.UTC),
MatchETag: objInfo.ETag,
}
// Copy destination options, overwrite the existing object
dst := minio.CopyDestOptions{
Bucket: bucketName,
Object: objectName,
// S3 requires that we send some new metadata otherwise it complains that the
// CopyObject is illegal.
UserMetadata: map[string]string{
"TestMeta": objectName + "-meta-" + test.csType.String(),
},
ReplaceMetadata: true,
}
if test.csType != minio.ChecksumNone {
// Request the server-side checksum on the copy.
// ChecksumNone is a flag to leave off the header
dst.ChecksumType = test.csType
}
args["destOpts"] = dst
// Perform the Copy
args["section"] = "CopyObject"
_, err = c.CopyObject(context.Background(), dst, src)
if err != nil {
logError(testName, function, args, startTime, "", "CopyObject failed", err)
return
}
// Checksum verification
args["section"] = "HeadObject"
st, err := c.StatObject(context.Background(), bucketName, objectName, minio.StatObjectOptions{Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "StatObject failed", err)
return
}
if st.ChecksumMode != "FULL_OBJECT" {
logError(testName, function, args, startTime, "", "ChecksumMode want: FULL_OBJECT, got "+st.ChecksumMode, nil)
return
}
err = cmpChecksum(st, test.cs)
if err != nil {
logError(testName, function, args, startTime, "", "Checksum mismatch", err)
return
}
logSuccess(testName, function, args, startTime)
}
}
func testComposeObjectErrorCasesWrapper(c *minio.Client) {
// initialize logging params
startTime := time.Now()
@@ -8976,6 +9363,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
testName := getFuncNameLoc(2)
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
args["testName"] = testName
var srcEncryption, dstEncryption encrypt.ServerSide
// Make a new bucket in 'us-east-1' (source bucket).
@@ -8990,8 +9378,19 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
// 1. create an sse-c encrypted object to copy by uploading
const srcSize = 1024 * 1024
buf := bytes.Repeat([]byte("abcde"), srcSize) // gives a buffer of 5MiB
// Calculate the CRC32C checksum for the object
meta := map[string]string{}
h := minio.ChecksumCRC32C.Hasher()
h.Reset()
h.Write(buf)
meta[minio.ChecksumCRC32C.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil))
_, err = c.PutObject(context.Background(), bucketName, "srcObject", bytes.NewReader(buf), int64(len(buf)), minio.PutObjectOptions{
ServerSideEncryption: sseSrc,
DisableMultipart: true,
DisableContentSha256: true,
UserMetadata: meta,
})
if err != nil {
logError(testName, function, args, startTime, "", "PutObject call failed", err)
@@ -9028,7 +9427,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
// 3. get copied object and check if content is equal
coreClient := minio.Core{Client: c}
reader, _, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption})
reader, oi, _, err := coreClient.GetObject(context.Background(), bucketName, "dstObject", minio.GetObjectOptions{ServerSideEncryption: dstEncryption, Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
@@ -9045,6 +9444,12 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
reader.Close()
err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="})
if err != nil {
logError(testName, function, args, startTime, "", "Checksum mismatch on dstObject", err)
return
}
// Test key rotation for source object in-place.
var newSSE encrypt.ServerSide
if sseSrc != nil && sseSrc.Type() == encrypt.SSEC {
@@ -9068,7 +9473,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
// Get copied object and check if content is equal
reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE})
reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{ServerSideEncryption: newSSE, Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
@@ -9085,6 +9490,13 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
reader.Close()
err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="})
if err != nil {
fmt.Printf("srcObject objectInfo: %+v\n", oi)
logError(testName, function, args, startTime, "", "Checksum mismatch on srcObject for in-place", err)
return
}
// Test in-place decryption.
dst = minio.CopyDestOptions{
Bucket: bucketName,
@@ -9106,7 +9518,7 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
}
// Get copied decrypted object and check if content is equal
reader, _, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{})
reader, oi, _, err = coreClient.GetObject(context.Background(), bucketName, "srcObject", minio.GetObjectOptions{Checksum: true})
if err != nil {
logError(testName, function, args, startTime, "", "GetObject failed", err)
return
@@ -9123,6 +9535,12 @@ func testEncryptedCopyObjectWrapper(c *minio.Client, bucketName string, sseSrc,
return
}
err = cmpChecksum(oi, wantChecksums{minio.ChecksumCRC32C: "bSoobA=="})
if err != nil {
logError(testName, function, args, startTime, "", "Checksum mismatch for decrypted object", err)
return
}
logSuccess(testName, function, args, startTime)
}
@@ -9134,7 +9552,7 @@ func testUnencryptedToSSECCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9154,7 +9572,7 @@ func testUnencryptedToSSES3CopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9175,7 +9593,7 @@ func testUnencryptedToUnencryptedCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9195,7 +9613,7 @@ func testEncryptedSSECToSSECCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9216,7 +9634,7 @@ func testEncryptedSSECToSSES3CopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9237,7 +9655,7 @@ func testEncryptedSSECToUnencryptedCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9258,7 +9676,7 @@ func testEncryptedSSES3ToSSECCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9279,7 +9697,7 @@ func testEncryptedSSES3ToSSES3CopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9300,7 +9718,7 @@ func testEncryptedSSES3ToUnencryptedCopyObject() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{})
c, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -9321,7 +9739,7 @@ func testEncryptedCopyObjectV2() {
function := "CopyObject(destination, source)"
args := map[string]interface{}{}
c, err := NewClient(ClientConfig{CredsV2: true})
c, err := NewClient(ClientConfig{CredsV2: true, TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v2 client object creation failed", err)
return
@@ -10080,7 +10498,7 @@ func testUnencryptedToSSECCopyObjectPart() {
function := "CopyObjectPart(destination, source)"
args := map[string]interface{}{}
client, err := NewClient(ClientConfig{})
client, err := NewClient(ClientConfig{TrailingHeaders: true})
if err != nil {
logError(testName, function, args, startTime, "", "MinIO v4 client object creation failed", err)
return
@@ -11080,7 +11498,7 @@ func testUserMetadataCopyingWrapper(c *minio.Client) {
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
return h
}
h = make(http.Header)
for k, vs := range objInfo.Metadata {
@@ -11256,7 +11674,7 @@ func testStorageClassMetadataPutObject() {
objInfo, err := c.StatObject(context.Background(), bucketName, object, minio.StatObjectOptions{})
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
return h
}
h = make(http.Header)
for k, vs := range objInfo.Metadata {
@@ -11377,7 +11795,7 @@ func testStorageClassMetadataCopyObject() {
args["object"] = object
if err != nil {
logError(testName, function, args, startTime, "", "Stat failed", err)
return
return h
}
h = make(http.Header)
for k, vs := range objInfo.Metadata {
@@ -14195,6 +14613,29 @@ func mustParseBool(str string) bool {
return b
}
// wantChecksums is a map of expected checksums for an object.
type wantChecksums map[minio.ChecksumType]string
// cmpChecksum compares the checksums of an object against expected values.
func cmpChecksum(oi minio.ObjectInfo, chksums wantChecksums) error {
if oi.ChecksumCRC64NVME != chksums[minio.ChecksumCRC64NVME] {
return fmt.Errorf("Checksum mismatch for CRC64NVME, want: %s, got: %s", chksums[minio.ChecksumCRC64NVME], oi.ChecksumCRC64NVME)
}
if oi.ChecksumCRC32C != chksums[minio.ChecksumCRC32C] {
return fmt.Errorf("Checksum mismatch for CRC32C, want: %s, got: %s", chksums[minio.ChecksumCRC32C], oi.ChecksumCRC32C)
}
if oi.ChecksumCRC32 != chksums[minio.ChecksumCRC32] {
return fmt.Errorf("Checksum mismatch for CRC32, want: %s, got: %s", chksums[minio.ChecksumCRC32], oi.ChecksumCRC32)
}
if oi.ChecksumSHA1 != chksums[minio.ChecksumSHA1] {
return fmt.Errorf("Checksum mismatch for SHA1, want: %s, got: %s", chksums[minio.ChecksumSHA1], oi.ChecksumSHA1)
}
if oi.ChecksumSHA256 != chksums[minio.ChecksumSHA256] {
return fmt.Errorf("Checksum mismatch for SHA256, want: %s, got: %s", chksums[minio.ChecksumSHA256], oi.ChecksumSHA256)
}
return nil
}
func main() {
slog.SetDefault(slog.New(slog.NewJSONHandler(
os.Stdout,
@@ -14219,6 +14660,8 @@ func main() {
// execute tests
if isFullMode() {
testCopyObjectWithChecksums()
testReplaceObjectWithChecksums()
testCorsSetGetDelete()
testCors()
testListMultipartUpload()
@@ -14253,6 +14696,7 @@ func main() {
testPutObjectWithMetadata()
testPutObjectReadAt()
testPutObjectStreaming()
testPutObjectPreconditionOnNonExistent()
testGetObjectSeekEnd()
testGetObjectClosedTwice()
testGetObjectS3Zip()
vendor/github.com/minio/minio-go/v7/internal/json/json_goccy.go
Generated Vendored
-49
View File
@@ -1,49 +0,0 @@
//go:build !stdlibjson
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2025 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package json
import "github.com/goccy/go-json"
// This file defines the JSON functions used internally and forwards them
// to goccy/go-json. Alternatively, the standard library can be used by setting
// the build tag stdlibjson. This can be useful for testing purposes or if
// goccy/go-json causes issues.
//
// This file does not contain all definitions from goccy/go-json; if needed, more
// can be added, but keep in mind that json_stdlib.go will also need to be
// updated.
var (
// Unmarshal is a wrapper around goccy/go-json Unmarshal function.
Unmarshal = json.Unmarshal
// Marshal is a wrapper around goccy/go-json Marshal function.
Marshal = json.Marshal
// NewEncoder is a wrapper around goccy/go-json NewEncoder function.
NewEncoder = json.NewEncoder
// NewDecoder is a wrapper around goccy/go-json NewDecoder function.
NewDecoder = json.NewDecoder
)
type (
// Encoder is an alias for goccy/go-json Encoder.
Encoder = json.Encoder
// Decoder is an alias for goccy/go-json Decoder.
Decoder = json.Decoder
)
vendor/github.com/minio/minio-go/v7/internal/json/json_stdlib.go
Generated Vendored
-49
View File
@@ -1,49 +0,0 @@
//go:build stdlibjson
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2025 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package json
import "encoding/json"
// This file defines the JSON functions used internally and forwards them
// to encoding/json. This is only enabled by setting the build tag stdlibjson,
// otherwise json_goccy.go applies.
// This can be useful for testing purposes or if goccy/go-json (which is used otherwise) causes issues.
//
// This file does not contain all definitions from encoding/json; if needed, more
// can be added, but keep in mind that json_goccy.go will also need to be
// updated.
var (
// Unmarshal is a wrapper around encoding/json Unmarshal function.
Unmarshal = json.Unmarshal
// Marshal is a wrapper around encoding/json Marshal function.
Marshal = json.Marshal
// NewEncoder is a wrapper around encoding/json NewEncoder function.
NewEncoder = json.NewEncoder
// NewDecoder is a wrapper around encoding/json NewDecoder function.
NewDecoder = json.NewDecoder
)
type (
// Encoder is an alias for encoding/json Encoder.
Encoder = json.Encoder
// Decoder is an alias for encoding/json Decoder.
Decoder = json.Decoder
)
vendor/github.com/minio/minio-go/v7/pkg/credentials/file_aws_credentials.go
Generated Vendored
+1 -1
View File
@@ -18,6 +18,7 @@
package credentials
import (
"encoding/json"
"errors"
"os"
"os/exec"
@@ -26,7 +27,6 @@ import (
"time"
"github.com/go-ini/ini"
"github.com/minio/minio-go/v7/internal/json"
)
// A externalProcessCredentials stores the output of a credential_process
vendor/github.com/minio/minio-go/v7/pkg/credentials/file_minio_client.go
Generated Vendored
+1 -2
View File
@@ -18,11 +18,10 @@
package credentials
import (
"encoding/json"
"os"
"path/filepath"
"runtime"
"github.com/minio/minio-go/v7/internal/json"
)
// A FileMinioClient retrieves credentials from the current user's home
vendor/github.com/minio/minio-go/v7/pkg/credentials/iam_aws.go
Generated Vendored
+1 -2
View File
@@ -20,6 +20,7 @@ package credentials
import (
"bufio"
"context"
"encoding/json"
"errors"
"fmt"
"io"
@@ -30,8 +31,6 @@ import (
"path"
"strings"
"time"
"github.com/minio/minio-go/v7/internal/json"
)
// DefaultExpiryWindow - Default expiry window.
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_custom_identity.go
Generated Vendored
+1 -1
View File
@@ -132,7 +132,7 @@ func (c *CustomTokenIdentity) RetrieveWithCredContext(cc *CredContext) (value Va
r := AssumeRoleWithCustomTokenResponse{}
if err = xml.NewDecoder(resp.Body).Decode(&r); err != nil {
return
return value, err
}
cr := r.Result.Credentials
vendor/github.com/minio/minio-go/v7/pkg/credentials/sts_ldap_identity.go
Generated Vendored
+14 -1
View File
@@ -74,6 +74,9 @@ type LDAPIdentity struct {
// requested from LDAP.
RequestedExpiry time.Duration
// Optional, if empty applies to default config
ConfigName string
// Optional, used for token revokation
TokenRevokeType string
}
@@ -110,6 +113,13 @@ func LDAPIdentityExpiryOpt(d time.Duration) LDAPIdentityOpt {
}
}
// LDAPIdentityConfigNameOpt sets the config name for requested credentials.
func LDAPIdentityConfigNameOpt(name string) LDAPIdentityOpt {
return func(k *LDAPIdentity) {
k.ConfigName = name
}
}
// NewLDAPIdentityWithSessionPolicy returns new credentials object that uses
// LDAP Identity with a specified session policy. The `policy` parameter must be
// a JSON string specifying the policy document.
@@ -158,6 +168,9 @@ func (k *LDAPIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, er
if k.TokenRevokeType != "" {
v.Set("TokenRevokeType", k.TokenRevokeType)
}
if k.ConfigName != "" {
v.Set("ConfigName", k.ConfigName)
}
req, err := http.NewRequest(http.MethodPost, u.String(), strings.NewReader(v.Encode()))
if err != nil {
@@ -201,7 +214,7 @@ func (k *LDAPIdentity) RetrieveWithCredContext(cc *CredContext) (value Value, er
r := AssumeRoleWithLDAPResponse{}
if err = xml.NewDecoder(resp.Body).Decode(&r); err != nil {
return
return value, err
}
cr := r.Result.Credentials
vendor/github.com/minio/minio-go/v7/pkg/encrypt/server-side.go
Generated Vendored
+1 -1
View File
@@ -20,10 +20,10 @@ package encrypt
import (
"crypto/md5"
"encoding/base64"
"encoding/json"
"errors"
"net/http"
"github.com/minio/minio-go/v7/internal/json"
"golang.org/x/crypto/argon2"
)
vendor/github.com/minio/minio-go/v7/pkg/lifecycle/lifecycle.go
Generated Vendored
+1 -2
View File
@@ -19,11 +19,10 @@
package lifecycle
import (
"encoding/json"
"encoding/xml"
"errors"
"time"
"github.com/minio/minio-go/v7/internal/json"
)
var errMissingStorageClass = errors.New("storage-class cannot be empty")
vendor/github.com/minio/minio-go/v7/pkg/peeker/peek-reader-closer.go
Generated Vendored
-73
View File
@@ -1,73 +0,0 @@
/*
* MinIO Go Library for Amazon S3 Compatible Cloud Storage
* Copyright 2015-2025 MinIO, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package peeker
import (
"bytes"
"errors"
"io"
)
// PeekReadCloser offers a way to peek a ReadCloser stream and then
// return the exact stream of the underlying ReadCloser
type PeekReadCloser struct {
io.ReadCloser
recordMode bool
recordMaxBuf int
recordBuf *bytes.Buffer
}
// ReplayFromStart ensures next Read() will restart to stream the
// underlying ReadCloser stream from the beginning
func (prc *PeekReadCloser) ReplayFromStart() {
prc.recordMode = false
}
func (prc *PeekReadCloser) Read(p []byte) (int, error) {
if prc.recordMode {
if prc.recordBuf.Len() > prc.recordMaxBuf {
return 0, errors.New("maximum peek buffer exceeded")
}
n, err := prc.ReadCloser.Read(p)
prc.recordBuf.Write(p[:n])
return n, err
}
// Replay mode
if prc.recordBuf.Len() > 0 {
pn, _ := prc.recordBuf.Read(p)
return pn, nil
}
return prc.ReadCloser.Read(p)
}
// Close releases the record buffer memory and close the underlying ReadCloser
func (prc *PeekReadCloser) Close() error {
prc.recordBuf.Reset()
return prc.ReadCloser.Close()
}
// NewPeekReadCloser returns a new peek reader
func NewPeekReadCloser(rc io.ReadCloser, maxBufSize int) *PeekReadCloser {
return &PeekReadCloser{
ReadCloser: rc,
recordMode: true, // recording mode by default
recordBuf: bytes.NewBuffer(make([]byte, 0, 1024)),
recordMaxBuf: maxBufSize,
}
}
vendor/github.com/minio/minio-go/v7/pkg/replication/replication.go
Generated Vendored
+1 -1
View File
@@ -1014,7 +1014,7 @@ func (q ReplQueueStats) QStats() (r ReplQStats) {
if len(q.Nodes) > 0 {
r.Uptime /= int64(len(q.Nodes)) // average uptime
}
return
return r
}
// MetricsV2 represents replication metrics for a bucket.
vendor/github.com/minio/minio-go/v7/pkg/set/stringset.go
Generated Vendored
+1 -2
View File
@@ -18,10 +18,9 @@
package set
import (
"encoding/json"
"fmt"
"sort"
"github.com/minio/minio-go/v7/internal/json"
)
// StringSet - uses map as set of strings.
vendor/github.com/minio/minio-go/v7/pkg/signer/request-signature-v2.go
Generated Vendored
+2 -2
View File
@@ -48,11 +48,11 @@ func encodeURL2Path(req *http.Request, virtualHost bool) (path string) {
path = "/" + bucketName
path += req.URL.Path
path = s3utils.EncodePath(path)
return
return path
}
}
path = s3utils.EncodePath(req.URL.Path)
return
return path
}
// PreSignV2 - presign the request in following style.
vendor/github.com/minio/minio-go/v7/utils.go
Generated Vendored
+2 -2
View File
@@ -80,7 +80,7 @@ func amzRestoreToStruct(restore string) (ongoing bool, expTime time.Time, err er
return false, time.Time{}, err
}
}
return
return ongoing, expTime, err
}
// xmlDecoder provide decoded value in xml.
@@ -438,7 +438,7 @@ var readFull = func(r io.Reader, buf []byte) (n int, err error) {
} else if n > 0 && err == io.EOF {
err = io.ErrUnexpectedEOF
}
return
return n, err
}
// regCred matches credential string in HTTP header
vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go
Generated Vendored
+14
View File
@@ -727,6 +727,20 @@ func (t *Tree) isIndex(path string) bool {
}
func (t *Tree) isTemporary(path string) bool {
if filepath.IsAbs(path) {
tmpDirPattern := filepath.Join(t.options.Root, "*", "*", blobstore.TMPDir)
isTempDir, err := filepath.Match(tmpDirPattern, path)
if err != nil {
t.log.Error().Err(err).Str("pattern", tmpDirPattern).Str("path", path).Msg("error matching temporary path")
return false
}
isTempParentDir, err := filepath.Match(tmpDirPattern, filepath.Dir(path))
if err != nil {
t.log.Error().Err(err).Str("pattern", tmpDirPattern).Str("path", filepath.Dir(path)).Msg("error matching temporary path")
return false
}
return isTempDir || isTempParentDir
}
return path == blobstore.TMPDir || filepath.Dir(path) == blobstore.TMPDir
}
vendor/modules.txt
Vendored
+9 -8
View File
@@ -235,8 +235,8 @@ github.com/cenkalti/backoff/v4
# github.com/cenkalti/backoff/v5 v5.0.3
## explicit; go 1.23
github.com/cenkalti/backoff/v5
# github.com/ceph/go-ceph v0.35.0
## explicit; go 1.23.0
# github.com/ceph/go-ceph v0.36.0
## explicit; go 1.24.0
github.com/ceph/go-ceph/cephfs
github.com/ceph/go-ceph/cephfs/admin
github.com/ceph/go-ceph/common/admin/manager
@@ -719,7 +719,7 @@ github.com/golang/protobuf/ptypes/empty
# github.com/golang/snappy v0.0.4
## explicit
github.com/golang/snappy
# github.com/gomodule/redigo v1.9.2
# github.com/gomodule/redigo v1.9.3
## explicit; go 1.17
github.com/gomodule/redigo/redis
# github.com/google/go-cmp v0.7.0
@@ -876,6 +876,9 @@ github.com/klauspost/compress/zstd/internal/xxhash
# github.com/klauspost/cpuid/v2 v2.2.11
## explicit; go 1.22
github.com/klauspost/cpuid/v2
# github.com/klauspost/crc32 v1.3.0
## explicit; go 1.23.0
github.com/klauspost/crc32
# github.com/kovidgoyal/go-parallel v1.0.1
## explicit; go 1.23
github.com/kovidgoyal/go-parallel
@@ -1037,7 +1040,7 @@ github.com/miekg/dns
# github.com/mileusna/useragent v1.3.5
## explicit; go 1.14
github.com/mileusna/useragent
# github.com/minio/crc64nvme v1.0.2
# github.com/minio/crc64nvme v1.1.0
## explicit; go 1.22
github.com/minio/crc64nvme
# github.com/minio/highwayhash v1.0.3
@@ -1046,17 +1049,15 @@ github.com/minio/highwayhash
# github.com/minio/md5-simd v1.1.2
## explicit; go 1.14
github.com/minio/md5-simd
# github.com/minio/minio-go/v7 v7.0.95
# github.com/minio/minio-go/v7 v7.0.97
## explicit; go 1.23.0
github.com/minio/minio-go/v7
github.com/minio/minio-go/v7/internal/json
github.com/minio/minio-go/v7/pkg/cors
github.com/minio/minio-go/v7/pkg/credentials
github.com/minio/minio-go/v7/pkg/encrypt
github.com/minio/minio-go/v7/pkg/kvcache
github.com/minio/minio-go/v7/pkg/lifecycle
github.com/minio/minio-go/v7/pkg/notification
github.com/minio/minio-go/v7/pkg/peeker
github.com/minio/minio-go/v7/pkg/replication
github.com/minio/minio-go/v7/pkg/s3utils
github.com/minio/minio-go/v7/pkg/set
@@ -1341,7 +1342,7 @@ github.com/opencloud-eu/icap-client
# github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76
## explicit; go 1.18
github.com/opencloud-eu/libre-graph-api-go
# github.com/opencloud-eu/reva/v2 v2.39.2-0.20251030154544-cac8a0257da6
# github.com/opencloud-eu/reva/v2 v2.39.2-0.20251106122902-c13e27f55362
## explicit; go 1.24.1
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
github.com/opencloud-eu/reva/v2/cmd/revad/runtime