[full-ci] chore: bump reva to v2.44.0 update opencloud version 6.2.0 (#2734)

This commit is contained in:
Viktor Scharf
2026-05-11 17:08:39 +02:00
committed by GitHub
parent a84820865e
commit cf0ec50da9
171 changed files with 15555 additions and 4260 deletions
+15 -14
View File
@@ -55,17 +55,17 @@ require (
github.com/mitchellh/mapstructure v1.5.0
github.com/mna/pigeon v1.3.0
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
github.com/nats-io/nats-server/v2 v2.12.6
github.com/nats-io/nats-server/v2 v2.14.0
github.com/nats-io/nats.go v1.51.0
github.com/oklog/run v1.2.0
github.com/olekukonko/tablewriter v1.1.4
github.com/onsi/ginkgo v1.16.5
github.com/onsi/ginkgo/v2 v2.28.1
github.com/onsi/gomega v1.39.1
github.com/onsi/gomega v1.40.0
github.com/open-policy-agent/opa v1.15.2
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d
github.com/opencloud-eu/reva/v2 v2.43.1-0.20260428125302-b94a4bd193be
github.com/opencloud-eu/reva/v2 v2.44.0
github.com/opensearch-project/opensearch-go/v4 v4.6.0
github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1
@@ -103,14 +103,14 @@ require (
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.43.0
go.opentelemetry.io/otel/sdk v1.43.0
go.opentelemetry.io/otel/trace v1.43.0
golang.org/x/crypto v0.49.0
golang.org/x/crypto v0.50.0
golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
golang.org/x/image v0.38.0
golang.org/x/net v0.52.0
golang.org/x/oauth2 v0.36.0
golang.org/x/sync v0.20.0
golang.org/x/term v0.41.0
golang.org/x/text v0.35.0
golang.org/x/term v0.42.0
golang.org/x/text v0.36.0
google.golang.org/genproto/googleapis/api v0.0.0-20260401024825-9d38bb4040a9
google.golang.org/grpc v1.80.0
google.golang.org/protobuf v1.36.11
@@ -122,7 +122,7 @@ require (
require (
contrib.go.opencensus.io/exporter/prometheus v0.4.2 // indirect
filippo.io/edwards25519 v1.1.1 // indirect
filippo.io/edwards25519 v1.2.0 // indirect
github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect
github.com/Azure/go-ntlmssp v0.1.1 // indirect
github.com/BurntSushi/toml v1.6.0 // indirect
@@ -136,7 +136,7 @@ require (
github.com/ajg/form v1.5.1 // indirect
github.com/alexedwards/argon2id v1.0.0 // indirect
github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964 // indirect
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op // indirect
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op // indirect
github.com/armon/go-radix v1.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@@ -220,7 +220,7 @@ require (
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/go-redis/redis/v8 v8.11.5 // indirect
github.com/go-sql-driver/mysql v1.9.3 // indirect
github.com/go-sql-driver/mysql v1.10.0 // indirect
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect
github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
github.com/go-test/deep v1.1.0 // indirect
@@ -246,7 +246,7 @@ require (
github.com/gorilla/schema v1.4.1 // indirect
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 // indirect
github.com/hashicorp/go-hclog v1.6.3 // indirect
github.com/hashicorp/go-plugin v1.7.0 // indirect
github.com/hashicorp/go-plugin v1.8.0 // indirect
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
github.com/hashicorp/yamux v0.1.2 // indirect
github.com/huandu/xstrings v1.5.0 // indirect
@@ -286,9 +286,9 @@ require (
github.com/miekg/dns v1.1.68 // indirect
github.com/mileusna/useragent v1.3.5 // indirect
github.com/minio/crc64nvme v1.1.1 // indirect
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 // indirect
github.com/minio/highwayhash v1.0.4 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
github.com/minio/minio-go/v7 v7.0.99 // indirect
github.com/minio/minio-go/v7 v7.1.0 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
@@ -376,6 +376,7 @@ require (
github.com/xrash/smetrics v0.0.0-20240521201337-686a1a2994c1 // indirect
github.com/yashtewari/glob-intersection v0.2.0 // indirect
github.com/yusufpapurcu/wmi v1.2.4 // indirect
github.com/zeebo/xxh3 v1.1.0 // indirect
go.etcd.io/etcd/api/v3 v3.6.10 // indirect
go.etcd.io/etcd/client/pkg/v3 v3.6.10 // indirect
go.etcd.io/etcd/client/v3 v3.6.10 // indirect
@@ -388,10 +389,10 @@ require (
go.uber.org/zap v1.27.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/mod v0.34.0 // indirect
golang.org/x/sys v0.43.0 // indirect
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.42.0 // indirect
golang.org/x/tools v0.43.0 // indirect
google.golang.org/genproto v0.0.0-20260128011058-8636f8732409 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20260406210006-6f92a3bedf2d // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
+32 -28
View File
@@ -39,8 +39,8 @@ contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9
dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8=
dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
filippo.io/edwards25519 v1.1.1 h1:YpjwWWlNmGIDyXOn8zLzqiD+9TyIlPhGFG96P39uBpw=
filippo.io/edwards25519 v1.1.1/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
filippo.io/edwards25519 v1.2.0 h1:crnVqOiS4jqYleHd9vaKZ+HKtHfllngJIiOpNpoJsjo=
filippo.io/edwards25519 v1.2.0/go.mod h1:xzAOLCNug/yB62zG1bQ8uziwrIqIuxhctzJT18Q77mc=
github.com/Acconut/go-httptest-recorder v1.0.0 h1:TAv2dfnqp/l+SUvIaMAUK4GeN4+wqb6KZsFFFTGhoJg=
github.com/Acconut/go-httptest-recorder v1.0.0/go.mod h1:CwQyhTH1kq/gLyWiRieo7c0uokpu3PXeyF/nZjUNtmM=
github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk=
@@ -117,8 +117,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE=
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op h1:Z/MZK75wC/NSrkgqeNIa7jexam9uWzhLmFTSCPI/kn0=
github.com/antithesishq/antithesis-sdk-go v0.7.0-default-no-op/go.mod h1:FQyySiasQQM8735Ddel3MRojmy4dA1IqCeyJ5jmPMbI=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
@@ -456,8 +456,8 @@ github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq
github.com/go-resty/resty/v2 v2.1.1-0.20191201195748-d7b97669fe48/go.mod h1:dZGr0i9PLlaaTD4H/hoZIDjQ+r6xq8mgbRzHZf7f2J8=
github.com/go-resty/resty/v2 v2.17.2 h1:FQW5oHYcIlkCNrMD2lloGScxcHJ0gkjshV3qcQAyHQk=
github.com/go-resty/resty/v2 v2.17.2/go.mod h1:kCKZ3wWmwJaNc7S29BRtUhJwy7iqmn+2mLtQrOyQlVA=
github.com/go-sql-driver/mysql v1.9.3 h1:U/N249h2WzJ3Ukj8SowVFjdtZKfu9vlLZxjPXV1aweo=
github.com/go-sql-driver/mysql v1.9.3/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU=
github.com/go-sql-driver/mysql v1.10.0 h1:Q+1LV8DkHJvSYAdR83XzuhDaTykuDx0l6fkXxoWCWfw=
github.com/go-sql-driver/mysql v1.10.0/go.mod h1:M+cqaI7+xxXGG9swrdeUIoPG3Y3KCkF0pZej+SK+nWk=
github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
@@ -626,8 +626,8 @@ github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVH
github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
github.com/hashicorp/go-plugin v1.7.0 h1:YghfQH/0QmPNc/AZMTFE3ac8fipZyZECHdDPshfk+mA=
github.com/hashicorp/go-plugin v1.7.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8=
github.com/hashicorp/go-plugin v1.8.0 h1:ie8S6RRY8RvB2usYZv+AAZ/wBvx2AU5p5QeP5j/FORs=
github.com/hashicorp/go-plugin v1.8.0/go.mod h1:BExt6KEaIYx804z8k4gRzRLEvxKVb+kn0NMcihqOqb8=
github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=
github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
@@ -839,12 +839,12 @@ github.com/mileusna/useragent v1.3.5 h1:SJM5NzBmh/hO+4LGeATKpaEX9+b4vcGg2qXGLiNG
github.com/mileusna/useragent v1.3.5/go.mod h1:3d8TOmwL/5I8pJjyVDteHtgDGcefrFUX4ccGOMKNYYc=
github.com/minio/crc64nvme v1.1.1 h1:8dwx/Pz49suywbO+auHCBpCtlW1OfpcLN7wYgVR6wAI=
github.com/minio/crc64nvme v1.1.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/minio/highwayhash v1.0.4 h1:asJizugGgchQod2ja9NJlGOWq4s7KsAWr5XUc9Clgl4=
github.com/minio/highwayhash v1.0.4/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
github.com/minio/minio-go/v7 v7.0.99 h1:2vH/byrwUkIpFQFOilvTfaUpvAX3fEFhEzO+DR3DlCE=
github.com/minio/minio-go/v7 v7.0.99/go.mod h1:EtGNKtlX20iL2yaYnxEigaIvj0G0GwSDnifnG8ClIdw=
github.com/minio/minio-go/v7 v7.1.0 h1:QEt5IStDpxgGjEdtOgpiZ5QhmSl3ax7qy61vi2SwHO8=
github.com/minio/minio-go/v7 v7.1.0/go.mod h1:Dm7WS1AgLmBa0NcQD6SeJnJf+K/EUW3GR7Ks6olB3OA=
github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw=
github.com/mitchellh/copystructure v1.2.0/go.mod h1:qLl+cE2AmVv+CoeAwDPye/v+N2HKCj9FbZEVFJRxO9s=
@@ -900,8 +900,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRW
github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.12.6 h1:Egbx9Vl7Ch8wTtpXPGqbehkZ+IncKqShUxvrt1+Enc8=
github.com/nats-io/nats-server/v2 v2.12.6/go.mod h1:4HPlrvtmSO3yd7KcElDNMx9kv5EBJBnJJzQPptXlheo=
github.com/nats-io/nats-server/v2 v2.14.0 h1:+8q0HrDFotwLLcGH/legOEOnowunhK+aZ4GYBIWpQlM=
github.com/nats-io/nats-server/v2 v2.14.0/go.mod h1:ImVUUDvfClJbb6cuJQRc1VmgDCXKM5ds0OoiG9MVOKo=
github.com/nats-io/nats.go v1.51.0 h1:ByW84XTz6W03GSSsygsZcA+xgKK8vPGaa/FCAAEHnAI=
github.com/nats-io/nats.go v1.51.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno=
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
@@ -940,8 +940,8 @@ github.com/onsi/ginkgo/v2 v2.28.1/go.mod h1:CLtbVInNckU3/+gC8LzkGUb9oF+e8W8TdUsx
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/onsi/gomega v1.39.1 h1:1IJLAad4zjPn2PsnhH70V4DKRFlrCzGBNrNaru+Vf28=
github.com/onsi/gomega v1.39.1/go.mod h1:hL6yVALoTOxeWudERyfppUcZXjMwIMLnuSfruD2lcfg=
github.com/onsi/gomega v1.40.0 h1:Vtol0e1MghCD2ZVIilPDIg44XSL9l2QAn8ZNaljWcJc=
github.com/onsi/gomega v1.40.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A=
github.com/open-policy-agent/opa v1.15.2 h1:dS9q+0Yvruq/VNvWJc5qCvCchn715OWc3HLHXn/UCCc=
github.com/open-policy-agent/opa v1.15.2/go.mod h1:c6SN+7jSsUcKJLQc5P4yhwx8YYDRbjpAiGkBOTqxaa4=
github.com/opencloud-eu/go-micro-plugins/v4/store/nats-js-kv v0.0.0-20250512152754-23325793059a h1:Sakl76blJAaM6NxylVkgSzktjo2dS504iDotEFJsh3M=
@@ -952,8 +952,8 @@ github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9 h1:dIft
github.com/opencloud-eu/inotifywaitgo v0.0.0-20251111171128-a390bae3c5e9/go.mod h1:JWyDC6H+5oZRdUJUgKuaye+8Ph5hEs6HVzVoPKzWSGI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d h1:JcqGDiyrcaQwVyV861TUyQgO7uEmsjkhfm7aQd84dOw=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260310090739-853d972b282d/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q=
github.com/opencloud-eu/reva/v2 v2.43.1-0.20260428125302-b94a4bd193be h1:484jMXXoGFmOA6ll4AoBb5FTzBDHv4QNtmupfZ0mnk4=
github.com/opencloud-eu/reva/v2 v2.43.1-0.20260428125302-b94a4bd193be/go.mod h1:mBKl0Qc+fG/xkWbp79QfbTlq8Ao5JTJcZaTRo1dTwfo=
github.com/opencloud-eu/reva/v2 v2.44.0 h1:5SZRIjFzyc4e20dCJQbeZ15Lo7COvrSQ1jgjFVQWmRE=
github.com/opencloud-eu/reva/v2 v2.44.0/go.mod h1:ZYjHYcg3rLInH81yNnC5GgS4I0JS9KRFVwxJptw1/CM=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -1275,6 +1275,10 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0=
github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ=
github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs=
github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.4.3 h1:dEadXpI6G79deX5prL3QRNP6JB8UxVkqo4UPnHaNXJo=
go.etcd.io/bbolt v1.4.3/go.mod h1:tKQlpPaYCVFctUIgFKFnAlvbmB3tpy1vkTnDWohtc0E=
@@ -1358,8 +1362,8 @@ golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0
golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/crypto v0.50.0 h1:zO47/JPrL6vsNkINmLoo/PH1gcxpls50DNogFvB5ZGI=
golang.org/x/crypto v0.50.0/go.mod h1:3muZ7vA7PBCE6xgPX7nkzzjiUq87kRItoJQM1Yo8S+Q=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1397,8 +1401,8 @@ golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/mod v0.34.0 h1:xIHgNUUnW6sYkcM5Jleh05DvLOtwc6RitGHbDk4akRI=
golang.org/x/mod v0.34.0/go.mod h1:ykgH52iCZe79kzLLMhyCUzhMci+nQj+0XkbXpNYtVjY=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1566,8 +1570,8 @@ golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
golang.org/x/term v0.42.0 h1:UiKe+zDFmJobeJ5ggPwOshJIVt6/Ft0rcfrXZDLWAWY=
golang.org/x/term v0.42.0/go.mod h1:Dq/D+snpsbazcBG5+F9Q1n2rXV8Ma+71xEjTRufARgY=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1580,8 +1584,8 @@ golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg=
golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
@@ -1642,8 +1646,8 @@ golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4f
golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/tools v0.43.0 h1:12BdW9CeB3Z+J/I/wj34VMl8X+fEXBxVR90JeMX5E7s=
golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+1 -1
View File
@@ -34,7 +34,7 @@ var (
// LatestTag is the latest released version plus the dev meta version.
// Will be overwritten by the release pipeline
// Needs a manual change for every tagged release
LatestTag = "6.1.0+dev"
LatestTag = "6.2.0+dev"
// Date indicates the build date.
// This has been removed, it looks like you can only replace static strings with recent go versions
+1 -1
View File
@@ -204,7 +204,7 @@ type PosixDriver struct {
EnableFSRevisions bool `yaml:"enable_fs_revisions" env:"STORAGE_USERS_POSIX_ENABLE_FS_REVISIONS" desc:"Allow for generating revisions from changes done to the local storage. Note: This doubles the number of bytes stored on disk because a copy of the current revision is stored to be turned into a revision later." introductionVersion:"1.0.0"`
ScanFS bool `yaml:"scan_fs" env:"STORAGE_USERS_POSIX_SCAN_FS" desc:"Scan the filesystem at startup for changes and update the metadata accordingly." introductionVersion:"%%NEXT%%"`
ScanFS bool `yaml:"scan_fs" env:"STORAGE_USERS_POSIX_SCAN_FS" desc:"Scan the filesystem at startup for changes and update the metadata accordingly." introductionVersion:"6.2.0"`
WatchFS bool `yaml:"watch_fs" env:"STORAGE_USERS_POSIX_WATCH_FS" desc:"Enable the filesystem watcher to detect changes to the filesystem. This is used to detect changes to the filesystem and update the metadata accordingly." introductionVersion:"2.0.0"`
WatchType string `yaml:"watch_type" env:"STORAGE_USERS_POSIX_WATCH_TYPE" desc:"Type of the watcher to use for getting notified about changes to the filesystem. Currently available options are 'inotifywait' (default), 'cephfs', 'gpfswatchfolder' and 'gpfsfileauditlogging'." introductionVersion:"1.0.0"`
WatchPath string `yaml:"watch_path" env:"STORAGE_USERS_POSIX_WATCH_PATH" desc:"Path to the watch directory/file. Only applies to the 'gpfsfileauditlogging' and 'inotifywait' watcher, in which case it is the path of the file audit log file/base directory to watch." introductionVersion:"1.0.0"`
+4 -2
View File
@@ -7,8 +7,10 @@ import "filippo.io/edwards25519"
This library implements the edwards25519 elliptic curve, exposing the necessary APIs to build a wide array of higher-level primitives.
Read the docs at [pkg.go.dev/filippo.io/edwards25519](https://pkg.go.dev/filippo.io/edwards25519).
The code is originally derived from Adam Langley's internal implementation in the Go standard library, and includes George Tankersley's [performance improvements](https://golang.org/cl/71950). It was then further developed by Henry de Valence for use in ristretto255, and was finally [merged back into the Go standard library](https://golang.org/cl/276272) as of Go 1.17. It now tracks the upstream codebase and extends it with additional functionality.
The package tracks the upstream standard library package `crypto/internal/fips140/edwards25519` and extends it with additional functionality.
Most users don't need this package, and should instead use `crypto/ed25519` for signatures, `golang.org/x/crypto/curve25519` for Diffie-Hellman, or `github.com/gtank/ristretto255` for prime order group logic. However, for anyone currently using a fork of `crypto/internal/edwards25519`/`crypto/ed25519/internal/edwards25519` or `github.com/agl/edwards25519`, this package should be a safer, faster, and more powerful alternative.
The code is originally derived from Adam Langley's internal implementation in the Go standard library, and includes George Tankersley's [performance improvements](https://golang.org/cl/71950). It was then further developed by Henry de Valence for use in ristretto255, and was finally [merged back into the Go standard library](https://golang.org/cl/276272) as of Go 1.17.
Most users don't need this package, and should instead use `crypto/ed25519` for signatures, `crypto/ecdh` for Diffie-Hellman, or `github.com/gtank/ristretto255` for prime order group logic. However, for anyone currently using a fork of the internal `edwards25519` package or of `github.com/agl/edwards25519`, this package should be a safer, faster, and more powerful alternative.
Since this package is meant to curb proliferation of edwards25519 implementations in the Go ecosystem, it welcomes requests for new APIs or reviewable performance improvements.
+3 -3
View File
@@ -10,11 +10,11 @@
// the curve used by the Ed25519 signature scheme.
//
// Most users don't need this package, and should instead use crypto/ed25519 for
// signatures, golang.org/x/crypto/curve25519 for Diffie-Hellman, or
// github.com/gtank/ristretto255 for prime order group logic.
// signatures, crypto/ecdh for Diffie-Hellman, or github.com/gtank/ristretto255
// for prime order group logic.
//
// However, developers who do need to interact with low-level edwards25519
// operations can use this package, which is an extended version of
// crypto/internal/edwards25519 from the standard library repackaged as
// crypto/internal/fips140/edwards25519 from the standard library repackaged as
// an importable module.
package edwards25519
+59 -8
View File
@@ -9,6 +9,7 @@ package edwards25519
import (
"errors"
"slices"
"filippo.io/edwards25519/field"
)
@@ -100,13 +101,15 @@ func (v *Point) bytesMontgomery(buf *[32]byte) []byte {
//
// u = (1 + y) / (1 - y)
//
// where y = Y / Z.
// where y = Y / Z and therefore
//
// u = (Z + Y) / (Z - Y)
var y, recip, u field.Element
var n, r, u field.Element
y.Multiply(&v.y, y.Invert(&v.z)) // y = Y / Z
recip.Invert(recip.Subtract(feOne, &y)) // r = 1/(1 - y)
u.Multiply(u.Add(feOne, &y), &recip) // u = (1 + y)*r
n.Add(&v.z, &v.y) // n = Z + Y
r.Invert(r.Subtract(&v.z, &v.y)) // r = 1 / (Z - Y)
u.Multiply(&n, &r) // u = n * r
return copyFieldElement(buf, &u)
}
@@ -124,7 +127,7 @@ func (v *Point) MultByCofactor(p *Point) *Point {
return v.fromP1xP1(&result)
}
// Given k > 0, set s = s**(2*i).
// Given k > 0, set s = s**(2*k).
func (s *Scalar) pow2k(k int) {
for i := 0; i < k; i++ {
s.Multiply(s, s)
@@ -250,12 +253,14 @@ func (v *Point) MultiScalarMult(scalars []*Scalar, points []*Point) *Point {
// between each point in the multiscalar equation.
// Build lookup tables for each point
tables := make([]projLookupTable, len(points))
tables := make([]projLookupTable, 0, 2) // avoid allocation for small sizes
tables = slices.Grow(tables, len(points))[:len(points)]
for i := range tables {
tables[i].FromP3(points[i])
}
// Compute signed radix-16 digits for each scalar
digits := make([][64]int8, len(scalars))
digits := make([][64]int8, 0, 2) // avoid allocation for small sizes
digits = slices.Grow(digits, len(scalars))[:len(scalars)]
for i := range digits {
digits[i] = scalars[i].signedRadix16()
}
@@ -348,3 +353,49 @@ func (v *Point) VarTimeMultiScalarMult(scalars []*Scalar, points []*Point) *Poin
v.fromP2(tmp2)
return v
}
// Select sets v to a if cond == 1 and to b if cond == 0.
func (v *Point) Select(a, b *Point, cond int) *Point {
checkInitialized(a, b)
v.x.Select(&a.x, &b.x, cond)
v.y.Select(&a.y, &b.y, cond)
v.z.Select(&a.z, &b.z, cond)
v.t.Select(&a.t, &b.t, cond)
return v
}
// Double sets v = p + p, and returns v.
func (v *Point) Double(p *Point) *Point {
checkInitialized(p)
pp := new(projP2).FromP3(p)
p1 := new(projP1xP1).Double(pp)
return v.fromP1xP1(p1)
}
func (v *Point) addCached(p *Point, qCached *projCached) *Point {
result := new(projP1xP1).Add(p, qCached)
return v.fromP1xP1(result)
}
// ScalarMultSlow sets v = x * q, and returns v. It doesn't precompute a large
// table, so it is considerably slower, but requires less memory.
//
// The scalar multiplication is done in constant time.
func (v *Point) ScalarMultSlow(x *Scalar, q *Point) *Point {
checkInitialized(q)
s := x.Bytes()
qCached := new(projCached).FromP3(q)
v.Set(NewIdentityPoint())
t := new(Point)
for i := 255; i >= 0; i-- {
v.Double(v)
t.addCached(v, qCached)
cond := (s[i/8] >> (i % 8)) & 1
v.Select(t, v, int(cond))
}
return v
}
+17 -17
View File
@@ -90,11 +90,7 @@ func (v *Element) Add(a, b *Element) *Element {
v.l2 = a.l2 + b.l2
v.l3 = a.l3 + b.l3
v.l4 = a.l4 + b.l4
// Using the generic implementation here is actually faster than the
// assembly. Probably because the body of this function is so simple that
// the compiler can figure out better optimizations by inlining the carry
// propagation.
return v.carryPropagateGeneric()
return v.carryPropagate()
}
// Subtract sets v = a - b, and returns v.
@@ -232,18 +228,22 @@ func (v *Element) bytes(out *[32]byte) []byte {
t := *v
t.reduce()
var buf [8]byte
for i, l := range [5]uint64{t.l0, t.l1, t.l2, t.l3, t.l4} {
bitsOffset := i * 51
binary.LittleEndian.PutUint64(buf[:], l<<uint(bitsOffset%8))
for i, bb := range buf {
off := bitsOffset/8 + i
if off >= len(out) {
break
}
out[off] |= bb
}
}
// Pack five 51-bit limbs into four 64-bit words:
//
// 255 204 153 102 51 0
// ├──l4──┼──l3──┼──l2──┼──l1──┼──l0──┤
// ├───u3───┼───u2───┼───u1───┼───u0───┤
// 256 192 128 64 0
u0 := t.l1<<51 | t.l0
u1 := t.l2<<(102-64) | t.l1>>(64-51)
u2 := t.l3<<(153-128) | t.l2>>(128-102)
u3 := t.l4<<(204-192) | t.l3>>(192-153)
binary.LittleEndian.PutUint64(out[0*8:], u0)
binary.LittleEndian.PutUint64(out[1*8:], u1)
binary.LittleEndian.PutUint64(out[2*8:], u2)
binary.LittleEndian.PutUint64(out[3*8:], u3)
return out[:]
}
+1 -2
View File
@@ -1,7 +1,6 @@
// Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT.
//go:build amd64 && gc && !purego
// +build amd64,gc,!purego
//go:build !purego
package field
+111 -92
View File
@@ -1,7 +1,6 @@
// Code generated by command: go run fe_amd64_asm.go -out ../fe_amd64.s -stubs ../fe_amd64.go -pkg field. DO NOT EDIT.
//go:build amd64 && gc && !purego
// +build amd64,gc,!purego
//go:build !purego
#include "textflag.h"
@@ -17,32 +16,36 @@ TEXT ·feMul(SB), NOSPLIT, $0-24
MOVQ DX, SI
// r0 += 19×a1×b4
MOVQ 8(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 32(BX)
ADDQ AX, DI
ADCQ DX, SI
MOVQ 8(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 32(BX)
ADDQ AX, DI
ADCQ DX, SI
// r0 += 19×a2×b3
MOVQ 16(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 24(BX)
ADDQ AX, DI
ADCQ DX, SI
MOVQ 16(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 24(BX)
ADDQ AX, DI
ADCQ DX, SI
// r0 += 19×a3×b2
MOVQ 24(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 16(BX)
ADDQ AX, DI
ADCQ DX, SI
MOVQ 24(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 16(BX)
ADDQ AX, DI
ADCQ DX, SI
// r0 += 19×a4×b1
MOVQ 32(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 8(BX)
ADDQ AX, DI
ADCQ DX, SI
MOVQ 32(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 8(BX)
ADDQ AX, DI
ADCQ DX, SI
// r1 = a0×b1
MOVQ (CX), AX
@@ -57,25 +60,28 @@ TEXT ·feMul(SB), NOSPLIT, $0-24
ADCQ DX, R8
// r1 += 19×a2×b4
MOVQ 16(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 32(BX)
ADDQ AX, R9
ADCQ DX, R8
MOVQ 16(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 32(BX)
ADDQ AX, R9
ADCQ DX, R8
// r1 += 19×a3×b3
MOVQ 24(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 24(BX)
ADDQ AX, R9
ADCQ DX, R8
MOVQ 24(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 24(BX)
ADDQ AX, R9
ADCQ DX, R8
// r1 += 19×a4×b2
MOVQ 32(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 16(BX)
ADDQ AX, R9
ADCQ DX, R8
MOVQ 32(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 16(BX)
ADDQ AX, R9
ADCQ DX, R8
// r2 = a0×b2
MOVQ (CX), AX
@@ -96,18 +102,20 @@ TEXT ·feMul(SB), NOSPLIT, $0-24
ADCQ DX, R10
// r2 += 19×a3×b4
MOVQ 24(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 32(BX)
ADDQ AX, R11
ADCQ DX, R10
MOVQ 24(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 32(BX)
ADDQ AX, R11
ADCQ DX, R10
// r2 += 19×a4×b3
MOVQ 32(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 24(BX)
ADDQ AX, R11
ADCQ DX, R10
MOVQ 32(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 24(BX)
ADDQ AX, R11
ADCQ DX, R10
// r3 = a0×b3
MOVQ (CX), AX
@@ -134,11 +142,12 @@ TEXT ·feMul(SB), NOSPLIT, $0-24
ADCQ DX, R12
// r3 += 19×a4×b4
MOVQ 32(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 32(BX)
ADDQ AX, R13
ADCQ DX, R12
MOVQ 32(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 32(BX)
ADDQ AX, R13
ADCQ DX, R12
// r4 = a0×b4
MOVQ (CX), AX
@@ -232,18 +241,22 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16
MOVQ DX, BX
// r0 += 38×l1×l4
MOVQ 8(CX), AX
IMUL3Q $0x26, AX, AX
MULQ 32(CX)
ADDQ AX, SI
ADCQ DX, BX
MOVQ 8(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
SHLQ $0x01, AX
MULQ 32(CX)
ADDQ AX, SI
ADCQ DX, BX
// r0 += 38×l2×l3
MOVQ 16(CX), AX
IMUL3Q $0x26, AX, AX
MULQ 24(CX)
ADDQ AX, SI
ADCQ DX, BX
MOVQ 16(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
SHLQ $0x01, AX
MULQ 24(CX)
ADDQ AX, SI
ADCQ DX, BX
// r1 = 2×l0×l1
MOVQ (CX), AX
@@ -253,18 +266,21 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16
MOVQ DX, DI
// r1 += 38×l2×l4
MOVQ 16(CX), AX
IMUL3Q $0x26, AX, AX
MULQ 32(CX)
ADDQ AX, R8
ADCQ DX, DI
MOVQ 16(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
SHLQ $0x01, AX
MULQ 32(CX)
ADDQ AX, R8
ADCQ DX, DI
// r1 += 19×l3×l3
MOVQ 24(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 24(CX)
ADDQ AX, R8
ADCQ DX, DI
MOVQ 24(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 24(CX)
ADDQ AX, R8
ADCQ DX, DI
// r2 = 2×l0×l2
MOVQ (CX), AX
@@ -280,11 +296,13 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16
ADCQ DX, R9
// r2 += 38×l3×l4
MOVQ 24(CX), AX
IMUL3Q $0x26, AX, AX
MULQ 32(CX)
ADDQ AX, R10
ADCQ DX, R9
MOVQ 24(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
SHLQ $0x01, AX
MULQ 32(CX)
ADDQ AX, R10
ADCQ DX, R9
// r3 = 2×l0×l3
MOVQ (CX), AX
@@ -294,18 +312,19 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16
MOVQ DX, R11
// r3 += 2×l1×l2
MOVQ 8(CX), AX
IMUL3Q $0x02, AX, AX
MULQ 16(CX)
ADDQ AX, R12
ADCQ DX, R11
MOVQ 8(CX), AX
SHLQ $0x01, AX
MULQ 16(CX)
ADDQ AX, R12
ADCQ DX, R11
// r3 += 19×l4×l4
MOVQ 32(CX), AX
IMUL3Q $0x13, AX, AX
MULQ 32(CX)
ADDQ AX, R12
ADCQ DX, R11
MOVQ 32(CX), DX
LEAQ (DX)(DX*8), AX
LEAQ (DX)(AX*2), AX
MULQ 32(CX)
ADDQ AX, R12
ADCQ DX, R11
// r4 = 2×l0×l4
MOVQ (CX), AX
@@ -315,11 +334,11 @@ TEXT ·feSquare(SB), NOSPLIT, $0-16
MOVQ DX, R13
// r4 += 2×l1×l3
MOVQ 8(CX), AX
IMUL3Q $0x02, AX, AX
MULQ 24(CX)
ADDQ AX, R14
ADCQ DX, R13
MOVQ 8(CX), AX
SHLQ $0x01, AX
MULQ 24(CX)
ADDQ AX, R14
ADCQ DX, R13
// r4 += l2×l2
MOVQ 16(CX), AX
+1 -2
View File
@@ -2,8 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !amd64 || !gc || purego
// +build !amd64 !gc purego
//go:build !amd64 || purego
package field
-16
View File
@@ -1,16 +0,0 @@
// Copyright (c) 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build arm64 && gc && !purego
// +build arm64,gc,!purego
package field
//go:noescape
func carryPropagate(v *Element)
func (v *Element) carryPropagate() *Element {
carryPropagate(v)
return v
}
-42
View File
@@ -1,42 +0,0 @@
// Copyright (c) 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build arm64 && gc && !purego
#include "textflag.h"
// carryPropagate works exactly like carryPropagateGeneric and uses the
// same AND, ADD, and LSR+MADD instructions emitted by the compiler, but
// avoids loading R0-R4 twice and uses LDP and STP.
//
// See https://golang.org/issues/43145 for the main compiler issue.
//
// func carryPropagate(v *Element)
TEXT ·carryPropagate(SB),NOFRAME|NOSPLIT,$0-8
MOVD v+0(FP), R20
LDP 0(R20), (R0, R1)
LDP 16(R20), (R2, R3)
MOVD 32(R20), R4
AND $0x7ffffffffffff, R0, R10
AND $0x7ffffffffffff, R1, R11
AND $0x7ffffffffffff, R2, R12
AND $0x7ffffffffffff, R3, R13
AND $0x7ffffffffffff, R4, R14
ADD R0>>51, R11, R11
ADD R1>>51, R12, R12
ADD R2>>51, R13, R13
ADD R3>>51, R14, R14
// R4>>51 * 19 + R10 -> R10
LSR $51, R4, R21
MOVD $19, R22
MADD R22, R10, R21, R10
STP (R10, R11), 0(R20)
STP (R12, R13), 16(R20)
MOVD R14, 32(R20)
RET
-12
View File
@@ -1,12 +0,0 @@
// Copyright (c) 2021 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !arm64 || !gc || purego
// +build !arm64 !gc purego
package field
func (v *Element) carryPropagate() *Element {
return v.carryPropagateGeneric()
}
+88 -82
View File
@@ -12,20 +12,42 @@ type uint128 struct {
lo, hi uint64
}
// mul64 returns a * b.
func mul64(a, b uint64) uint128 {
// mul returns a * b.
func mul(a, b uint64) uint128 {
hi, lo := bits.Mul64(a, b)
return uint128{lo, hi}
}
// addMul64 returns v + a * b.
func addMul64(v uint128, a, b uint64) uint128 {
// addMul returns v + a * b.
func addMul(v uint128, a, b uint64) uint128 {
hi, lo := bits.Mul64(a, b)
lo, c := bits.Add64(lo, v.lo, 0)
hi, _ = bits.Add64(hi, v.hi, c)
return uint128{lo, hi}
}
// mul19 returns v * 19.
func mul19(v uint64) uint64 {
// Using this approach seems to yield better optimizations than *19.
return v + (v+v<<3)<<1
}
// addMul19 returns v + 19 * a * b, where a and b are at most 52 bits.
func addMul19(v uint128, a, b uint64) uint128 {
hi, lo := bits.Mul64(mul19(a), b)
lo, c := bits.Add64(lo, v.lo, 0)
hi, _ = bits.Add64(hi, v.hi, c)
return uint128{lo, hi}
}
// addMul38 returns v + 38 * a * b, where a and b are at most 52 bits.
func addMul38(v uint128, a, b uint64) uint128 {
hi, lo := bits.Mul64(mul19(a), b*2)
lo, c := bits.Add64(lo, v.lo, 0)
hi, _ = bits.Add64(hi, v.hi, c)
return uint128{lo, hi}
}
// shiftRightBy51 returns a >> 51. a is assumed to be at most 115 bits.
func shiftRightBy51(a uint128) uint64 {
return (a.hi << (64 - 51)) | (a.lo >> 51)
@@ -76,45 +98,40 @@ func feMulGeneric(v, a, b *Element) {
//
// Finally we add up the columns into wide, overlapping limbs.
a1_19 := a1 * 19
a2_19 := a2 * 19
a3_19 := a3 * 19
a4_19 := a4 * 19
// r0 = a0×b0 + 19×(a1×b4 + a2×b3 + a3×b2 + a4×b1)
r0 := mul64(a0, b0)
r0 = addMul64(r0, a1_19, b4)
r0 = addMul64(r0, a2_19, b3)
r0 = addMul64(r0, a3_19, b2)
r0 = addMul64(r0, a4_19, b1)
r0 := mul(a0, b0)
r0 = addMul19(r0, a1, b4)
r0 = addMul19(r0, a2, b3)
r0 = addMul19(r0, a3, b2)
r0 = addMul19(r0, a4, b1)
// r1 = a0×b1 + a1×b0 + 19×(a2×b4 + a3×b3 + a4×b2)
r1 := mul64(a0, b1)
r1 = addMul64(r1, a1, b0)
r1 = addMul64(r1, a2_19, b4)
r1 = addMul64(r1, a3_19, b3)
r1 = addMul64(r1, a4_19, b2)
r1 := mul(a0, b1)
r1 = addMul(r1, a1, b0)
r1 = addMul19(r1, a2, b4)
r1 = addMul19(r1, a3, b3)
r1 = addMul19(r1, a4, b2)
// r2 = a0×b2 + a1×b1 + a2×b0 + 19×(a3×b4 + a4×b3)
r2 := mul64(a0, b2)
r2 = addMul64(r2, a1, b1)
r2 = addMul64(r2, a2, b0)
r2 = addMul64(r2, a3_19, b4)
r2 = addMul64(r2, a4_19, b3)
r2 := mul(a0, b2)
r2 = addMul(r2, a1, b1)
r2 = addMul(r2, a2, b0)
r2 = addMul19(r2, a3, b4)
r2 = addMul19(r2, a4, b3)
// r3 = a0×b3 + a1×b2 + a2×b1 + a3×b0 + 19×a4×b4
r3 := mul64(a0, b3)
r3 = addMul64(r3, a1, b2)
r3 = addMul64(r3, a2, b1)
r3 = addMul64(r3, a3, b0)
r3 = addMul64(r3, a4_19, b4)
r3 := mul(a0, b3)
r3 = addMul(r3, a1, b2)
r3 = addMul(r3, a2, b1)
r3 = addMul(r3, a3, b0)
r3 = addMul19(r3, a4, b4)
// r4 = a0×b4 + a1×b3 + a2×b2 + a3×b1 + a4×b0
r4 := mul64(a0, b4)
r4 = addMul64(r4, a1, b3)
r4 = addMul64(r4, a2, b2)
r4 = addMul64(r4, a3, b1)
r4 = addMul64(r4, a4, b0)
r4 := mul(a0, b4)
r4 = addMul(r4, a1, b3)
r4 = addMul(r4, a2, b2)
r4 = addMul(r4, a3, b1)
r4 = addMul(r4, a4, b0)
// After the multiplication, we need to reduce (carry) the five coefficients
// to obtain a result with limbs that are at most slightly larger than 2⁵¹,
@@ -149,7 +166,7 @@ func feMulGeneric(v, a, b *Element) {
c3 := shiftRightBy51(r3)
c4 := shiftRightBy51(r4)
rr0 := r0.lo&maskLow51Bits + c4*19
rr0 := r0.lo&maskLow51Bits + mul19(c4)
rr1 := r1.lo&maskLow51Bits + c0
rr2 := r2.lo&maskLow51Bits + c1
rr3 := r3.lo&maskLow51Bits + c2
@@ -158,8 +175,12 @@ func feMulGeneric(v, a, b *Element) {
// Now all coefficients fit into 64-bit registers but are still too large to
// be passed around as an Element. We therefore do one last carry chain,
// where the carries will be small enough to fit in the wiggle room above 2⁵¹.
*v = Element{rr0, rr1, rr2, rr3, rr4}
v.carryPropagate()
v.l0 = rr0&maskLow51Bits + mul19(rr4>>51)
v.l1 = rr1&maskLow51Bits + rr0>>51
v.l2 = rr2&maskLow51Bits + rr1>>51
v.l3 = rr3&maskLow51Bits + rr2>>51
v.l4 = rr4&maskLow51Bits + rr3>>51
}
func feSquareGeneric(v, a *Element) {
@@ -190,44 +211,31 @@ func feSquareGeneric(v, a *Element) {
// l0l4 19×l4l4 19×l3l4 19×l2l4 19×l1l4 =
// --------------------------------------
// r4 r3 r2 r1 r0
//
// With precomputed 2×, 19×, and 2×19× terms, we can compute each limb with
// only three Mul64 and four Add64, instead of five and eight.
l0_2 := l0 * 2
l1_2 := l1 * 2
l1_38 := l1 * 38
l2_38 := l2 * 38
l3_38 := l3 * 38
l3_19 := l3 * 19
l4_19 := l4 * 19
// r0 = l0×l0 + 19×(l1×l4 + l2×l3 + l3×l2 + l4×l1) = l0×l0 + 19×2×(l1×l4 + l2×l3)
r0 := mul64(l0, l0)
r0 = addMul64(r0, l1_38, l4)
r0 = addMul64(r0, l2_38, l3)
r0 := mul(l0, l0)
r0 = addMul38(r0, l1, l4)
r0 = addMul38(r0, l2, l3)
// r1 = l0×l1 + l1×l0 + 19×(l2×l4 + l3×l3 + l4×l2) = 2×l0×l1 + 19×2×l2×l4 + 19×l3×l3
r1 := mul64(l0_2, l1)
r1 = addMul64(r1, l2_38, l4)
r1 = addMul64(r1, l3_19, l3)
r1 := mul(l0*2, l1)
r1 = addMul38(r1, l2, l4)
r1 = addMul19(r1, l3, l3)
// r2 = l0×l2 + l1×l1 + l2×l0 + 19×(l3×l4 + l4×l3) = 2×l0×l2 + l1×l1 + 19×2×l3×l4
r2 := mul64(l0_2, l2)
r2 = addMul64(r2, l1, l1)
r2 = addMul64(r2, l3_38, l4)
r2 := mul(l0*2, l2)
r2 = addMul(r2, l1, l1)
r2 = addMul38(r2, l3, l4)
// r3 = l0×l3 + l1×l2 + l2×l1 + l3×l0 + 19×l4×l4 = 2×l0×l3 + 2×l1×l2 + 19×l4×l4
r3 := mul64(l0_2, l3)
r3 = addMul64(r3, l1_2, l2)
r3 = addMul64(r3, l4_19, l4)
r3 := mul(l0*2, l3)
r3 = addMul(r3, l1*2, l2)
r3 = addMul19(r3, l4, l4)
// r4 = l0×l4 + l1×l3 + l2×l2 + l3×l1 + l4×l0 = 2×l0×l4 + 2×l1×l3 + l2×l2
r4 := mul64(l0_2, l4)
r4 = addMul64(r4, l1_2, l3)
r4 = addMul64(r4, l2, l2)
r4 := mul(l0*2, l4)
r4 = addMul(r4, l1*2, l3)
r4 = addMul(r4, l2, l2)
c0 := shiftRightBy51(r0)
c1 := shiftRightBy51(r1)
@@ -235,32 +243,30 @@ func feSquareGeneric(v, a *Element) {
c3 := shiftRightBy51(r3)
c4 := shiftRightBy51(r4)
rr0 := r0.lo&maskLow51Bits + c4*19
rr0 := r0.lo&maskLow51Bits + mul19(c4)
rr1 := r1.lo&maskLow51Bits + c0
rr2 := r2.lo&maskLow51Bits + c1
rr3 := r3.lo&maskLow51Bits + c2
rr4 := r4.lo&maskLow51Bits + c3
*v = Element{rr0, rr1, rr2, rr3, rr4}
v.carryPropagate()
v.l0 = rr0&maskLow51Bits + mul19(rr4>>51)
v.l1 = rr1&maskLow51Bits + rr0>>51
v.l2 = rr2&maskLow51Bits + rr1>>51
v.l3 = rr3&maskLow51Bits + rr2>>51
v.l4 = rr4&maskLow51Bits + rr3>>51
}
// carryPropagateGeneric brings the limbs below 52 bits by applying the reduction
// carryPropagate brings the limbs below 52 bits by applying the reduction
// identity (a * 2²⁵⁵ + b = a * 19 + b) to the l4 carry.
func (v *Element) carryPropagateGeneric() *Element {
c0 := v.l0 >> 51
c1 := v.l1 >> 51
c2 := v.l2 >> 51
c3 := v.l3 >> 51
c4 := v.l4 >> 51
// c4 is at most 64 - 51 = 13 bits, so c4*19 is at most 18 bits, and
func (v *Element) carryPropagate() *Element {
// (l4>>51) is at most 64 - 51 = 13 bits, so (l4>>51)*19 is at most 18 bits, and
// the final l0 will be at most 52 bits. Similarly for the rest.
v.l0 = v.l0&maskLow51Bits + c4*19
v.l1 = v.l1&maskLow51Bits + c0
v.l2 = v.l2&maskLow51Bits + c1
v.l3 = v.l3&maskLow51Bits + c2
v.l4 = v.l4&maskLow51Bits + c3
l0 := v.l0
v.l0 = v.l0&maskLow51Bits + mul19(v.l4>>51)
v.l4 = v.l4&maskLow51Bits + v.l3>>51
v.l3 = v.l3&maskLow51Bits + v.l2>>51
v.l2 = v.l2&maskLow51Bits + v.l1>>51
v.l1 = v.l1&maskLow51Bits + l0>>51
return v
}
+53
View File
@@ -0,0 +1,53 @@
#!/usr/bin/env bash
set -euo pipefail
if [ "$#" -ne 1 ]; then
echo "Usage: $0 <tag>"
exit 1
fi
TAG="$1"
TMPDIR="$(mktemp -d)"
cleanup() {
rm -rf "$TMPDIR"
}
trap cleanup EXIT
command -v git >/dev/null
command -v git-filter-repo >/dev/null
if [ -d "$HOME/go/.git" ]; then
REFERENCE=(--reference "$HOME/go" --dissociate)
else
REFERENCE=()
fi
git -c advice.detachedHead=false clone --no-checkout "${REFERENCE[@]}" \
-b "$TAG" https://go.googlesource.com/go.git "$TMPDIR"
# Simplify the history graph by removing the dev.boringcrypto branches, whose
# merges end up empty after grafting anyway. This also fixes a weird quirk
# (maybe a git-filter-repo bug?) where only one file from an old path,
# src/crypto/ed25519/internal/edwards25519/const.go, would still exist in the
# filtered repo.
git -C "$TMPDIR" replace --graft f771edd7f9 99f1bf54eb
git -C "$TMPDIR" replace --graft 109c13b64f c2f96e686f
git -C "$TMPDIR" replace --graft aa4da4f189 912f075047
git -C "$TMPDIR" filter-repo --force \
--paths-from-file /dev/stdin \
--prune-empty always \
--prune-degenerate always \
--tag-callback 'tag.skip()' <<'EOF'
src/crypto/internal/fips140/edwards25519
src/crypto/internal/edwards25519
src/crypto/ed25519/internal/edwards25519
EOF
git fetch "$TMPDIR"
git update-ref "refs/heads/upstream/$TAG" FETCH_HEAD
echo
echo "Fetched upstream history up to $TAG. Merge with:"
echo -e "\tgit merge --no-ff --no-commit --allow-unrelated-histories upstream/$TAG"
+18 -9
View File
@@ -7,6 +7,7 @@ package edwards25519
import (
"encoding/binary"
"errors"
"math/bits"
)
// A Scalar is an integer modulo
@@ -179,15 +180,23 @@ func isReduced(s []byte) bool {
return false
}
for i := len(s) - 1; i >= 0; i-- {
switch {
case s[i] > scalarMinusOneBytes[i]:
return false
case s[i] < scalarMinusOneBytes[i]:
return true
}
}
return true
s0 := binary.LittleEndian.Uint64(s[:8])
s1 := binary.LittleEndian.Uint64(s[8:16])
s2 := binary.LittleEndian.Uint64(s[16:24])
s3 := binary.LittleEndian.Uint64(s[24:])
l0 := binary.LittleEndian.Uint64(scalarMinusOneBytes[:8])
l1 := binary.LittleEndian.Uint64(scalarMinusOneBytes[8:16])
l2 := binary.LittleEndian.Uint64(scalarMinusOneBytes[16:24])
l3 := binary.LittleEndian.Uint64(scalarMinusOneBytes[24:])
// Do a constant time subtraction chain scalarMinusOneBytes - s. If there is
// a borrow at the end, then s > scalarMinusOneBytes.
_, b := bits.Sub64(l0, s0, 0)
_, b = bits.Sub64(l1, s1, b)
_, b = bits.Sub64(l2, s2, b)
_, b = bits.Sub64(l3, s3, b)
return b == 0
}
// SetBytesWithClamping applies the buffer pruning described in RFC 8032,
+1 -3
View File
@@ -4,9 +4,7 @@
package edwards25519
import (
"crypto/subtle"
)
import "crypto/subtle"
// A dynamic lookup table for variable-base, constant-time scalar muls.
type projLookupTable struct {
+2 -2
View File
@@ -14,8 +14,8 @@
//
// [Antithesis Go SDK]: https://antithesis.com/docs/using_antithesis/sdk/go/
// [Antithesis platform]: https://antithesis.com
// [test properties]: https://antithesis.com/docs/using_antithesis/properties/
// [workload]: https://antithesis.com/docs/getting_started/first_test/
// [test properties]: https://antithesis.com/docs/properties_assertions/properties/
// [workload]: https://antithesis.com/docs/test_templates/first_test/
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
// [triage report]: https://antithesis.com/docs/reports/
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
+1 -1
View File
@@ -3,7 +3,7 @@ package internal
// --------------------------------------------------------------------------------
// Versions
// --------------------------------------------------------------------------------
const SDK_Version = "0.6.0"
const SDK_Version = "0.7.0"
const Protocol_Version = "1.1.0"
// --------------------------------------------------------------------------------
+7 -3
View File
@@ -18,8 +18,8 @@ Alex Snast <alexsn at fb.com>
Alexey Palazhchenko <alexey.palazhchenko at gmail.com>
Andrew Reid <andrew.reid at tixtrack.com>
Animesh Ray <mail.rayanimesh at gmail.com>
Arne Hormann <arnehormann at gmail.com>
Ariel Mashraki <ariel at mashraki.co.il>
Arne Hormann <arnehormann at gmail.com>
Artur Melanchyk <artur.melanchyk@gmail.com>
Asta Xie <xiemengjun at gmail.com>
B Lamarche <blam413 at gmail.com>
@@ -38,6 +38,7 @@ Daniel Montoya <dsmontoyam at gmail.com>
Daniel Nichter <nil at codenode.com>
Daniël van Eeden <git at myname.nl>
Dave Protasowski <dprotaso at gmail.com>
Demouth <yuya at demouth.net>
Diego Dupin <diego.dupin at gmail.com>
Dirkjan Bussink <d.bussink at gmail.com>
DisposaBoy <disposaboy at dby.me>
@@ -66,6 +67,7 @@ Jeff Hodges <jeff at somethingsimilar.com>
Jeffrey Charles <jeffreycharles at gmail.com>
Jennifer Purevsuren <jennifer at dolthub.com>
Jerome Meyer <jxmeyer at gmail.com>
Jiabin Zhang <jiabin.z at qq.com>
Jiajia Zhong <zhong2plus at gmail.com>
Jian Zhen <zhenjl at gmail.com>
Joe Mann <contact at joemann.co.uk>
@@ -85,10 +87,12 @@ Linh Tran Tuan <linhduonggnu at gmail.com>
Lion Yang <lion at aosc.xyz>
Luca Looz <luca.looz92 at gmail.com>
Lucas Liu <extrafliu at gmail.com>
Lunny Xiao <xiaolunwen at gmail.com>
Luke Scott <luke at webconnex.com>
Lunny Xiao <xiaolunwen at gmail.com>
Maciej Zimnoch <maciej.zimnoch at codilime.com>
Michael Woolnough <michael.woolnough at gmail.com>
Minh Quang <minhquang4334 at gmail.com>
Morgan Tocker <tocker at gmail.com>
Nao Yokotsuka <yokotukanao at gmail.com>
Nathanial Murphy <nathanial.murphy at gmail.com>
Nicola Peduzzi <thenikso at gmail.com>
@@ -99,7 +103,6 @@ Paul Bonser <misterpib at gmail.com>
Paulius Lozys <pauliuslozys at gmail.com>
Peter Schultz <peter.schultz at classmarkets.com>
Phil Porada <philporada at gmail.com>
Minh Quang <minhquang4334 at gmail.com>
Rebecca Chin <rchin at pivotal.io>
Reed Allman <rdallman10 at gmail.com>
Richard Wilkes <wilkes at me.com>
@@ -134,6 +137,7 @@ Ziheng Lyu <zihenglv at gmail.com>
# Organizations
Barracuda Networks, Inc.
Block, Inc.
Counting Ltd.
Defined Networking Inc.
DigitalOcean Inc.
+16 -3
View File
@@ -1,13 +1,26 @@
# Changelog
## v1.10.0 (2026-04-28)
* Fix `getSystemVar("max_allowed_packet")` potentially returned wrong value. (#1754)
This affects only when `maxAllowedPacket=0` is set.
* Bump filippo.io/edwards25519 from 1.1.1 to 1.2.0. (#1756)
While older versions have reported CVEs, they do not affect go-mysql.
* Update Go versions to 1.24-1.26. (#1763)
* Enhance interpolateParams to correctly handle placeholders. (#1732)
The question mark (?) within strings and comments will no longer be treated as a placeholder.
## v1.9.3 (2025-06-13)
* `tx.Commit()` and `tx.Rollback()` returned `ErrInvalidConn` always.
Now they return cached real error if present. (#1690)
* Optimize reading small resultsets to fix performance regression
introduced by compression protocol support. (#1707)
* Optimize reading small result sets to fix a performance regression
introduced by compression protocol support. (`#1707`)
* Fix `db.Ping()` on compressed connection. (#1723)
+5 -2
View File
@@ -1,5 +1,8 @@
# Go-MySQL-Driver
[![DeepWiki](https://img.shields.io/badge/DeepWiki-go--sql--driver%2Fmysql-blue.svg?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAyCAYAAAAnWDnqAAAAAXNSR0IArs4c6QAAA05JREFUaEPtmUtyEzEQhtWTQyQLHNak2AB7ZnyXZMEjXMGeK/AIi+QuHrMnbChYY7MIh8g01fJoopFb0uhhEqqcbWTp06/uv1saEDv4O3n3dV60RfP947Mm9/SQc0ICFQgzfc4CYZoTPAswgSJCCUJUnAAoRHOAUOcATwbmVLWdGoH//PB8mnKqScAhsD0kYP3j/Yt5LPQe2KvcXmGvRHcDnpxfL2zOYJ1mFwrryWTz0advv1Ut4CJgf5uhDuDj5eUcAUoahrdY/56ebRWeraTjMt/00Sh3UDtjgHtQNHwcRGOC98BJEAEymycmYcWwOprTgcB6VZ5JK5TAJ+fXGLBm3FDAmn6oPPjR4rKCAoJCal2eAiQp2x0vxTPB3ALO2CRkwmDy5WohzBDwSEFKRwPbknEggCPB/imwrycgxX2NzoMCHhPkDwqYMr9tRcP5qNrMZHkVnOjRMWwLCcr8ohBVb1OMjxLwGCvjTikrsBOiA6fNyCrm8V1rP93iVPpwaE+gO0SsWmPiXB+jikdf6SizrT5qKasx5j8ABbHpFTx+vFXp9EnYQmLx02h1QTTrl6eDqxLnGjporxl3NL3agEvXdT0WmEost648sQOYAeJS9Q7bfUVoMGnjo4AZdUMQku50McDcMWcBPvr0SzbTAFDfvJqwLzgxwATnCgnp4wDl6Aa+Ax283gghmj+vj7feE2KBBRMW3FzOpLOADl0Isb5587h/U4gGvkt5v60Z1VLG8BhYjbzRwyQZemwAd6cCR5/XFWLYZRIMpX39AR0tjaGGiGzLVyhse5C9RKC6ai42ppWPKiBagOvaYk8lO7DajerabOZP46Lby5wKjw1HCRx7p9sVMOWGzb/vA1hwiWc6jm3MvQDTogQkiqIhJV0nBQBTU+3okKCFDy9WwferkHjtxib7t3xIUQtHxnIwtx4mpg26/HfwVNVDb4oI9RHmx5WGelRVlrtiw43zboCLaxv46AZeB3IlTkwouebTr1y2NjSpHz68WNFjHvupy3q8TFn3Hos2IAk4Ju5dCo8B3wP7VPr/FGaKiG+T+v+TQqIrOqMTL1VdWV1DdmcbO8KXBz6esmYWYKPwDL5b5FA1a0hwapHiom0r/cKaoqr+27/XcrS5UwSMbQAAAABJRU5ErkJggg==)](https://deepwiki.com/go-sql-driver/mysql)
A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package
![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin")
@@ -42,8 +45,8 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac
## Requirements
* Go 1.21 or higher. We aim to support the 3 latest versions of Go.
* MySQL (5.7+) and MariaDB (10.5+) are supported.
* Go 1.24 or higher. We aim to support the 3 latest versions of Go.
* MySQL (5.7+) and MariaDB (10.5+) are supported by maintainers.
* [TiDB](https://github.com/pingcap/tidb) is supported by PingCAP.
* Do not ask questions about TiDB in our issue tracker or forum.
* [Document](https://docs.pingcap.com/tidb/v6.1/dev-guide-sample-application-golang)
+1 -1
View File
@@ -305,7 +305,7 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) {
if !mc.cfg.AllowNativePasswords {
return nil, ErrNativePassword
}
// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html
// https://dev.mysql.com/doc/dev/mysql-server/8.4.5/page_protocol_connection_phase_authentication_methods_native_password_authentication.html
// Native password authentication only need and will need 20-byte challenge.
authResp := scramblePassword(authData[:20], mc.cfg.Passwd)
return authResp, nil
-1
View File
@@ -7,7 +7,6 @@
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd || solaris || illumos
// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
package mysql
-1
View File
@@ -7,7 +7,6 @@
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build !linux && !darwin && !dragonfly && !freebsd && !netbsd && !openbsd && !solaris && !illumos
// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos
package mysql
+185 -90
View File
@@ -33,7 +33,8 @@ type mysqlConn struct {
connector *connector
maxAllowedPacket int
maxWriteSize int
flags clientFlag
capabilities capabilityFlag
extCapabilities extendedCapabilityFlag
status statusFlag
sequence uint8
compressSequence uint8
@@ -171,7 +172,7 @@ func (mc *mysqlConn) close() {
}
// Closes the network connection and unsets internal variables. Do not call this
// function after successfully authentication, call Close instead. This function
// function after successful authentication, call Close instead. This function
// is called before auth or on auth failure because MySQL will have already
// closed the network connection.
func (mc *mysqlConn) cleanup() {
@@ -223,13 +224,21 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
columnCount, err := stmt.readPrepareResultPacket()
if err == nil {
if stmt.paramCount > 0 {
if err = mc.readUntilEOF(); err != nil {
if err = mc.skipColumns(stmt.paramCount); err != nil {
return nil, err
}
}
if columnCount > 0 {
err = mc.readUntilEOF()
if mc.extCapabilities&clientCacheMetadata != 0 {
if stmt.columns, err = mc.readColumns(int(columnCount), nil); err != nil {
return nil, err
}
} else {
if err = mc.skipColumns(int(columnCount)); err != nil {
return nil, err
}
}
}
}
@@ -237,100 +246,184 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
}
func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) {
// Number of ? should be same to len(args)
if strings.Count(query, "?") != len(args) {
return "", driver.ErrSkip
}
noBackslashEscapes := (mc.status & statusNoBackslashEscapes) != 0
const (
stateNormal = iota
stateString
stateEscape
stateEOLComment
stateSlashStarComment
stateBacktick
)
const (
QUOTE_BYTE = byte('\'')
DBL_QUOTE_BYTE = byte('"')
BACKSLASH_BYTE = byte('\\')
QUESTION_MARK_BYTE = byte('?')
SLASH_BYTE = byte('/')
STAR_BYTE = byte('*')
HASH_BYTE = byte('#')
MINUS_BYTE = byte('-')
LINE_FEED_BYTE = byte('\n')
BACKTICK_BYTE = byte('`')
)
buf, err := mc.buf.takeCompleteBuffer()
if err != nil {
// can not take the buffer. Something must be wrong with the connection
mc.cleanup()
// interpolateParams would be called before sending any query.
// So its safe to retry.
return "", driver.ErrBadConn
}
buf = buf[:0]
state := stateNormal
singleQuotes := false
lastChar := byte(0)
argPos := 0
lenQuery := len(query)
lastIdx := 0
for i := 0; i < len(query); i++ {
q := strings.IndexByte(query[i:], '?')
if q == -1 {
buf = append(buf, query[i:]...)
break
}
buf = append(buf, query[i:i+q]...)
i += q
arg := args[argPos]
argPos++
if arg == nil {
buf = append(buf, "NULL"...)
for i := range lenQuery {
currentChar := query[i]
if state == stateEscape && !((currentChar == QUOTE_BYTE && singleQuotes) || (currentChar == DBL_QUOTE_BYTE && !singleQuotes)) {
state = stateString
lastChar = currentChar
continue
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case uint64:
// Handle uint64 explicitly because our custom ConvertValue emits unsigned values
buf = strconv.AppendUint(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
switch currentChar {
case STAR_BYTE:
if state == stateNormal && lastChar == SLASH_BYTE {
state = stateSlashStarComment
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
buf = append(buf, '\'')
buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate)
if err != nil {
return "", err
}
buf = append(buf, '\'')
case SLASH_BYTE:
if state == stateSlashStarComment && lastChar == STAR_BYTE {
state = stateNormal
// Clear lastChar so the '/' that closed the comment isn't
// reused to start a new comment with a following '*'.
lastChar = 0
continue
}
case json.RawMessage:
buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
} else {
buf = escapeBytesQuotes(buf, v)
case HASH_BYTE:
if state == stateNormal {
state = stateEOLComment
}
buf = append(buf, '\'')
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
buf = append(buf, "_binary'"...)
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
case MINUS_BYTE:
if state == stateNormal && lastChar == MINUS_BYTE {
// -- only starts a comment if followed by whitespace or control char
if i+1 < lenQuery {
nextChar := query[i+1]
if nextChar == ' ' || nextChar == '\t' || nextChar == '\n' || nextChar == '\r' {
state = stateEOLComment
}
} else {
buf = escapeBytesQuotes(buf, v)
state = stateEOLComment
}
buf = append(buf, '\'')
}
case string:
buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeStringBackslash(buf, v)
} else {
buf = escapeStringQuotes(buf, v)
case LINE_FEED_BYTE:
if state == stateEOLComment {
state = stateNormal
}
buf = append(buf, '\'')
default:
return "", driver.ErrSkip
}
case DBL_QUOTE_BYTE:
if state == stateNormal {
state = stateString
singleQuotes = false
} else if state == stateString && !singleQuotes {
state = stateNormal
} else if state == stateEscape {
state = stateString
}
case QUOTE_BYTE:
if state == stateNormal {
state = stateString
singleQuotes = true
} else if state == stateString && singleQuotes {
state = stateNormal
} else if state == stateEscape {
state = stateString
}
case BACKSLASH_BYTE:
if state == stateString && !noBackslashEscapes {
state = stateEscape
}
case QUESTION_MARK_BYTE:
if state == stateNormal {
if argPos >= len(args) {
return "", driver.ErrSkip
}
buf = append(buf, query[lastIdx:i]...)
arg := args[argPos]
argPos++
if len(buf)+4 > mc.maxAllowedPacket {
return "", driver.ErrSkip
if arg == nil {
buf = append(buf, "NULL"...)
lastIdx = i + 1
break
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case uint64:
buf = strconv.AppendUint(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
buf = append(buf, '\'')
buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate)
if err != nil {
return "", err
}
buf = append(buf, '\'')
}
case json.RawMessage:
if noBackslashEscapes {
buf = escapeBytesQuotes(buf, v, false)
} else {
buf = escapeBytesBackslash(buf, v, false)
}
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
if noBackslashEscapes {
buf = escapeBytesQuotes(buf, v, true)
} else {
buf = escapeBytesBackslash(buf, v, true)
}
}
case string:
if noBackslashEscapes {
buf = escapeStringQuotes(buf, v)
} else {
buf = escapeStringBackslash(buf, v)
}
default:
return "", driver.ErrSkip
}
if len(buf)+4 > mc.maxAllowedPacket {
return "", driver.ErrSkip
}
lastIdx = i + 1
}
case BACKTICK_BYTE:
if state == stateBacktick {
state = stateNormal
} else if state == stateNormal {
state = stateBacktick
}
}
lastChar = currentChar
}
buf = append(buf, query[lastIdx:]...)
if argPos != len(args) {
return "", driver.ErrSkip
}
@@ -370,19 +463,19 @@ func (mc *mysqlConn) exec(query string) error {
}
// Read Result
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, _, err := handleOk.readResultSetHeaderPacket()
if err != nil {
return err
}
if resLen > 0 {
// columns
if err := mc.readUntilEOF(); err != nil {
if err := mc.skipColumns(resLen); err != nil {
return err
}
// rows
if err := mc.readUntilEOF(); err != nil {
if err := mc.skipRows(); err != nil {
return err
}
}
@@ -419,7 +512,7 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
// Read Result
var resLen int
resLen, err = handleOk.readResultSetHeaderPacket()
resLen, _, err = handleOk.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
@@ -439,21 +532,20 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
}
// Columns
rows.rs.columns, err = mc.readColumns(resLen)
rows.rs.columns, err = mc.readColumns(resLen, nil)
return rows, err
}
// Gets the value of the given MySQL System Variable
// The returned byte slice is only valid until the next read
func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
func (mc *mysqlConn) getSystemVar(name string) (string, error) {
// Send command
handleOk := mc.clearResult()
if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil {
return nil, err
return "", err
}
// Read Result
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, _, err := handleOk.readResultSetHeaderPacket()
if err == nil {
rows := new(textRows)
rows.mc = mc
@@ -461,17 +553,20 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
if resLen > 0 {
// Columns
if err := mc.readUntilEOF(); err != nil {
return nil, err
if err := mc.skipColumns(resLen); err != nil {
return "", err
}
}
dest := make([]driver.Value, resLen)
if err = rows.readRow(dest); err == nil {
return dest[0].([]byte), mc.readUntilEOF()
// Convert to string before skipRows, which may
// overwrite the read buffer that dest[0] points into.
val := string(dest[0].([]byte))
return val, mc.skipRows()
}
}
return nil, err
return "", err
}
// cancel is called when the query has canceled.
+7 -5
View File
@@ -42,7 +42,7 @@ func encodeConnectionAttributes(cfg *Config) string {
}
// user-defined connection attributes
for _, connAttr := range strings.Split(cfg.ConnectionAttributes, ",") {
for connAttr := range strings.SplitSeq(cfg.ConnectionAttributes, ",") {
k, v, found := strings.Cut(connAttr, ":")
if !found {
continue
@@ -131,7 +131,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
mc.buf = newBuffer()
// Reading Handshake Initialization Packet
authData, plugin, err := mc.readHandshakePacket()
authData, serverCapabilities, serverExtCapabilities, plugin, err := mc.readHandshakePacket()
if err != nil {
mc.cleanup()
return nil, err
@@ -153,6 +153,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
return nil, err
}
}
mc.initCapabilities(serverCapabilities, serverExtCapabilities, mc.cfg)
if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil {
mc.cleanup()
return nil, err
@@ -161,13 +162,14 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
// Handle response to auth packet, switch methods if possible
if err = mc.handleAuthResult(authData, plugin); err != nil {
// Authentication failed and MySQL has already closed the connection
// (https://dev.mysql.com/doc/internals/en/authentication-fails.html).
// (https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase.html#sect_protocol_connection_phase_fast_path_fails).
// Do not send COM_QUIT, just cleanup and return the error.
mc.cleanup()
return nil, err
}
if mc.cfg.compress && mc.flags&clientCompress == clientCompress {
// compression is enabled after auth, not right after sending handshake response.
if mc.capabilities&clientCompress > 0 {
mc.compress = true
mc.compIO = newCompIO(mc)
}
@@ -180,7 +182,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
mc.Close()
return nil, err
}
n, err := strconv.Atoi(string(maxap))
n, err := strconv.Atoi(maxap)
if err != nil {
mc.Close()
return nil, fmt.Errorf("invalid max_allowed_packet value (%q): %w", maxap, err)
+17 -4
View File
@@ -32,7 +32,7 @@ const (
)
// MySQL constants documentation:
// http://dev.mysql.com/doc/internals/en/client-server-protocol.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_PROTOCOL.html
const (
iOK byte = 0x00
@@ -42,11 +42,12 @@ const (
iERR byte = 0xff
)
// https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags
type clientFlag uint32
// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__capabilities__flags.html
// https://mariadb.com/kb/en/connection/#capabilities
type capabilityFlag uint32
const (
clientLongPassword clientFlag = 1 << iota
clientMySQL capabilityFlag = 1 << iota
clientFoundRows
clientLongFlag
clientConnectWithDB
@@ -73,6 +74,18 @@ const (
clientDeprecateEOF
)
// https://mariadb.com/kb/en/connection/#capabilities
type extendedCapabilityFlag uint32
const (
progressIndicator extendedCapabilityFlag = 1 << iota
clientComMulti
clientStmtBulkOperations
clientExtendedMetadata
clientCacheMetadata
clientUnitBulkResult
)
const (
comQuit byte = iota + 1
comInitDB
+4 -5
View File
@@ -15,6 +15,7 @@ import (
"crypto/tls"
"errors"
"fmt"
"maps"
"math/big"
"net"
"net/url"
@@ -157,9 +158,7 @@ func (cfg *Config) Clone() *Config {
}
if len(cp.Params) > 0 {
cp.Params = make(map[string]string, len(cfg.Params))
for k, v := range cfg.Params {
cp.Params[k] = v
}
maps.Copy(cp.Params, cfg.Params)
}
if cfg.pubKey != nil {
cp.pubKey = &rsa.PublicKey{
@@ -414,7 +413,7 @@ func ParseDSN(dsn string) (cfg *Config, err error) {
if dsn[j] == '@' {
// username[:password]
// Find the first ':' in dsn[:j]
for k = 0; k < j; k++ {
for k = 0; k < j; k++ { // We cannot use k = range j here, because we use dsn[:k] below
if dsn[k] == ':' {
cfg.Passwd = dsn[k+1 : j]
break
@@ -477,7 +476,7 @@ func ParseDSN(dsn string) (cfg *Config, err error) {
// parseDSNParams parses the DSN "query string"
// Values must be url.QueryEscape'ed
func parseDSNParams(cfg *Config, params string) (err error) {
for _, v := range strings.Split(params, "&") {
for v := range strings.SplitSeq(params, "&") {
key, value, found := strings.Cut(v, "=")
if !found {
continue
+21 -17
View File
@@ -120,23 +120,24 @@ func (mf *mysqlField) typeDatabaseName() string {
}
var (
scanTypeFloat32 = reflect.TypeOf(float32(0))
scanTypeFloat64 = reflect.TypeOf(float64(0))
scanTypeInt8 = reflect.TypeOf(int8(0))
scanTypeInt16 = reflect.TypeOf(int16(0))
scanTypeInt32 = reflect.TypeOf(int32(0))
scanTypeInt64 = reflect.TypeOf(int64(0))
scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{})
scanTypeNullInt = reflect.TypeOf(sql.NullInt64{})
scanTypeNullTime = reflect.TypeOf(sql.NullTime{})
scanTypeUint8 = reflect.TypeOf(uint8(0))
scanTypeUint16 = reflect.TypeOf(uint16(0))
scanTypeUint32 = reflect.TypeOf(uint32(0))
scanTypeUint64 = reflect.TypeOf(uint64(0))
scanTypeString = reflect.TypeOf("")
scanTypeNullString = reflect.TypeOf(sql.NullString{})
scanTypeBytes = reflect.TypeOf([]byte{})
scanTypeUnknown = reflect.TypeOf(new(any))
scanTypeFloat32 = reflect.TypeFor[float32]()
scanTypeFloat64 = reflect.TypeFor[float64]()
scanTypeInt8 = reflect.TypeFor[int8]()
scanTypeInt16 = reflect.TypeFor[int16]()
scanTypeInt32 = reflect.TypeFor[int32]()
scanTypeInt64 = reflect.TypeFor[int64]()
scanTypeNullFloat = reflect.TypeFor[sql.NullFloat64]()
scanTypeNullInt = reflect.TypeFor[sql.NullInt64]()
scanTypeNullUint = reflect.TypeFor[sql.Null[uint64]]()
scanTypeNullTime = reflect.TypeFor[sql.NullTime]()
scanTypeUint8 = reflect.TypeFor[uint8]()
scanTypeUint16 = reflect.TypeFor[uint16]()
scanTypeUint32 = reflect.TypeFor[uint32]()
scanTypeUint64 = reflect.TypeFor[uint64]()
scanTypeString = reflect.TypeFor[string]()
scanTypeNullString = reflect.TypeFor[sql.NullString]()
scanTypeBytes = reflect.TypeFor[[]byte]()
scanTypeUnknown = reflect.TypeFor[*any]()
)
type mysqlField struct {
@@ -185,6 +186,9 @@ func (mf *mysqlField) scanType() reflect.Type {
}
return scanTypeInt64
}
if mf.flags&flagUnsigned != 0 {
return scanTypeNullUint
}
return scanTypeNullInt
case fieldTypeFloat:
+1 -4
View File
@@ -95,10 +95,7 @@ const defaultPacketSize = 16 * 1024 // 16KB is small enough for disk readahead a
func (mc *okHandler) handleInFileRequest(name string) (err error) {
var rdr io.Reader
packetSize := defaultPacketSize
if mc.maxWriteSize < packetSize {
packetSize = mc.maxWriteSize
}
packetSize := min(mc.maxWriteSize, defaultPacketSize)
if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader
// The server might return an an absolute path. See issue #355.
+205 -138
View File
@@ -179,20 +179,22 @@ func (mc *mysqlConn) writePacket(data []byte) error {
******************************************************************************/
// Handshake Initialization Packet
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake
func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) {
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_v10.html
// https://mariadb.com/kb/en/connection/#initial-handshake-packet
func (mc *mysqlConn) readHandshakePacket() (data []byte, capabilities capabilityFlag, extendedCapabilities extendedCapabilityFlag, plugin string, err error) {
data, err = mc.readPacket()
if err != nil {
return
}
if data[0] == iERR {
return nil, "", mc.handleErrorPacket(data)
err = mc.handleErrorPacket(data)
return
}
// protocol version [1 byte]
if data[0] < minProtocolVersion {
return nil, "", fmt.Errorf(
return nil, 0, 0, "", fmt.Errorf(
"unsupported protocol version %d. Version %d or higher is required",
data[0],
minProtocolVersion,
@@ -210,15 +212,15 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
pos += 8 + 1
// capability flags (lower 2 bytes) [2 bytes]
mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2]))
if mc.flags&clientProtocol41 == 0 {
return nil, "", ErrOldProtocol
capabilities = capabilityFlag(binary.LittleEndian.Uint16(data[pos : pos+2]))
if capabilities&clientProtocol41 == 0 {
return nil, capabilities, 0, "", ErrOldProtocol
}
if mc.flags&clientSSL == 0 && mc.cfg.TLS != nil {
if capabilities&clientSSL == 0 && mc.cfg.TLS != nil {
if mc.cfg.AllowFallbackToPlaintext {
mc.cfg.TLS = nil
} else {
return nil, "", ErrNoTLS
return nil, capabilities, 0, "", ErrNoTLS
}
}
pos += 2
@@ -228,11 +230,16 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
// status flags [2 bytes]
pos += 3
// capability flags (upper 2 bytes) [2 bytes]
mc.flags |= clientFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16
capabilities |= capabilityFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16
pos += 2
// length of auth-plugin-data [1 byte]
// reserved (all [00]) [10 bytes]
pos += 11
// reserved (all [00]) [6 bytes]
pos += 7
if capabilities&clientMySQL == 0 {
// MariaDB server extended flag
extendedCapabilities = extendedCapabilityFlag(binary.LittleEndian.Uint32(data[pos : pos+4]))
}
pos += 4
// second part of the password cipher [minimum 13 bytes],
// where len=MAX(13, length of auth-plugin-data - 8)
@@ -260,82 +267,72 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
// make a memory safe copy of the cipher slice
var b [20]byte
copy(b[:], authData)
return b[:], plugin, nil
return b[:], capabilities, extendedCapabilities, plugin, nil
}
// make a memory safe copy of the cipher slice
var b [8]byte
copy(b[:], authData)
return b[:], plugin, nil
return b[:], capabilities, 0, plugin, nil
}
// Client Authentication Packet
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse
func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error {
// Adjust client flags based on server support
clientFlags := clientProtocol41 |
clientSecureConn |
clientLongPassword |
clientTransactions |
clientLocalFiles |
clientPluginAuth |
clientMultiResults |
mc.flags&clientConnectAttrs |
mc.flags&clientLongFlag
// initCapabilities initializes the capabilities based on server support and configuration
func (mc *mysqlConn) initCapabilities(serverCapabilities capabilityFlag, serverExtCapabilities extendedCapabilityFlag, cfg *Config) {
clientCapabilities :=
clientMySQL |
clientLongFlag |
clientProtocol41 |
clientSecureConn |
clientTransactions |
clientPluginAuthLenEncClientData |
clientLocalFiles |
clientPluginAuth |
clientMultiResults |
clientConnectAttrs |
clientDeprecateEOF
sendConnectAttrs := mc.flags&clientConnectAttrs != 0
if mc.cfg.ClientFoundRows {
clientFlags |= clientFoundRows
if cfg.ClientFoundRows {
clientCapabilities |= clientFoundRows
}
if mc.cfg.compress && mc.flags&clientCompress == clientCompress {
clientFlags |= clientCompress
if cfg.compress {
clientCapabilities |= clientCompress
}
// To enable TLS / SSL
if mc.cfg.TLS != nil {
clientFlags |= clientSSL
clientCapabilities |= clientSSL
}
if mc.cfg.MultiStatements {
clientFlags |= clientMultiStatements
clientCapabilities |= clientMultiStatements
}
if n := len(cfg.DBName); n > 0 {
clientCapabilities |= clientConnectWithDB
}
// encode length of the auth plugin data
var authRespLEIBuf [9]byte
authRespLen := len(authResp)
authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen))
if len(authRespLEI) > 1 {
// if the length can not be written in 1 byte, it must be written as a
// length encoded integer
clientFlags |= clientPluginAuthLenEncClientData
}
// only keep client capabilities that server have
mc.capabilities = clientCapabilities & serverCapabilities
pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1
// set MariaDB extended clientCacheMetadata capability if server support it
mc.extCapabilities = clientCacheMetadata & serverExtCapabilities
}
// To specify a db name
if n := len(mc.cfg.DBName); n > 0 {
clientFlags |= clientConnectWithDB
pktLen += n + 1
}
// encode length of the connection attributes
var connAttrsLEI []byte
if sendConnectAttrs {
var connAttrsLEIBuf [9]byte
connAttrsLen := len(mc.connector.encodedAttributes)
connAttrsLEI = appendLengthEncodedInteger(connAttrsLEIBuf[:0], uint64(connAttrsLen))
pktLen += len(connAttrsLEI) + len(mc.connector.encodedAttributes)
}
// Calculate packet length and get buffer with that size
data, err := mc.buf.takeBuffer(pktLen + 4)
// Client Authentication Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_response.html
func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error {
// packet header 4
// capabilities 4
// maxPacketSize 4
// collation id 1
// filler 23
data, err := mc.buf.takeSmallBuffer(4*3 + 24)
if err != nil {
mc.cleanup()
return err
}
_ = data[4*3+23] // boundery check
// ClientFlags [32 bit]
binary.LittleEndian.PutUint32(data[4:], uint32(clientFlags))
// clientCapabilities [32 bit]
binary.LittleEndian.PutUint32(data[4:], uint32(mc.capabilities))
// MaxPacketSize [32 bit] (none)
binary.LittleEndian.PutUint32(data[8:], 0)
@@ -353,16 +350,26 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
}
// Filler [23 bytes] (all 0x00)
// or filler 19bytes + mariadb extCapabilities
pos := 13
for ; pos < 13+23; pos++ {
data[pos] = 0
if mc.capabilities&clientMySQL == 0 {
for ; pos < 13+19; pos++ {
data[pos] = 0
}
// MariaDB Extended Capabilities
binary.LittleEndian.PutUint32(data[13+19:], uint32(mc.extCapabilities))
} else {
for ; pos < 13+23; pos++ {
data[pos] = 0
}
}
// SSL Connection Request Packet
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_ssl_request.html
// https://mariadb.com/kb/en/connection/#sslrequest-packet
if mc.cfg.TLS != nil {
// Send TLS / SSL request packet
if err := mc.writePacket(data[:(4+4+1+23)+4]); err != nil {
if err := mc.writePacket(data); err != nil {
return err
}
@@ -379,37 +386,35 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
// User [null terminated string]
if len(mc.cfg.User) > 0 {
pos += copy(data[pos:], mc.cfg.User)
data = append(data, mc.cfg.User...)
}
data[pos] = 0x00
pos++
data = append(data, 0)
// Auth Data [length encoded integer]
pos += copy(data[pos:], authRespLEI)
pos += copy(data[pos:], authResp)
data = appendLengthEncodedInteger(data, uint64(len(authResp)))
data = append(data, authResp...)
// Databasename [null terminated string]
if len(mc.cfg.DBName) > 0 {
pos += copy(data[pos:], mc.cfg.DBName)
data[pos] = 0x00
pos++
// Database name [null terminated string]
if mc.capabilities&clientConnectWithDB != 0 {
data = append(data, mc.cfg.DBName...)
data = append(data, 0)
}
pos += copy(data[pos:], plugin)
data[pos] = 0x00
pos++
data = append(data, plugin...)
data = append(data, 0)
// Connection Attributes
if sendConnectAttrs {
pos += copy(data[pos:], connAttrsLEI)
pos += copy(data[pos:], []byte(mc.connector.encodedAttributes))
if mc.capabilities&clientConnectAttrs != 0 {
connAttrsLen := len(mc.connector.encodedAttributes)
data = appendLengthEncodedInteger(data, uint64(connAttrsLen))
data = append(data, mc.connector.encodedAttributes...)
}
// Send Auth packet
return mc.writePacket(data[:pos])
return mc.writePacket(data)
}
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_auth_switch_response.html
func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error {
pktLen := 4 + len(authData)
data, err := mc.buf.takeBuffer(pktLen)
@@ -511,7 +516,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) {
case iEOF:
if len(data) == 1 {
// https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_old_auth_switch_request.html
return nil, "mysql_old_password", nil
}
pluginEndIndex := bytes.IndexByte(data, 0x00)
@@ -545,36 +550,41 @@ func (mc *okHandler) readResultOK() error {
// Result Set Header Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response.html
func (mc *okHandler) readResultSetHeaderPacket() (int, error) {
func (mc *okHandler) readResultSetHeaderPacket() (int, bool, error) {
// handleOkPacket replaces both values; other cases leave the values unchanged.
mc.result.affectedRows = append(mc.result.affectedRows, 0)
mc.result.insertIds = append(mc.result.insertIds, 0)
data, err := mc.conn().readPacket()
if err != nil {
return 0, err
return 0, false, err
}
switch data[0] {
case iOK:
return 0, mc.handleOkPacket(data)
return 0, false, mc.handleOkPacket(data)
case iERR:
return 0, mc.conn().handleErrorPacket(data)
return 0, false, mc.conn().handleErrorPacket(data)
case iLocalInFile:
return 0, mc.handleInFileRequest(string(data[1:]))
return 0, false, mc.handleInFileRequest(string(data[1:]))
}
// column count
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html
num, _, _ := readLengthEncodedInteger(data)
// https://mariadb.com/kb/en/result-set-packets/#column-count-packet
num, _, len := readLengthEncodedInteger(data)
if mc.extCapabilities&clientCacheMetadata != 0 {
return int(num), data[len] == 0x01, nil
}
// ignore remaining data in the packet. see #1478.
return int(num), nil
return int(num), true, nil
}
// Error Packet
// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-ERR_Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html
func (mc *mysqlConn) handleErrorPacket(data []byte) error {
if data[0] != iERR {
return ErrMalformPkt
@@ -656,7 +666,7 @@ func (mc *mysqlConn) clearResult() *okHandler {
}
// Ok Packet
// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_ok_packet.html
func (mc *okHandler) handleOkPacket(data []byte) error {
var n, m int
var affectedRows, insertId uint64
@@ -690,24 +700,19 @@ func (mc *okHandler) handleOkPacket(data []byte) error {
}
// Read Packets as Field Packets until EOF-Packet or an Error appears
// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41
func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_column_definition.html#sect_protocol_com_query_response_text_resultset_column_definition_41
func (mc *mysqlConn) readColumns(count int, old []mysqlField) ([]mysqlField, error) {
columns := make([]mysqlField, count)
if len(old) != count {
old = nil
}
for i := 0; ; i++ {
for i := range count {
data, err := mc.readPacket()
if err != nil {
return nil, err
}
// EOF Packet
if data[0] == iEOF && (len(data) == 5 || len(data) == 1) {
if i == count {
return columns, nil
}
return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns))
}
// Catalog
pos, err := skipLengthEncodedString(data)
if err != nil {
@@ -728,7 +733,12 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
return nil, err
}
pos += n
columns[i].tableName = string(tableName)
if old != nil && old[i].tableName == string(tableName) {
// avoid allocating new string
columns[i].tableName = old[i].tableName
} else {
columns[i].tableName = string(tableName)
}
} else {
n, err = skipLengthEncodedString(data[pos:])
if err != nil {
@@ -749,7 +759,12 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
if err != nil {
return nil, err
}
columns[i].name = string(name)
if old != nil && old[i].name == string(name) {
// avoid allocating new string
columns[i].name = old[i].name
} else {
columns[i].name = string(name)
}
pos += n
// Original name [len coded string]
@@ -780,17 +795,17 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
// Decimals [uint8]
columns[i].decimals = data[pos]
//pos++
// Default value [len coded binary]
//if pos < len(data) {
// defaultVal, _, err = bytesToLengthCodedBinary(data[pos:])
//}
}
// skip EOF packet if client does not support deprecateEOF
if err := mc.skipEof(); err != nil {
return nil, err
}
return columns, nil
}
// Read Packets as Field Packets until EOF-Packet or an Error appears
// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::ResultsetRow
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_row.html
func (rows *textRows) readRow(dest []driver.Value) error {
mc := rows.mc
@@ -804,9 +819,20 @@ func (rows *textRows) readRow(dest []driver.Value) error {
}
// EOF Packet
if data[0] == iEOF && len(data) == 5 {
// server_status [2 bytes]
rows.mc.status = readStatus(data[3:])
// text row packets may starts with LengthEncodedString.
// In such case, 0xFE can mean string larger than 0xffffff.
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_dt_integers.html#sect_protocol_basic_dt_int_le
if data[0] == iEOF && len(data) <= 0xffffff {
if mc.capabilities&clientDeprecateEOF == 0 {
// Deprecated EOF packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_eof_packet.html
mc.status = readStatus(data[3:])
} else {
// Ok Packet with an 0xFE header
_, _, n := readLengthEncodedInteger(data[1:]) // affected_rows
_, _, m := readLengthEncodedInteger(data[1+n:]) // last_insert_id
mc.status = readStatus(data[1+n+m:])
}
rows.rs.done = true
if !rows.HasNextResultSet() {
rows.mc = nil
@@ -880,8 +906,34 @@ func (rows *textRows) readRow(dest []driver.Value) error {
return nil
}
// Reads Packets until EOF-Packet or an Error appears. Returns count of Packets read
func (mc *mysqlConn) readUntilEOF() error {
func (mc *mysqlConn) skipPackets(n int) error {
for range n {
if _, err := mc.readPacket(); err != nil {
return err
}
}
return nil
}
// skips EOF packet after n * ColumnDefinition packets when clientDeprecateEOF is not set
func (mc *mysqlConn) skipEof() error {
if mc.capabilities&clientDeprecateEOF == 0 {
if _, err := mc.readPacket(); err != nil {
return err
}
}
return nil
}
func (mc *mysqlConn) skipColumns(n int) error {
if err := mc.skipPackets(n); err != nil {
return err
}
return mc.skipEof()
}
// Reads Packets until EOF-Packet or an Error appears.
func (mc *mysqlConn) skipRows() error {
for {
data, err := mc.readPacket()
if err != nil {
@@ -892,10 +944,20 @@ func (mc *mysqlConn) readUntilEOF() error {
case iERR:
return mc.handleErrorPacket(data)
case iEOF:
if len(data) == 5 {
mc.status = readStatus(data[3:])
// text row packets may starts with LengthEncodedString.
// In such case, 0xFE can mean string larger than 0xffffff.
if len(data) <= 0xffffff {
if mc.capabilities&clientDeprecateEOF == 0 {
// EOF packet
mc.status = readStatus(data[3:])
} else {
// OK packet with an 0xFE header
_, _, n := readLengthEncodedInteger(data[1:]) // affected_rows
_, _, m := readLengthEncodedInteger(data[1+n:]) // last_insert_id
mc.status = readStatus(data[1+n+m:])
}
return nil
}
return nil
}
}
}
@@ -905,7 +967,7 @@ func (mc *mysqlConn) readUntilEOF() error {
******************************************************************************/
// Prepare Result Packets
// http://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_prepare.html#sect_protocol_com_stmt_prepare_response
func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) {
data, err := stmt.mc.readPacket()
if err == nil {
@@ -932,7 +994,7 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) {
return 0, err
}
// http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_send_long_data.html
func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error {
maxLen := stmt.mc.maxAllowedPacket - 1
pktLen := maxLen
@@ -979,7 +1041,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error {
}
// Execute Prepared Statement
// http://dev.mysql.com/doc/internals/en/com-stmt-execute.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html
func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
if len(args) != stmt.paramCount {
return fmt.Errorf(
@@ -993,10 +1055,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
mc := stmt.mc
// Determine threshold dynamically to avoid packet size shortage.
longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1)
if longDataSize < 64 {
longDataSize = 64
}
longDataSize := max(mc.maxAllowedPacket/(stmt.paramCount+1), 64)
// Reset packet-sequence
mc.resetSequence()
@@ -1185,17 +1244,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
// mc.affectedRows and mc.insertIds.
func (mc *okHandler) discardResults() error {
for mc.status&statusMoreResultsExists != 0 {
resLen, err := mc.readResultSetHeaderPacket()
resLen, _, err := mc.readResultSetHeaderPacket()
if err != nil {
return err
}
if resLen > 0 {
// columns
if err := mc.conn().readUntilEOF(); err != nil {
if err := mc.conn().skipColumns(resLen); err != nil {
return err
}
// rows
if err := mc.conn().readUntilEOF(); err != nil {
if err := mc.conn().skipRows(); err != nil {
return err
}
}
@@ -1203,7 +1262,7 @@ func (mc *okHandler) discardResults() error {
return nil
}
// http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row
func (rows *binaryRows) readRow(dest []driver.Value) error {
data, err := rows.mc.readPacket()
if err != nil {
@@ -1212,9 +1271,17 @@ func (rows *binaryRows) readRow(dest []driver.Value) error {
// packet indicator [1 byte]
if data[0] != iOK {
// EOF Packet
if data[0] == iEOF && len(data) == 5 {
rows.mc.status = readStatus(data[3:])
// EOF/OK Packet
if data[0] == iEOF {
if rows.mc.capabilities&clientDeprecateEOF == 0 {
// EOF packet
rows.mc.status = readStatus(data[3:])
} else {
// OK Packet with an 0xFE header
_, _, n := readLengthEncodedInteger(data[1:])
_, _, m := readLengthEncodedInteger(data[1+n:])
rows.mc.status = readStatus(data[1+n+m:])
}
rows.rs.done = true
if !rows.HasNextResultSet() {
rows.mc = nil
+4 -2
View File
@@ -8,6 +8,8 @@
package mysql
import "slices"
import "database/sql/driver"
// Result exposes data not available through *connection.Result.
@@ -42,9 +44,9 @@ func (res *mysqlResult) RowsAffected() (int64, error) {
}
func (res *mysqlResult) AllLastInsertIds() []int64 {
return append([]int64{}, res.insertIds...) // defensive copy
return slices.Clone(res.insertIds) // defensive copy
}
func (res *mysqlResult) AllRowsAffected() []int64 {
return append([]int64{}, res.affectedRows...) // defensive copy
return slices.Clone(res.affectedRows) // defensive copy
}
+5 -5
View File
@@ -113,7 +113,7 @@ func (rows *mysqlRows) Close() (err error) {
// Remove unread packets from stream
if !rows.rs.done {
err = mc.readUntilEOF()
err = mc.skipRows()
}
if err == nil {
handleOk := mc.clearResult()
@@ -143,7 +143,7 @@ func (rows *mysqlRows) nextResultSet() (int, error) {
// Remove unread packets from stream
if !rows.rs.done {
if err := rows.mc.readUntilEOF(); err != nil {
if err := rows.mc.skipRows(); err != nil {
return 0, err
}
rows.rs.done = true
@@ -156,7 +156,7 @@ func (rows *mysqlRows) nextResultSet() (int, error) {
rows.rs = resultSet{}
// rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to
// nextResultSet.
resLen, err := rows.mc.resultUnchanged().readResultSetHeaderPacket()
resLen, _, err := rows.mc.resultUnchanged().readResultSetHeaderPacket()
if err != nil {
// Clean up about multi-results flag
rows.rs.done = true
@@ -186,7 +186,7 @@ func (rows *binaryRows) NextResultSet() error {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
rows.rs.columns, err = rows.mc.readColumns(resLen, nil)
return err
}
@@ -208,7 +208,7 @@ func (rows *textRows) NextResultSet() (err error) {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
rows.rs.columns, err = rows.mc.readColumns(resLen, nil)
return err
}
+26 -8
View File
@@ -20,6 +20,7 @@ type mysqlStmt struct {
mc *mysqlConn
id uint32
paramCount int
columns []mysqlField
}
func (stmt *mysqlStmt) Close() error {
@@ -64,19 +65,26 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) {
handleOk := stmt.mc.clearResult()
// Read Result
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, metadataFollows, err := handleOk.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
if resLen > 0 {
// Columns
if err = mc.readUntilEOF(); err != nil {
return nil, err
if metadataFollows && stmt.mc.extCapabilities&clientCacheMetadata != 0 {
// we can not skip column metadata because next stmt.Query() may use it.
if stmt.columns, err = mc.readColumns(resLen, stmt.columns); err != nil {
return nil, err
}
} else {
if err = mc.skipColumns(resLen); err != nil {
return nil, err
}
}
// Rows
if err := mc.readUntilEOF(); err != nil {
if err = mc.skipRows(); err != nil {
return nil, err
}
}
@@ -107,7 +115,7 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
// Read Result
handleOk := stmt.mc.clearResult()
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, metadataFollows, err := handleOk.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
@@ -116,7 +124,17 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
if resLen > 0 {
rows.mc = mc
rows.rs.columns, err = mc.readColumns(resLen)
if metadataFollows {
if rows.rs.columns, err = mc.readColumns(resLen, stmt.columns); err != nil {
return nil, err
}
stmt.columns = rows.rs.columns
} else {
if err = mc.skipEof(); err != nil {
return nil, err
}
rows.rs.columns = stmt.columns
}
} else {
rows.rs.done = true
@@ -131,7 +149,7 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
return rows, err
}
var jsonType = reflect.TypeOf(json.RawMessage{})
var jsonType = reflect.TypeFor[json.RawMessage]()
type converter struct{}
@@ -193,7 +211,7 @@ func (c converter) ConvertValue(v any) (driver.Value, error) {
return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind())
}
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
var valuerReflectType = reflect.TypeFor[driver.Valuer]()
// callValuerValue returns vr.Value(), with one exception:
// If vr.Value is an auto-generated method on a pointer type and the
+65 -90
View File
@@ -182,7 +182,7 @@ func parseDateTime(b []byte, loc *time.Location) (time.Time, error) {
func parseByteYear(b []byte) (int, error) {
year, n := 0, 1000
for i := 0; i < 4; i++ {
for i := range 4 {
v, err := bToi(b[i])
if err != nil {
return 0, err
@@ -207,7 +207,7 @@ func parseByte2Digits(b1, b2 byte) (int, error) {
func parseByteNanoSec(b []byte) (int, error) {
ns, digit := 0, 100000 // max is 6-digits
for i := 0; i < len(b); i++ {
for i := range b {
v, err := bToi(b[i])
if err != nil {
return 0, err
@@ -625,108 +625,80 @@ func reserveBuffer(buf []byte, appendSize int) []byte {
return buf[:newSize]
}
// escapeBytesBackslash escapes []byte with backslashes (\)
// This escapes the contents of a string (provided as []byte) by adding backslashes before special
// characters, and turning others into specific escape sequences, such as
// turning newlines into \n and null bytes into \0.
// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L823-L932
func escapeBytesBackslash(buf, v []byte) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
// Lookup table for backslash escapes (used for both string and bytes)
var backslashEscapeTable [256]byte
for _, c := range v {
switch c {
case '\x00':
buf[pos+1] = '0'
buf[pos] = '\\'
pos += 2
case '\n':
buf[pos+1] = 'n'
buf[pos] = '\\'
pos += 2
case '\r':
buf[pos+1] = 'r'
buf[pos] = '\\'
pos += 2
case '\x1a':
buf[pos+1] = 'Z'
buf[pos] = '\\'
pos += 2
case '\'':
buf[pos+1] = '\''
buf[pos] = '\\'
pos += 2
case '"':
buf[pos+1] = '"'
buf[pos] = '\\'
pos += 2
case '\\':
buf[pos+1] = '\\'
buf[pos] = '\\'
pos += 2
default:
buf[pos] = c
pos++
}
}
return buf[:pos]
func init() {
backslashEscapeTable['\x00'] = '0'
backslashEscapeTable['\n'] = 'n'
backslashEscapeTable['\r'] = 'r'
backslashEscapeTable['\x1a'] = 'Z'
backslashEscapeTable['\''] = '\''
backslashEscapeTable['"'] = '"'
backslashEscapeTable['\\'] = '\\'
}
// escapeStringBackslash is similar to escapeBytesBackslash but for string.
func escapeStringBackslash(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
for i := 0; i < len(v); i++ {
c := v[i]
switch c {
case '\x00':
buf[pos+1] = '0'
if esc := backslashEscapeTable[c]; esc != 0 {
buf[pos+1] = esc
buf[pos] = '\\'
pos += 2
case '\n':
buf[pos+1] = 'n'
buf[pos] = '\\'
pos += 2
case '\r':
buf[pos+1] = 'r'
buf[pos] = '\\'
pos += 2
case '\x1a':
buf[pos+1] = 'Z'
buf[pos] = '\\'
pos += 2
case '\'':
buf[pos+1] = '\''
buf[pos] = '\\'
pos += 2
case '"':
buf[pos+1] = '"'
buf[pos] = '\\'
pos += 2
case '\\':
buf[pos+1] = '\\'
buf[pos] = '\\'
pos += 2
default:
} else {
buf[pos] = c
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
// escapeBytesQuotes escapes apostrophes in []byte by doubling them up.
// This escapes the contents of a string by doubling up any apostrophes that
// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in
// effect on the server.
// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L963-L1038
func escapeBytesQuotes(buf, v []byte) []byte {
// escapeBytesBackslash appends _binary'...' or '...' with backslash escaping for bytes.
func escapeBytesBackslash(buf, v []byte, binary bool) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
if binary {
buf = reserveBuffer(buf, len(v)*2+9)
copy(buf[pos:], []byte("_binary'"))
pos += 8
} else {
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
}
for _, c := range v {
if esc := backslashEscapeTable[c]; esc != 0 {
buf[pos+1] = esc
buf[pos] = '\\'
pos += 2
} else {
buf[pos] = c
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
// escapeBytesQuotes appends _binary'...' or '...' with single-quote escaping for bytes.
func escapeBytesQuotes(buf, v []byte, binary bool) []byte {
pos := len(buf)
if binary {
buf = reserveBuffer(buf, len(v)*2+9)
copy(buf[pos:], []byte("_binary'"))
pos += 8
} else {
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
}
for _, c := range v {
if c == '\'' {
buf[pos+1] = '\''
@@ -737,16 +709,18 @@ func escapeBytesQuotes(buf, v []byte) []byte {
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
// escapeStringQuotes is similar to escapeBytesQuotes but for string.
func escapeStringQuotes(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for i := 0; i < len(v); i++ {
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
for i := range len(v) {
c := v[i]
if c == '\'' {
buf[pos+1] = '\''
@@ -757,7 +731,8 @@ func escapeStringQuotes(buf []byte, v string) []byte {
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
+1 -1
View File
@@ -1,4 +1,4 @@
Copyright (c) 2016 HashiCorp, Inc.
Copyright IBM Corp. 2016, 2025
Mozilla Public License, version 2.0
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) HashiCorp, Inc.
# Copyright IBM Corp. 2016, 2025
# SPDX-License-Identifier: MPL-2.0
version: v1
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) HashiCorp, Inc.
# Copyright IBM Corp. 2016, 2025
# SPDX-License-Identifier: MPL-2.0
version: v1
+6 -4
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
@@ -669,7 +669,7 @@ func (c *Client) Start() (addr net.Addr, err error) {
// Setup a temporary certificate for client/server mtls, and send the public
// certificate to the plugin.
if c.config.AutoMTLS {
c.logger.Info("configuring client automatic mTLS")
c.logger.Debug("configuring client automatic mTLS")
certPEM, keyPEM, err := generateCert()
if err != nil {
c.logger.Error("failed to generate client certificate", "error", err)
@@ -753,9 +753,12 @@ func (c *Client) Start() (addr net.Addr, err error) {
// Create a context for when we kill
c.doneCtx, c.ctxCancel = context.WithCancel(context.Background())
// Add two to pipesWaitGroup: one for logStderr, one for the goroutine
// below that consumes Stdout. We mustn't continue to Add once we might Wait.
c.pipesWaitGroup.Add(2)
// Start goroutine that logs the stderr
c.clientWaitGroup.Add(1)
c.pipesWaitGroup.Add(1)
// logStderr calls c.pipesWaitGroup.Done()
go c.logStderr(runner.Name(), runner.Stderr())
@@ -791,7 +794,6 @@ func (c *Client) Start() (addr net.Addr, err error) {
// out of stdout
linesCh := make(chan string)
c.clientWaitGroup.Add(1)
c.pipesWaitGroup.Add(1)
go func() {
defer c.clientWaitGroup.Done()
defer c.pipesWaitGroup.Done()
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+1 -2
View File
@@ -1,8 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
//go:build !windows
// +build !windows
package cmdrunner
+1 -2
View File
@@ -1,8 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
//go:build windows
// +build windows
package cmdrunner
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+6 -2
View File
@@ -1,8 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
//go:build !windows
// +build !windows
package cmdrunner
@@ -16,6 +15,11 @@ import (
func _pidAlive(pid int) bool {
proc, err := os.FindProcess(pid)
if err == nil {
// On Linux with Go 1.23+, FindProcess opens a pidfd which must be
// released or it leaks an FD on every call. Release errors are
// intentionally ignored; the handle is short-lived and there's
// nothing actionable to recover from a release failure.
defer func() { _ = proc.Release() }()
err = proc.Signal(syscall.Signal(0))
}
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
+2 -2
View File
@@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
syntax = "proto3";
package plugin;
option go_package = "./plugin";
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
message ConnInfo {
uint32 service_id = 1;
@@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
syntax = "proto3";
package plugin;
option go_package = "./plugin";
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
message Empty {
}
+2 -2
View File
@@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
syntax = "proto3";
package plugin;
option go_package = "./plugin";
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
import "google/protobuf/empty.proto";
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
// The plugin package exposes functions and helpers for communicating to
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package runner
+2 -2
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
@@ -307,7 +307,7 @@ func Serve(opts *ServeConfig) {
// If the client is configured using AutoMTLS, the certificate will be here,
// and we need to generate our own in response.
if tlsConfig == nil && clientCert != "" {
logger.Info("configuring server automatic mTLS")
logger.Debug("configuring server automatic mTLS")
clientCertPool := x509.NewCertPool()
if !clientCertPool.AppendCertsFromPEM([]byte(clientCert)) {
logger.Error("client cert provided but failed to parse", "cert", clientCert)
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+5
View File
@@ -175,6 +175,11 @@ func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName stri
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+17 -5
View File
@@ -150,6 +150,11 @@ type UploadInfo struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string
ChecksumXXHash3 string
ChecksumXXHash128 string
ChecksumMode string
}
@@ -168,11 +173,12 @@ type ObjectInfo struct {
// each parts concatenated into one string.
ETag string `json:"etag"`
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
ContentEncoding string `json:"contentEncoding"` // A standard MIME type describing encoding of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
// Collection of additional metadata on the object.
// eg: x-amz-meta-*, content-encoding etc.
@@ -226,6 +232,12 @@ type ObjectInfo struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string
ChecksumXXHash3 string
ChecksumXXHash128 string
ChecksumAlgorithm string
ChecksumMode string `xml:"ChecksumType"`
Internal *struct {
+96 -10
View File
@@ -23,6 +23,7 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -88,10 +89,17 @@ type ObjectAttributesResponse struct {
StorageClass string
ObjectSize int
Checksum struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
ChecksumType string `xml:",omitempty"`
}
ObjectParts struct {
PartsCount int
@@ -105,12 +113,90 @@ type ObjectAttributesResponse struct {
// ObjectAttributePart is used by ObjectAttributesResponse to describe an object part
type ObjectAttributePart struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
PartNumber int
Size int
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
PartNumber int
Size int
}
// ChecksumMap returns a map of checksums for the object.
func (o *ObjectAttributesResponse) ChecksumMap() map[string]string {
res := make(map[string]string)
setif := func(typ ChecksumType, value string) {
if value != "" {
res[typ.Key()] = value
}
}
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
return res
}
// ChecksumMode returns the checksum mode of the object.
// If unable to determine, returns ChecksumUnknownMode.
func (o *ObjectAttributesResponse) ChecksumMode() ChecksumMode {
t := o.ChecksumType()
if !t.IsSet() {
return ChecksumUnknownMode
}
switch o.Checksum.ChecksumType {
case amzChecksumModeComposite:
return ChecksumCompositeMode
case amzChecksumModeFullObject:
return ChecksumFullObjectMode
case "":
// Likely not supported by the server.
if o.Checksum.ChecksumCRC64NVME != "" || !strings.ContainsRune(o.ETag, '-') {
// Always full object.
return ChecksumFullObjectMode
}
if !t.CanMergeCRC() {
// Only composite possible.
return ChecksumCompositeMode
}
}
return ChecksumUnknownMode
}
// ChecksumType returns the checksum type of the object.
// If none is set, returns ChecksumNone.
func (o *ObjectAttributesResponse) ChecksumType() ChecksumType {
t := ChecksumNone
setif := func(typ ChecksumType, value string) {
if value != "" {
t = typ
}
}
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
if t.IsSet() && o.Checksum.ChecksumType == amzChecksumModeFullObject {
t |= ChecksumFullObject
}
return t
}
func (o *ObjectAttributes) parseResponse(resp *http.Response) (err error) {
+7 -1
View File
@@ -404,7 +404,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
}
var (
keyMarker = ""
keyMarker = opts.StartAfter
versionIDMarker = ""
preName = ""
preKey = ""
@@ -432,12 +432,18 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
UserMetadata: version.UserMetadata,
Internal: version.Internal,
NumVersions: numVersions,
ChecksumAlgorithm: version.ChecksumAlgorithm,
ChecksumMode: version.ChecksumType,
ChecksumCRC32: version.ChecksumCRC32,
ChecksumCRC32C: version.ChecksumCRC32C,
ChecksumSHA1: version.ChecksumSHA1,
ChecksumSHA256: version.ChecksumSHA256,
ChecksumCRC64NVME: version.ChecksumCRC64NVME,
ChecksumMD5: version.ChecksumMD5,
ChecksumSHA512: version.ChecksumSHA512,
ChecksumXXHash64: version.ChecksumXXHash64,
ChecksumXXHash3: version.ChecksumXXHash3,
ChecksumXXHash128: version.ChecksumXXHash128,
}
if !yield(info) {
return false
+15
View File
@@ -192,6 +192,11 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -353,6 +358,11 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
}
objPart.Size = p.size
objPart.PartNumber = p.partNumber
@@ -458,6 +468,11 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME,
ChecksumMD5: completeMultipartUploadResult.ChecksumMD5,
ChecksumSHA512: completeMultipartUploadResult.ChecksumSHA512,
ChecksumXXHash64: completeMultipartUploadResult.ChecksumXXHash64,
ChecksumXXHash3: completeMultipartUploadResult.ChecksumXXHash3,
ChecksumXXHash128: completeMultipartUploadResult.ChecksumXXHash128,
ChecksumMode: completeMultipartUploadResult.ChecksumType,
}, nil
}
+20
View File
@@ -259,6 +259,11 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME,
ChecksumMD5: uploadRes.Part.ChecksumMD5,
ChecksumSHA512: uploadRes.Part.ChecksumSHA512,
ChecksumXXHash64: uploadRes.Part.ChecksumXXHash64,
ChecksumXXHash3: uploadRes.Part.ChecksumXXHash3,
ChecksumXXHash128: uploadRes.Part.ChecksumXXHash128,
})
}
}
@@ -418,6 +423,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -617,6 +627,11 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -796,6 +811,11 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+5
View File
@@ -502,6 +502,11 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
+41
View File
@@ -113,6 +113,12 @@ type Version struct {
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
ChecksumAlgorithm string `xml:",omitempty"`
ChecksumType string `xml:",omitempty"`
isDeleteMarker bool
@@ -297,6 +303,11 @@ type ObjectPart struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
}
// Checksum will return the checksum for the given type.
@@ -313,6 +324,16 @@ func (c ObjectPart) Checksum(t ChecksumType) string {
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
case t.Is(ChecksumMD5):
return c.ChecksumMD5
case t.Is(ChecksumSHA512):
return c.ChecksumSHA512
case t.Is(ChecksumXXHash64):
return c.ChecksumXXHash64
case t.Is(ChecksumXXHash3):
return c.ChecksumXXHash3
case t.Is(ChecksumXXHash128):
return c.ChecksumXXHash128
}
return ""
}
@@ -382,6 +403,11 @@ type completeMultipartUploadResult struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string `xml:"ChecksumXXHASH64"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128"`
ChecksumType string
}
@@ -398,6 +424,11 @@ type CompletePart struct {
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
}
// Checksum will return the checksum for the given type.
@@ -414,6 +445,16 @@ func (c CompletePart) Checksum(t ChecksumType) string {
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
case t.Is(ChecksumMD5):
return c.ChecksumMD5
case t.Is(ChecksumSHA512):
return c.ChecksumSHA512
case t.Is(ChecksumXXHash64):
return c.ChecksumXXHash64
case t.Is(ChecksumXXHash3):
return c.ChecksumXXHash3
case t.Is(ChecksumXXHash128):
return c.ChecksumXXHash128
}
return ""
}
+17 -9
View File
@@ -52,6 +52,12 @@ type UpdateObjectEncryptionOptions struct {
VersionID string
}
// UpdateObjectEncryptionResult holds the result of an UpdateObjectEncryption call.
type UpdateObjectEncryptionResult struct {
// VersionID is the version ID of the object that was updated, if versioning is enabled.
VersionID string
}
// UpdateObjectEncryption changes the encryption configuration of an existing object in-place.
// The object must already be encrypted with SSE-S3 or SSE-KMS. SSE-C objects are not supported.
// This operation rotates the data encryption key envelope without re-reading/re-writing object data.
@@ -62,19 +68,19 @@ type UpdateObjectEncryptionOptions struct {
// - objectName: Name of the object
// - opts: Options including KMSKeyArn (required), optional BucketKeyEnabled, and optional VersionID
//
// Returns an error if the operation fails.
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) error {
// Returns the version ID of the updated object (if versioning is enabled) and an error if the operation fails.
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) (UpdateObjectEncryptionResult, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if opts.KMSKeyArn == "" {
return errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
return UpdateObjectEncryptionResult{}, errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
}
// Get resources properly escaped and lined up before
@@ -96,7 +102,7 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
bodyData, err := xml.Marshal(reqBody)
if err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
reqMetadata := requestMetadata{
@@ -113,10 +119,12 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, objectName)
return UpdateObjectEncryptionResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
}
return nil
return UpdateObjectEncryptionResult{
VersionID: resp.Header.Get(amzVersionID),
}, nil
}
+13 -1
View File
@@ -198,6 +198,9 @@ func New(endpoint string, opts *Options) (*Client, error) {
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
// for backwards compatibility.
clnt.s3DualstackEnabled = true
} else if s3utils.IsAmazonOutpostsEndpoint(*clnt.endpointURL) {
// S3 on Outposts uses signature v4 with service name s3-outposts.
clnt.overrideSignerType = credentials.SignatureV4
}
return clnt, nil
@@ -912,7 +915,11 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
} else if signerType.IsV4() {
// Presign URL with signature v4.
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.PreSignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
} else {
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
}
}
return req, nil
}
@@ -971,6 +978,9 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
req = signer.StreamingSignV4Express(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.StreamingSignV4Outposts(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
} else {
req = signer.StreamingSignV4(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
@@ -991,6 +1001,8 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.SignV4TrailerOutposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
} else {
// Add signature version '4' authorization header.
req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
+6 -1
View File
@@ -209,6 +209,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
}
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
region := getDefaultLocation(*c.endpointURL, c.region)
req = signer.SignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, region)
} else {
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
}
return req, nil
}
+67 -4
View File
@@ -18,8 +18,10 @@
package minio
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/binary"
"errors"
@@ -30,8 +32,10 @@ import (
"sort"
"strings"
"github.com/cespare/xxhash/v2"
"github.com/klauspost/crc32"
"github.com/minio/crc64nvme"
"github.com/zeebo/xxh3"
)
// ChecksumMode contains information about the checksum mode on the object
@@ -49,6 +53,9 @@ const (
// checksumModeMask is a mask for valid checksum mode types.
checksumModeMask = checksumLastMode - 1
// ChecksumUnknownMode indicates no or unknown checksum mode.
ChecksumUnknownMode ChecksumMode = 0
)
// Is returns if c is all of t.
@@ -64,9 +71,9 @@ func (c ChecksumMode) Key() string {
func (c ChecksumMode) String() string {
switch c & checksumModeMask {
case ChecksumFullObjectMode:
return "FULL_OBJECT"
return amzChecksumModeFullObject
case ChecksumCompositeMode:
return "COMPOSITE"
return amzChecksumModeComposite
}
return ""
}
@@ -86,6 +93,16 @@ const (
ChecksumCRC32C
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
ChecksumCRC64NVME
// ChecksumMD5 indicates an MD5 checksum.
ChecksumMD5
// ChecksumSHA512 indicates a SHA-512 checksum.
ChecksumSHA512
// ChecksumXXHash64 indicates an XXHash64 checksum.
ChecksumXXHash64
// ChecksumXXHash3 indicates an XXH3-64 checksum.
ChecksumXXHash3
// ChecksumXXHash128 indicates an XXH3-128 checksum.
ChecksumXXHash128
// Keep after all valid checksums
checksumLast
@@ -112,7 +129,15 @@ const (
amzChecksumSHA1 = "x-amz-checksum-sha1"
amzChecksumSHA256 = "x-amz-checksum-sha256"
amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme"
amzChecksumMD5 = "x-amz-checksum-md5"
amzChecksumSHA512 = "x-amz-checksum-sha512"
amzChecksumXXHash64 = "x-amz-checksum-xxhash64"
amzChecksumXXHash3 = "x-amz-checksum-xxhash3"
amzChecksumXXHash128 = "x-amz-checksum-xxhash128"
amzChecksumMode = "x-amz-checksum-type"
amzChecksumModeComposite = "COMPOSITE"
amzChecksumModeFullObject = "FULL_OBJECT"
)
// Base returns the base type, without modifiers.
@@ -139,6 +164,16 @@ func (c ChecksumType) Key() string {
return amzChecksumSHA256
case ChecksumCRC64NVME:
return amzChecksumCRC64NVME
case ChecksumMD5:
return amzChecksumMD5
case ChecksumSHA512:
return amzChecksumSHA512
case ChecksumXXHash64:
return amzChecksumXXHash64
case ChecksumXXHash3:
return amzChecksumXXHash3
case ChecksumXXHash128:
return amzChecksumXXHash128
}
return ""
}
@@ -146,7 +181,8 @@ func (c ChecksumType) Key() string {
// CanComposite will return if the checksum type can be used for composite multipart upload on AWS.
func (c ChecksumType) CanComposite() bool {
switch c & checksumMask {
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C:
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C,
ChecksumMD5, ChecksumSHA512, ChecksumXXHash64, ChecksumXXHash3, ChecksumXXHash128:
return true
}
return false
@@ -186,6 +222,14 @@ func (c ChecksumType) RawByteLen() int {
return sha256.Size
case ChecksumCRC64NVME:
return crc64nvme.Size
case ChecksumXXHash64, ChecksumXXHash3:
return 8
case ChecksumMD5:
return md5.Size
case ChecksumSHA512:
return sha512.Size
case ChecksumXXHash128:
return 16
}
return 0
}
@@ -206,6 +250,16 @@ func (c ChecksumType) Hasher() hash.Hash {
return sha256.New()
case ChecksumCRC64NVME:
return crc64nvme.New()
case ChecksumMD5:
return md5.New()
case ChecksumSHA512:
return sha512.New()
case ChecksumXXHash64:
return xxhash.New()
case ChecksumXXHash3:
return xxh3.New()
case ChecksumXXHash128:
return xxh3.New128()
}
return nil
}
@@ -233,7 +287,6 @@ func (c ChecksumType) EncodeToString(b []byte) string {
}
// String returns the type as a string.
// CRC32, CRC32C, SHA1, and SHA256 for valid values.
// Empty string for unset and "<invalid>" if not valid.
func (c ChecksumType) String() string {
switch c & checksumMask {
@@ -249,6 +302,16 @@ func (c ChecksumType) String() string {
return ""
case ChecksumCRC64NVME:
return "CRC64NVME"
case ChecksumMD5:
return "MD5"
case ChecksumSHA512:
return "SHA512"
case ChecksumXXHash64:
return "XXHASH64"
case ChecksumXXHash3:
return "XXHASH3"
case ChecksumXXHash128:
return "XXHASH128"
}
return "<invalid>"
}
+4 -4
View File
@@ -94,15 +94,15 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
},
"us-iso-east-1": {
"s3.us-iso-east-1.c2s.ic.gov",
"s3.dualstack.us-iso-east-1.c2s.ic.gov",
"", // dualstack endpoint doesn't exist
},
"us-isob-east-1": {
"s3.us-isob-east-1.sc2s.sgov.gov",
"s3.dualstack.us-isob-east-1.sc2s.sgov.gov",
"", // dualstack endpoint doesn't exist
},
"us-iso-west-1": {
"s3.us-iso-west-1.c2s.ic.gov",
"s3.dualstack.us-iso-west-1.c2s.ic.gov",
"", // dualstack endpoint doesn't exist
},
"us-west-2": {
"s3.us-west-2.amazonaws.com",
@@ -269,7 +269,7 @@ func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
}
return "s3.us-east-1.amazonaws.com"
}
if useDualstack {
if useDualstack && s3Endpoint.dualstackEndpoint != "" {
return s3Endpoint.dualstackEndpoint
}
return s3Endpoint.endpoint
+160 -4
View File
@@ -73,8 +73,25 @@ const (
enableKMS = "ENABLE_KMS"
appVersion = "0.1.0"
skipCERTValidation = "SKIP_CERT_VALIDATION"
// TODO: remove when server supports the 2026 checksum types.
ignore2026Checksums = true
)
func ignore2026ChecksumError(cs minio.ChecksumType, err error) bool {
if !ignore2026Checksums {
return false
}
switch cs.Base() {
case minio.ChecksumMD5, minio.ChecksumSHA512,
minio.ChecksumXXHash64, minio.ChecksumXXHash3, minio.ChecksumXXHash128:
default:
return false
}
var er minio.ErrorResponse
return errors.As(err, &er) && er.Code == "InvalidArgument"
}
func createHTTPTransport() (transport *http.Transport) {
var err error
transport, err = minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
@@ -2096,6 +2113,11 @@ func testPutObjectWithChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
@@ -2160,6 +2182,10 @@ func testPutObjectWithChecksums() {
UserMetadata: meta,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2168,6 +2194,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
@@ -2191,6 +2222,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if st.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
@@ -2238,6 +2274,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
cmpChecksum(st.ChecksumMD5, "")
cmpChecksum(st.ChecksumSHA512, "")
cmpChecksum(st.ChecksumXXHash64, "")
cmpChecksum(st.ChecksumXXHash3, "")
cmpChecksum(st.ChecksumXXHash128, "")
delete(args, "range")
delete(args, "metadata")
@@ -2283,6 +2324,11 @@ func testPutObjectWithTrailingChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumCRC64NVME},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
@@ -2329,6 +2375,10 @@ func testPutObjectWithTrailingChecksums() {
Checksum: test.cs,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2341,6 +2391,11 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@@ -2361,7 +2416,12 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@@ -2407,6 +2467,11 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
cmpChecksum(st.ChecksumMD5, "")
cmpChecksum(st.ChecksumSHA512, "")
cmpChecksum(st.ChecksumXXHash64, "")
cmpChecksum(st.ChecksumXXHash3, "")
cmpChecksum(st.ChecksumXXHash128, "")
function = "GetObjectAttributes(...)"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
@@ -2418,6 +2483,16 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(s.Checksum.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(s.Checksum.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(s.Checksum.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(s.Checksum.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(s.Checksum.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(s.Checksum.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if s.Checksum.ChecksumType != "" && s.Checksum.ChecksumType != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "ChecksumType mismatch in GetObjectAttributes", fmt.Errorf("want %s, got %s", minio.ChecksumFullObjectMode.String(), s.Checksum.ChecksumType))
return
}
delete(args, "range")
delete(args, "metadata")
@@ -2495,6 +2570,11 @@ func testPutMultipartObjectWithChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumFullObjectCRC32},
{cs: minio.ChecksumFullObjectCRC32C},
{cs: minio.ChecksumCRC64NVME},
@@ -2552,6 +2632,10 @@ func testPutMultipartObjectWithChecksums() {
Checksum: cs,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2567,6 +2651,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(resp.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(resp.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(resp.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(resp.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(resp.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(resp.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(resp.ChecksumXXHash128, wantChksm)
}
args["section"] = "HeadObject"
@@ -2586,6 +2680,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(st.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantChksm)
}
// Use the CopyObject API to make a copy, in the case it was a composite checksum,
@@ -2622,6 +2726,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm)
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantFullObjectChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantFullObjectChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantFullObjectChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantFullObjectChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantFullObjectChksm)
}
args["section"] = "GetObjectAttributes"
@@ -2644,6 +2758,35 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(s.Checksum.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(s.Checksum.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(s.Checksum.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(s.Checksum.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(s.Checksum.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(s.Checksum.ChecksumXXHash128, wantChksm)
}
if s.Checksum.ChecksumType != "" {
var wantType string
if test.cs.FullObjectRequested() {
wantType = minio.ChecksumFullObjectMode.String()
} else {
wantType = minio.ChecksumCompositeMode.String()
}
cmpChecksum(s.Checksum.ChecksumType, wantType)
}
for _, part := range s.ObjectParts.Parts {
if test.cs == minio.ChecksumCRC64NVME && part.ChecksumCRC64NVME == "" {
logError(testName, function, args, startTime, "", "Part missing CRC64NVME checksum in GetObjectAttributes", fmt.Errorf("part %d", part.PartNumber))
return
}
}
// Read the data back
@@ -2684,6 +2827,16 @@ func testPutMultipartObjectWithChecksums() {
if st.ChecksumCRC64NVME != "" {
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
}
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantChksm)
}
delete(args, "metadata")
@@ -3475,6 +3628,8 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
checksumFound = true
} else if v.ChecksumCRC32C != "" {
checksumFound = true
} else if v.ChecksumCRC64NVME != "" {
checksumFound = true
}
if !checksumFound {
partsMissingChecksum = true
@@ -3497,6 +3652,7 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" ||
OA.Checksum.ChecksumCRC32C != "" ||
OA.Checksum.ChecksumCRC64NVME != "" ||
OA.Checksum.ChecksumSHA1 != "" ||
OA.Checksum.ChecksumSHA256 != "")
@@ -14789,6 +14945,9 @@ func main() {
// execute tests
if isFullMode() {
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums()
testCopyObjectWithChecksums()
testReplaceObjectWithChecksums()
testCorsSetGetDelete()
@@ -14808,9 +14967,6 @@ func main() {
testComposeObjectErrorCasesV2()
testCompose10KSourcesV2()
testUserMetadataCopyingV2()
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums()
testPutObject0ByteV2()
testPutObjectMetadataNonUSASCIIV2()
testPutObjectNoLengthV2()
+21
View File
@@ -119,6 +119,10 @@ var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`)
// amazonS3HostPrivateLink - regular expression used to determine if an arg is s3 host in AWS PrivateLink interface endpoints style
var amazonS3HostPrivateLink = regexp.MustCompile(`^(?:bucket|accesspoint).vpce-.*?.s3.(.*?).vpce.amazonaws.com$`)
// amazonS3HostOutposts - regular expression used to determine if an arg is S3 on Outposts endpoint.
// Pattern: <something>.s3-outposts.<region>.amazonaws.com
var amazonS3HostOutposts = regexp.MustCompile(`^(.+)\.s3-outposts\.([a-z0-9-]+)\.amazonaws\.com$`)
// GetRegionFromURL - returns a region from url host.
func GetRegionFromURL(endpointURL url.URL) string {
if endpointURL == sentinelURL {
@@ -181,6 +185,11 @@ func GetRegionFromURL(endpointURL url.URL) string {
return parts[1]
}
parts = amazonS3HostOutposts.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 2 {
return parts[2]
}
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
if strings.HasPrefix(parts[1], "xpress-") {
@@ -210,8 +219,20 @@ func IsAmazonExpressZonalEndpoint(endpointURL url.URL) bool {
return amazonS3HostExpress.MatchString(endpointURL.Hostname())
}
// IsAmazonOutpostsEndpoint - Match if the endpoint is S3 on Outposts endpoint.
func IsAmazonOutpostsEndpoint(endpointURL url.URL) bool {
if endpointURL == sentinelURL {
return false
}
return amazonS3HostOutposts.MatchString(endpointURL.Hostname())
}
// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint.
// S3 on Outposts is not treated as Amazon S3 here so that the client keeps path-style and does not replace the host.
func IsAmazonEndpoint(endpointURL url.URL) bool {
if IsAmazonOutpostsEndpoint(endpointURL) {
return false
}
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" || endpointURL.Hostname() == "s3.amazonaws.com" {
return true
}
@@ -90,32 +90,34 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
return streamLen
}
// buildChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
// buildChunkStringToSignWithService - like buildChunkStringToSign but with configurable service type.
func buildChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
if serviceType == "" {
serviceType = ServiceTypeS3
}
stringToSignParts := []string{
streamingPayloadHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
getScope(region, t, serviceType),
previousSig,
emptySHA256,
chunkChecksum,
}
return strings.Join(stringToSignParts, "\n")
}
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
// buildTrailerChunkStringToSignWithService - like buildTrailerChunkStringToSign but with configurable service type.
func buildTrailerChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
if serviceType == "" {
serviceType = ServiceTypeS3
}
stringToSignParts := []string{
streamingTrailerHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
getScope(region, t, serviceType),
previousSig,
chunkChecksum,
}
return strings.Join(stringToSignParts, "\n")
}
@@ -150,36 +152,42 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
// serviceType defaults to ServiceTypeS3 when empty.
func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
previousSignature, secretAccessKey string,
previousSignature, secretAccessKey, serviceType string,
) string {
chunkStringToSign := buildChunkStringToSign(reqTime, region,
previousSignature, chunkCheckSum)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
if serviceType == "" {
serviceType = ServiceTypeS3
}
chunkStringToSign := buildChunkStringToSignWithService(reqTime, region,
previousSignature, chunkCheckSum, serviceType)
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
return getSignature(signingKey, chunkStringToSign)
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
// buildTrailerChunkSignature - returns chunk signature for trailer chunk.
// serviceType defaults to ServiceTypeS3 when empty.
func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
previousSignature, secretAccessKey string,
previousSignature, secretAccessKey, serviceType string,
) string {
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
previousSignature, chunkChecksum)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
if serviceType == "" {
serviceType = ServiceTypeS3
}
chunkStringToSign := buildTrailerChunkStringToSignWithService(reqTime, region,
previousSignature, chunkChecksum, serviceType)
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
return getSignature(signingKey, chunkStringToSign)
}
// getSeedSignature - returns the seed signature for a given request.
func (s *StreamingReader) setSeedSignature(req *http.Request) {
// Get canonical request
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req))
// Get string to sign from canonical request.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3)
// Calculate signature.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, serviceType)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, serviceType)
s.seedSignature = getSignature(signingKey, stringToSign)
}
@@ -190,6 +198,7 @@ type StreamingReader struct {
secretAccessKey string
sessionToken string
region string
serviceType string // e.g. ServiceTypeS3, ServiceTypeS3Outposts; empty means S3
prevSignature string
seedSignature string
contentLen int64 // Content-Length from req header
@@ -214,8 +223,12 @@ func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
s.sh256.Write(s.chunkBuf[:chunkLen])
chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
signature := buildChunkSignature(chunckChecksum, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
s.region, s.prevSignature, s.secretAccessKey, serviceType)
// For next chunk signature computation
s.prevSignature = signature
@@ -249,9 +262,12 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
s.sh256.Reset()
s.sh256.Write(s.chunkBuf)
chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
// Compute chunk signature
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
s.region, s.prevSignature, s.secretAccessKey, serviceType)
// For next chunk signature computation
s.prevSignature = signature
@@ -376,6 +392,40 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
return req
}
// StreamingSignV4Outposts - provides chunked upload signatureV4 support for S3 on Outposts (service name s3-outposts).
func StreamingSignV4Outposts(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
) *http.Request {
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
if req.Body == nil {
req.Body = io.NopCloser(bytes.NewReader([]byte("")))
}
stReader := &StreamingReader{
baseReadCloser: req.Body,
accessKeyID: accessKeyID,
secretAccessKey: secretAccessKey,
sessionToken: sessionToken,
region: region,
serviceType: ServiceTypeS3Outposts,
reqTime: reqTime,
chunkBuf: make([]byte, payloadChunkSize),
contentLen: dataLen,
chunkNum: 1,
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
lastChunkSize: int(dataLen % payloadChunkSize),
sh256: sh256,
}
if len(req.Trailer) > 0 {
stReader.trailer = req.Trailer
req.Trailer = nil
}
stReader.setSeedSignature(req)
stReader.setStreamingAuthHeader(req, ServiceTypeS3Outposts)
stReader.prevSignature = stReader.seedSignature
req.Body = stReader
return req
}
// Read - this method performs chunk upload signature providing a
// io.Reader interface.
func (s *StreamingReader) Read(buf []byte) (int, error) {
+51 -3
View File
@@ -38,9 +38,10 @@ const (
// Different service types
const (
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
ServiceTypeS3Express = "s3express"
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
ServiceTypeS3Express = "s3express"
ServiceTypeS3Outposts = "s3-outposts"
)
// Excerpts from @lsegal -
@@ -256,6 +257,38 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
return &req
}
// PreSignV4Outposts presign the request for S3 on Outposts (service name s3-outposts).
func PreSignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
// Presign is not needed for anonymous credentials.
if accessKeyID == "" || secretAccessKey == "" {
return &req
}
t := time.Now().UTC()
credential := GetCredential(accessKeyID, location, t, ServiceTypeS3Outposts)
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
query := req.URL.Query()
query.Set("X-Amz-Algorithm", signV4Algorithm)
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
query.Set("X-Amz-SignedHeaders", signedHeaders)
query.Set("X-Amz-Credential", credential)
if sessionToken != "" {
if v := req.Header.Get("x-amz-s3session-token"); v != "" {
query.Set("X-Amz-S3session-Token", sessionToken)
} else {
query.Set("X-Amz-Security-Token", sessionToken)
}
}
req.URL.RawQuery = query.Encode()
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))
stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3Outposts)
signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3Outposts)
signature := getSignature(signingKey, stringToSign)
req.URL.RawQuery += "&X-Amz-Signature=" + signature
return &req
}
// PostPresignSignatureV4 - presigned signature for PostPolicy
// requests.
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
@@ -393,3 +426,18 @@ func SignV4TrailerExpress(req http.Request, accessKeyID, secretAccessKey, sessio
func SignV4Trailer(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, trailer)
}
// SignV4Outposts sign the request for S3 on Outposts (service name s3-outposts).
func SignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, nil)
}
// SignV4WithServiceType signs a request with AWS Signature Version 4 using a custom service type.
func SignV4WithServiceType(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, serviceType, nil)
}
// SignV4TrailerOutposts sign the request with trailer for S3 on Outposts (service name s3-outposts).
func SignV4TrailerOutposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, trailer)
}
+21
View File
@@ -18,6 +18,7 @@
package tags
import (
"encoding/json"
"encoding/xml"
"io"
"net/url"
@@ -293,6 +294,26 @@ func (tags Tags) ToMap() map[string]string {
return tags.TagSet.toMap()
}
// MarshalJSON encodes Tags as a flat JSON object {"key":"value",...}.
func (tags Tags) MarshalJSON() ([]byte, error) {
return json.Marshal(tags.ToMap())
}
// UnmarshalJSON decodes a flat JSON object {"key":"value",...} into Tags.
func (tags *Tags) UnmarshalJSON(data []byte) error {
var m map[string]string
if err := json.Unmarshal(data, &m); err != nil {
return err
}
if tags.TagSet == nil {
tags.TagSet = &tagSet{
tagMap: make(map[string]string),
}
}
tags.TagSet.tagMap = m
return nil
}
// MapToObjectTags converts an input map of key and value into
// *Tags data structure with validation.
func MapToObjectTags(tagMap map[string]string) (*Tags, error) {
+27
View File
@@ -179,6 +179,11 @@ func isValidEndpointURL(endpointURL url.URL) error {
return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
}
}
if strings.Contains(host, "s3-outposts") {
if !s3utils.IsAmazonOutpostsEndpoint(endpointURL) {
return errInvalidArgument("S3 Outposts endpoint must match <prefix>.s3-outposts.<region>.amazonaws.com")
}
}
return nil
}
@@ -304,6 +309,21 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Region: h.Get("x-amz-bucket-region"),
}
}
mtimeStr := h.Get("X-Minio-Source-Mtime")
if mtimeStr != "" {
mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
if err != nil {
return ObjectInfo{}, ErrorResponse{
Code: InternalError,
Message: fmt.Sprintf("X-Minio-Source-Mtime is not in supported format: %v", err),
BucketName: bucketName,
Key: objectName,
RequestID: h.Get("x-amz-request-id"),
HostID: h.Get("x-amz-id-2"),
Region: h.Get("x-amz-bucket-region"),
}
}
}
// Fetch content type if any present.
contentType := strings.TrimSpace(h.Get("Content-Type"))
@@ -381,6 +401,7 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Size: size,
LastModified: mtime,
ContentType: contentType,
ContentEncoding: strings.TrimSpace(h.Get("Content-Encoding")),
Expires: expiry,
VersionID: h.Get(amzVersionID),
IsDeleteMarker: deleteMarker,
@@ -402,6 +423,12 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumAlgorithm: h.Get(amzChecksumAlgo),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+33 -15
View File
@@ -696,7 +696,7 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
// If we are here we have an auth callout defined and we have failed auth so far
// so we will callout to our auth backend for processing.
if !skip {
authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy)
authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy, ujwt)
}
// Check if we are authorized and in the auth callout account, and if so add in deny publish permissions for the auth subject.
if authorized {
@@ -797,26 +797,42 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
token = opts.Authorization
}
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
if s.trustedKeys != nil {
ujwt = c.opts.JWT
if ujwt == _EMPTY_ && c.isMqtt() {
// For MQTT, we pass the password as the JWT too, but do so here so it's not
// publicly exposed in the client options if it isn't a JWT.
// MQTT can carry JWTs in the password field. Reconstruct it here for auth
// processing and auth callout, but do not populate c.opts.JWT yet or it would
// be exposed through monitoring and advisory paths even when the password is
// not actually a JWT.
if ujwt == _EMPTY_ && c.isMqtt() && c.opts.JWT == _EMPTY_ {
// Don't set juc here, leave that to the next s.trustedKeys != nil block,
// so that we don't try to trust a JWT when we aren't in operator mode. We
// will allow it to be passed through auth callout though.
if _, err := jwt.DecodeUserClaims(c.opts.Password); err == nil {
ujwt = c.opts.Password
}
if ujwt == _EMPTY_ && opts.DefaultSentinel != _EMPTY_ {
c.opts.JWT = opts.DefaultSentinel
ujwt = c.opts.JWT
}
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
if s.trustedKeys != nil {
if ujwt == _EMPTY_ {
// Need to be sure that it's a NATS JWT, otherwise we will not correctly
// attempt the default sentinel below.
if _, err = jwt.DecodeUserClaims(c.opts.JWT); err == nil {
ujwt = c.opts.JWT
}
}
if ujwt == _EMPTY_ {
// Didn't fall through with a valid NATS JWT, so try the default sentinel
// if configured.
if opts.DefaultSentinel != _EMPTY_ {
c.opts.JWT = opts.DefaultSentinel
ujwt = c.opts.JWT
}
}
if ujwt == _EMPTY_ {
s.mu.Unlock()
c.Debugf("Authentication requires a user JWT")
return false
}
// So we have a valid user jwt here.
juc, err = jwt.DecodeUserClaims(ujwt)
if err != nil {
if juc, err = jwt.DecodeUserClaims(ujwt); err != nil {
s.mu.Unlock()
c.Debugf("User JWT not valid: %v", err)
return false
@@ -1015,8 +1031,10 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
c.Debugf("Connection type not allowed")
return false
}
// skip validation of nonce when presented with a bearer token
// FIXME: if BearerToken is only for WSS, need check for server with that port enabled
// Skip validation of nonce when presented with a bearer token.
// While support for bearer tokens was added for WebSockets, there is no
// security benefit in restricting their use to that client protocol: the
// client can just go use the other protocol.
if !juc.BearerToken {
// Verify the signature against the nonce.
if c.opts.Sig == _EMPTY_ {
+10 -4
View File
@@ -41,7 +41,7 @@ func titleCase(m string) string {
}
// Process a callout on this client's behalf.
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool) (authorized bool, errStr string) {
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool, ujwt string) (authorized bool, errStr string) {
isOperatorMode := len(opts.TrustedKeys) > 0
// this is the account the user connected in, or the one running the callout
@@ -374,7 +374,7 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequi
// Grab client info for the request.
c.mu.Lock()
c.fillClientInfo(&claim.ClientInformation)
c.fillConnectOpts(&claim.ConnectOptions)
c.fillConnectOpts(&claim.ConnectOptions, ujwt)
// If we have a sig in the client opts, fill in nonce.
if claim.ConnectOptions.SignedNonce != _EMPTY_ {
claim.ClientInformation.Nonce = string(c.nonce)
@@ -474,16 +474,22 @@ func (c *client) fillClientInfo(ci *jwt.ClientInformation) {
// Fill in client options.
// Lock should be held.
func (c *client) fillConnectOpts(opts *jwt.ConnectOptions) {
func (c *client) fillConnectOpts(opts *jwt.ConnectOptions, ujwt string) {
if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) {
return
}
o := c.opts
if ujwt == _EMPTY_ {
// The caller may supply a reconstructed JWT that should be sent to auth
// callout without storing it in c.opts.JWT. If not, fall back to the client
// option as before.
ujwt = o.JWT
}
// Do it this way to fail to compile if fields are added to jwt.ClientInformation.
*opts = jwt.ConnectOptions{
JWT: o.JWT,
JWT: ujwt,
Nkey: o.Nkey,
SignedNonce: o.Sig,
Token: o.Token,
+2 -2
View File
@@ -239,7 +239,7 @@ func (ss SequenceSet) EncodeLen() int {
return minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2))
}
func (ss SequenceSet) Encode(buf []byte) ([]byte, error) {
func (ss SequenceSet) Encode(buf []byte) []byte {
nn, encLen := ss.Nodes(), ss.EncodeLen()
if cap(buf) < encLen {
@@ -268,7 +268,7 @@ func (ss SequenceSet) Encode(buf []byte) ([]byte, error) {
le.PutUint16(buf[i:], uint16(n.h))
i += 2
})
return buf[:i], nil
return buf[:i]
}
// ErrBadEncoding is returned when we can not decode properly.

Some files were not shown because too many files have changed in this diff Show More