[full-ci] chore: bump reva to v2.44.0 update opencloud version 6.2.0 (#2734)

This commit is contained in:
Viktor Scharf
2026-05-11 17:08:39 +02:00
committed by GitHub
parent a84820865e
commit cf0ec50da9
171 changed files with 15555 additions and 4260 deletions
+2 -2
View File
@@ -14,8 +14,8 @@
//
// [Antithesis Go SDK]: https://antithesis.com/docs/using_antithesis/sdk/go/
// [Antithesis platform]: https://antithesis.com
// [test properties]: https://antithesis.com/docs/using_antithesis/properties/
// [workload]: https://antithesis.com/docs/getting_started/first_test/
// [test properties]: https://antithesis.com/docs/properties_assertions/properties/
// [workload]: https://antithesis.com/docs/test_templates/first_test/
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
// [triage report]: https://antithesis.com/docs/reports/
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
+1 -1
View File
@@ -3,7 +3,7 @@ package internal
// --------------------------------------------------------------------------------
// Versions
// --------------------------------------------------------------------------------
const SDK_Version = "0.6.0"
const SDK_Version = "0.7.0"
const Protocol_Version = "1.1.0"
// --------------------------------------------------------------------------------
+7 -3
View File
@@ -18,8 +18,8 @@ Alex Snast <alexsn at fb.com>
Alexey Palazhchenko <alexey.palazhchenko at gmail.com>
Andrew Reid <andrew.reid at tixtrack.com>
Animesh Ray <mail.rayanimesh at gmail.com>
Arne Hormann <arnehormann at gmail.com>
Ariel Mashraki <ariel at mashraki.co.il>
Arne Hormann <arnehormann at gmail.com>
Artur Melanchyk <artur.melanchyk@gmail.com>
Asta Xie <xiemengjun at gmail.com>
B Lamarche <blam413 at gmail.com>
@@ -38,6 +38,7 @@ Daniel Montoya <dsmontoyam at gmail.com>
Daniel Nichter <nil at codenode.com>
Daniël van Eeden <git at myname.nl>
Dave Protasowski <dprotaso at gmail.com>
Demouth <yuya at demouth.net>
Diego Dupin <diego.dupin at gmail.com>
Dirkjan Bussink <d.bussink at gmail.com>
DisposaBoy <disposaboy at dby.me>
@@ -66,6 +67,7 @@ Jeff Hodges <jeff at somethingsimilar.com>
Jeffrey Charles <jeffreycharles at gmail.com>
Jennifer Purevsuren <jennifer at dolthub.com>
Jerome Meyer <jxmeyer at gmail.com>
Jiabin Zhang <jiabin.z at qq.com>
Jiajia Zhong <zhong2plus at gmail.com>
Jian Zhen <zhenjl at gmail.com>
Joe Mann <contact at joemann.co.uk>
@@ -85,10 +87,12 @@ Linh Tran Tuan <linhduonggnu at gmail.com>
Lion Yang <lion at aosc.xyz>
Luca Looz <luca.looz92 at gmail.com>
Lucas Liu <extrafliu at gmail.com>
Lunny Xiao <xiaolunwen at gmail.com>
Luke Scott <luke at webconnex.com>
Lunny Xiao <xiaolunwen at gmail.com>
Maciej Zimnoch <maciej.zimnoch at codilime.com>
Michael Woolnough <michael.woolnough at gmail.com>
Minh Quang <minhquang4334 at gmail.com>
Morgan Tocker <tocker at gmail.com>
Nao Yokotsuka <yokotukanao at gmail.com>
Nathanial Murphy <nathanial.murphy at gmail.com>
Nicola Peduzzi <thenikso at gmail.com>
@@ -99,7 +103,6 @@ Paul Bonser <misterpib at gmail.com>
Paulius Lozys <pauliuslozys at gmail.com>
Peter Schultz <peter.schultz at classmarkets.com>
Phil Porada <philporada at gmail.com>
Minh Quang <minhquang4334 at gmail.com>
Rebecca Chin <rchin at pivotal.io>
Reed Allman <rdallman10 at gmail.com>
Richard Wilkes <wilkes at me.com>
@@ -134,6 +137,7 @@ Ziheng Lyu <zihenglv at gmail.com>
# Organizations
Barracuda Networks, Inc.
Block, Inc.
Counting Ltd.
Defined Networking Inc.
DigitalOcean Inc.
+16 -3
View File
@@ -1,13 +1,26 @@
# Changelog
## v1.10.0 (2026-04-28)
* Fix `getSystemVar("max_allowed_packet")` potentially returned wrong value. (#1754)
This affects only when `maxAllowedPacket=0` is set.
* Bump filippo.io/edwards25519 from 1.1.1 to 1.2.0. (#1756)
While older versions have reported CVEs, they do not affect go-mysql.
* Update Go versions to 1.24-1.26. (#1763)
* Enhance interpolateParams to correctly handle placeholders. (#1732)
The question mark (?) within strings and comments will no longer be treated as a placeholder.
## v1.9.3 (2025-06-13)
* `tx.Commit()` and `tx.Rollback()` returned `ErrInvalidConn` always.
Now they return cached real error if present. (#1690)
* Optimize reading small resultsets to fix performance regression
introduced by compression protocol support. (#1707)
* Optimize reading small result sets to fix a performance regression
introduced by compression protocol support. (`#1707`)
* Fix `db.Ping()` on compressed connection. (#1723)
+5 -2
View File
@@ -1,5 +1,8 @@
# Go-MySQL-Driver
[![DeepWiki](https://img.shields.io/badge/DeepWiki-go--sql--driver%2Fmysql-blue.svg?logo=data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAACwAAAAyCAYAAAAnWDnqAAAAAXNSR0IArs4c6QAAA05JREFUaEPtmUtyEzEQhtWTQyQLHNak2AB7ZnyXZMEjXMGeK/AIi+QuHrMnbChYY7MIh8g01fJoopFb0uhhEqqcbWTp06/uv1saEDv4O3n3dV60RfP947Mm9/SQc0ICFQgzfc4CYZoTPAswgSJCCUJUnAAoRHOAUOcATwbmVLWdGoH//PB8mnKqScAhsD0kYP3j/Yt5LPQe2KvcXmGvRHcDnpxfL2zOYJ1mFwrryWTz0advv1Ut4CJgf5uhDuDj5eUcAUoahrdY/56ebRWeraTjMt/00Sh3UDtjgHtQNHwcRGOC98BJEAEymycmYcWwOprTgcB6VZ5JK5TAJ+fXGLBm3FDAmn6oPPjR4rKCAoJCal2eAiQp2x0vxTPB3ALO2CRkwmDy5WohzBDwSEFKRwPbknEggCPB/imwrycgxX2NzoMCHhPkDwqYMr9tRcP5qNrMZHkVnOjRMWwLCcr8ohBVb1OMjxLwGCvjTikrsBOiA6fNyCrm8V1rP93iVPpwaE+gO0SsWmPiXB+jikdf6SizrT5qKasx5j8ABbHpFTx+vFXp9EnYQmLx02h1QTTrl6eDqxLnGjporxl3NL3agEvXdT0WmEost648sQOYAeJS9Q7bfUVoMGnjo4AZdUMQku50McDcMWcBPvr0SzbTAFDfvJqwLzgxwATnCgnp4wDl6Aa+Ax283gghmj+vj7feE2KBBRMW3FzOpLOADl0Isb5587h/U4gGvkt5v60Z1VLG8BhYjbzRwyQZemwAd6cCR5/XFWLYZRIMpX39AR0tjaGGiGzLVyhse5C9RKC6ai42ppWPKiBagOvaYk8lO7DajerabOZP46Lby5wKjw1HCRx7p9sVMOWGzb/vA1hwiWc6jm3MvQDTogQkiqIhJV0nBQBTU+3okKCFDy9WwferkHjtxib7t3xIUQtHxnIwtx4mpg26/HfwVNVDb4oI9RHmx5WGelRVlrtiw43zboCLaxv46AZeB3IlTkwouebTr1y2NjSpHz68WNFjHvupy3q8TFn3Hos2IAk4Ju5dCo8B3wP7VPr/FGaKiG+T+v+TQqIrOqMTL1VdWV1DdmcbO8KXBz6esmYWYKPwDL5b5FA1a0hwapHiom0r/cKaoqr+27/XcrS5UwSMbQAAAABJRU5ErkJggg==)](https://deepwiki.com/go-sql-driver/mysql)
A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package
![Go-MySQL-Driver logo](https://raw.github.com/wiki/go-sql-driver/mysql/gomysql_m.png "Golang Gopher holding the MySQL Dolphin")
@@ -42,8 +45,8 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac
## Requirements
* Go 1.21 or higher. We aim to support the 3 latest versions of Go.
* MySQL (5.7+) and MariaDB (10.5+) are supported.
* Go 1.24 or higher. We aim to support the 3 latest versions of Go.
* MySQL (5.7+) and MariaDB (10.5+) are supported by maintainers.
* [TiDB](https://github.com/pingcap/tidb) is supported by PingCAP.
* Do not ask questions about TiDB in our issue tracker or forum.
* [Document](https://docs.pingcap.com/tidb/v6.1/dev-guide-sample-application-golang)
+1 -1
View File
@@ -305,7 +305,7 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) {
if !mc.cfg.AllowNativePasswords {
return nil, ErrNativePassword
}
// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html
// https://dev.mysql.com/doc/dev/mysql-server/8.4.5/page_protocol_connection_phase_authentication_methods_native_password_authentication.html
// Native password authentication only need and will need 20-byte challenge.
authResp := scramblePassword(authData[:20], mc.cfg.Passwd)
return authResp, nil
-1
View File
@@ -7,7 +7,6 @@
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd || solaris || illumos
// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
package mysql
-1
View File
@@ -7,7 +7,6 @@
// You can obtain one at http://mozilla.org/MPL/2.0/.
//go:build !linux && !darwin && !dragonfly && !freebsd && !netbsd && !openbsd && !solaris && !illumos
// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos
package mysql
+185 -90
View File
@@ -33,7 +33,8 @@ type mysqlConn struct {
connector *connector
maxAllowedPacket int
maxWriteSize int
flags clientFlag
capabilities capabilityFlag
extCapabilities extendedCapabilityFlag
status statusFlag
sequence uint8
compressSequence uint8
@@ -171,7 +172,7 @@ func (mc *mysqlConn) close() {
}
// Closes the network connection and unsets internal variables. Do not call this
// function after successfully authentication, call Close instead. This function
// function after successful authentication, call Close instead. This function
// is called before auth or on auth failure because MySQL will have already
// closed the network connection.
func (mc *mysqlConn) cleanup() {
@@ -223,13 +224,21 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
columnCount, err := stmt.readPrepareResultPacket()
if err == nil {
if stmt.paramCount > 0 {
if err = mc.readUntilEOF(); err != nil {
if err = mc.skipColumns(stmt.paramCount); err != nil {
return nil, err
}
}
if columnCount > 0 {
err = mc.readUntilEOF()
if mc.extCapabilities&clientCacheMetadata != 0 {
if stmt.columns, err = mc.readColumns(int(columnCount), nil); err != nil {
return nil, err
}
} else {
if err = mc.skipColumns(int(columnCount)); err != nil {
return nil, err
}
}
}
}
@@ -237,100 +246,184 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
}
func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) {
// Number of ? should be same to len(args)
if strings.Count(query, "?") != len(args) {
return "", driver.ErrSkip
}
noBackslashEscapes := (mc.status & statusNoBackslashEscapes) != 0
const (
stateNormal = iota
stateString
stateEscape
stateEOLComment
stateSlashStarComment
stateBacktick
)
const (
QUOTE_BYTE = byte('\'')
DBL_QUOTE_BYTE = byte('"')
BACKSLASH_BYTE = byte('\\')
QUESTION_MARK_BYTE = byte('?')
SLASH_BYTE = byte('/')
STAR_BYTE = byte('*')
HASH_BYTE = byte('#')
MINUS_BYTE = byte('-')
LINE_FEED_BYTE = byte('\n')
BACKTICK_BYTE = byte('`')
)
buf, err := mc.buf.takeCompleteBuffer()
if err != nil {
// can not take the buffer. Something must be wrong with the connection
mc.cleanup()
// interpolateParams would be called before sending any query.
// So its safe to retry.
return "", driver.ErrBadConn
}
buf = buf[:0]
state := stateNormal
singleQuotes := false
lastChar := byte(0)
argPos := 0
lenQuery := len(query)
lastIdx := 0
for i := 0; i < len(query); i++ {
q := strings.IndexByte(query[i:], '?')
if q == -1 {
buf = append(buf, query[i:]...)
break
}
buf = append(buf, query[i:i+q]...)
i += q
arg := args[argPos]
argPos++
if arg == nil {
buf = append(buf, "NULL"...)
for i := range lenQuery {
currentChar := query[i]
if state == stateEscape && !((currentChar == QUOTE_BYTE && singleQuotes) || (currentChar == DBL_QUOTE_BYTE && !singleQuotes)) {
state = stateString
lastChar = currentChar
continue
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case uint64:
// Handle uint64 explicitly because our custom ConvertValue emits unsigned values
buf = strconv.AppendUint(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
switch currentChar {
case STAR_BYTE:
if state == stateNormal && lastChar == SLASH_BYTE {
state = stateSlashStarComment
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
buf = append(buf, '\'')
buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate)
if err != nil {
return "", err
}
buf = append(buf, '\'')
case SLASH_BYTE:
if state == stateSlashStarComment && lastChar == STAR_BYTE {
state = stateNormal
// Clear lastChar so the '/' that closed the comment isn't
// reused to start a new comment with a following '*'.
lastChar = 0
continue
}
case json.RawMessage:
buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
} else {
buf = escapeBytesQuotes(buf, v)
case HASH_BYTE:
if state == stateNormal {
state = stateEOLComment
}
buf = append(buf, '\'')
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
buf = append(buf, "_binary'"...)
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeBytesBackslash(buf, v)
case MINUS_BYTE:
if state == stateNormal && lastChar == MINUS_BYTE {
// -- only starts a comment if followed by whitespace or control char
if i+1 < lenQuery {
nextChar := query[i+1]
if nextChar == ' ' || nextChar == '\t' || nextChar == '\n' || nextChar == '\r' {
state = stateEOLComment
}
} else {
buf = escapeBytesQuotes(buf, v)
state = stateEOLComment
}
buf = append(buf, '\'')
}
case string:
buf = append(buf, '\'')
if mc.status&statusNoBackslashEscapes == 0 {
buf = escapeStringBackslash(buf, v)
} else {
buf = escapeStringQuotes(buf, v)
case LINE_FEED_BYTE:
if state == stateEOLComment {
state = stateNormal
}
buf = append(buf, '\'')
default:
return "", driver.ErrSkip
}
case DBL_QUOTE_BYTE:
if state == stateNormal {
state = stateString
singleQuotes = false
} else if state == stateString && !singleQuotes {
state = stateNormal
} else if state == stateEscape {
state = stateString
}
case QUOTE_BYTE:
if state == stateNormal {
state = stateString
singleQuotes = true
} else if state == stateString && singleQuotes {
state = stateNormal
} else if state == stateEscape {
state = stateString
}
case BACKSLASH_BYTE:
if state == stateString && !noBackslashEscapes {
state = stateEscape
}
case QUESTION_MARK_BYTE:
if state == stateNormal {
if argPos >= len(args) {
return "", driver.ErrSkip
}
buf = append(buf, query[lastIdx:i]...)
arg := args[argPos]
argPos++
if len(buf)+4 > mc.maxAllowedPacket {
return "", driver.ErrSkip
if arg == nil {
buf = append(buf, "NULL"...)
lastIdx = i + 1
break
}
switch v := arg.(type) {
case int64:
buf = strconv.AppendInt(buf, v, 10)
case uint64:
buf = strconv.AppendUint(buf, v, 10)
case float64:
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
case bool:
if v {
buf = append(buf, '1')
} else {
buf = append(buf, '0')
}
case time.Time:
if v.IsZero() {
buf = append(buf, "'0000-00-00'"...)
} else {
buf = append(buf, '\'')
buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate)
if err != nil {
return "", err
}
buf = append(buf, '\'')
}
case json.RawMessage:
if noBackslashEscapes {
buf = escapeBytesQuotes(buf, v, false)
} else {
buf = escapeBytesBackslash(buf, v, false)
}
case []byte:
if v == nil {
buf = append(buf, "NULL"...)
} else {
if noBackslashEscapes {
buf = escapeBytesQuotes(buf, v, true)
} else {
buf = escapeBytesBackslash(buf, v, true)
}
}
case string:
if noBackslashEscapes {
buf = escapeStringQuotes(buf, v)
} else {
buf = escapeStringBackslash(buf, v)
}
default:
return "", driver.ErrSkip
}
if len(buf)+4 > mc.maxAllowedPacket {
return "", driver.ErrSkip
}
lastIdx = i + 1
}
case BACKTICK_BYTE:
if state == stateBacktick {
state = stateNormal
} else if state == stateNormal {
state = stateBacktick
}
}
lastChar = currentChar
}
buf = append(buf, query[lastIdx:]...)
if argPos != len(args) {
return "", driver.ErrSkip
}
@@ -370,19 +463,19 @@ func (mc *mysqlConn) exec(query string) error {
}
// Read Result
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, _, err := handleOk.readResultSetHeaderPacket()
if err != nil {
return err
}
if resLen > 0 {
// columns
if err := mc.readUntilEOF(); err != nil {
if err := mc.skipColumns(resLen); err != nil {
return err
}
// rows
if err := mc.readUntilEOF(); err != nil {
if err := mc.skipRows(); err != nil {
return err
}
}
@@ -419,7 +512,7 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
// Read Result
var resLen int
resLen, err = handleOk.readResultSetHeaderPacket()
resLen, _, err = handleOk.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
@@ -439,21 +532,20 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
}
// Columns
rows.rs.columns, err = mc.readColumns(resLen)
rows.rs.columns, err = mc.readColumns(resLen, nil)
return rows, err
}
// Gets the value of the given MySQL System Variable
// The returned byte slice is only valid until the next read
func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
func (mc *mysqlConn) getSystemVar(name string) (string, error) {
// Send command
handleOk := mc.clearResult()
if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil {
return nil, err
return "", err
}
// Read Result
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, _, err := handleOk.readResultSetHeaderPacket()
if err == nil {
rows := new(textRows)
rows.mc = mc
@@ -461,17 +553,20 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
if resLen > 0 {
// Columns
if err := mc.readUntilEOF(); err != nil {
return nil, err
if err := mc.skipColumns(resLen); err != nil {
return "", err
}
}
dest := make([]driver.Value, resLen)
if err = rows.readRow(dest); err == nil {
return dest[0].([]byte), mc.readUntilEOF()
// Convert to string before skipRows, which may
// overwrite the read buffer that dest[0] points into.
val := string(dest[0].([]byte))
return val, mc.skipRows()
}
}
return nil, err
return "", err
}
// cancel is called when the query has canceled.
+7 -5
View File
@@ -42,7 +42,7 @@ func encodeConnectionAttributes(cfg *Config) string {
}
// user-defined connection attributes
for _, connAttr := range strings.Split(cfg.ConnectionAttributes, ",") {
for connAttr := range strings.SplitSeq(cfg.ConnectionAttributes, ",") {
k, v, found := strings.Cut(connAttr, ":")
if !found {
continue
@@ -131,7 +131,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
mc.buf = newBuffer()
// Reading Handshake Initialization Packet
authData, plugin, err := mc.readHandshakePacket()
authData, serverCapabilities, serverExtCapabilities, plugin, err := mc.readHandshakePacket()
if err != nil {
mc.cleanup()
return nil, err
@@ -153,6 +153,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
return nil, err
}
}
mc.initCapabilities(serverCapabilities, serverExtCapabilities, mc.cfg)
if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil {
mc.cleanup()
return nil, err
@@ -161,13 +162,14 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
// Handle response to auth packet, switch methods if possible
if err = mc.handleAuthResult(authData, plugin); err != nil {
// Authentication failed and MySQL has already closed the connection
// (https://dev.mysql.com/doc/internals/en/authentication-fails.html).
// (https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase.html#sect_protocol_connection_phase_fast_path_fails).
// Do not send COM_QUIT, just cleanup and return the error.
mc.cleanup()
return nil, err
}
if mc.cfg.compress && mc.flags&clientCompress == clientCompress {
// compression is enabled after auth, not right after sending handshake response.
if mc.capabilities&clientCompress > 0 {
mc.compress = true
mc.compIO = newCompIO(mc)
}
@@ -180,7 +182,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
mc.Close()
return nil, err
}
n, err := strconv.Atoi(string(maxap))
n, err := strconv.Atoi(maxap)
if err != nil {
mc.Close()
return nil, fmt.Errorf("invalid max_allowed_packet value (%q): %w", maxap, err)
+17 -4
View File
@@ -32,7 +32,7 @@ const (
)
// MySQL constants documentation:
// http://dev.mysql.com/doc/internals/en/client-server-protocol.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_PROTOCOL.html
const (
iOK byte = 0x00
@@ -42,11 +42,12 @@ const (
iERR byte = 0xff
)
// https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags
type clientFlag uint32
// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__capabilities__flags.html
// https://mariadb.com/kb/en/connection/#capabilities
type capabilityFlag uint32
const (
clientLongPassword clientFlag = 1 << iota
clientMySQL capabilityFlag = 1 << iota
clientFoundRows
clientLongFlag
clientConnectWithDB
@@ -73,6 +74,18 @@ const (
clientDeprecateEOF
)
// https://mariadb.com/kb/en/connection/#capabilities
type extendedCapabilityFlag uint32
const (
progressIndicator extendedCapabilityFlag = 1 << iota
clientComMulti
clientStmtBulkOperations
clientExtendedMetadata
clientCacheMetadata
clientUnitBulkResult
)
const (
comQuit byte = iota + 1
comInitDB
+4 -5
View File
@@ -15,6 +15,7 @@ import (
"crypto/tls"
"errors"
"fmt"
"maps"
"math/big"
"net"
"net/url"
@@ -157,9 +158,7 @@ func (cfg *Config) Clone() *Config {
}
if len(cp.Params) > 0 {
cp.Params = make(map[string]string, len(cfg.Params))
for k, v := range cfg.Params {
cp.Params[k] = v
}
maps.Copy(cp.Params, cfg.Params)
}
if cfg.pubKey != nil {
cp.pubKey = &rsa.PublicKey{
@@ -414,7 +413,7 @@ func ParseDSN(dsn string) (cfg *Config, err error) {
if dsn[j] == '@' {
// username[:password]
// Find the first ':' in dsn[:j]
for k = 0; k < j; k++ {
for k = 0; k < j; k++ { // We cannot use k = range j here, because we use dsn[:k] below
if dsn[k] == ':' {
cfg.Passwd = dsn[k+1 : j]
break
@@ -477,7 +476,7 @@ func ParseDSN(dsn string) (cfg *Config, err error) {
// parseDSNParams parses the DSN "query string"
// Values must be url.QueryEscape'ed
func parseDSNParams(cfg *Config, params string) (err error) {
for _, v := range strings.Split(params, "&") {
for v := range strings.SplitSeq(params, "&") {
key, value, found := strings.Cut(v, "=")
if !found {
continue
+21 -17
View File
@@ -120,23 +120,24 @@ func (mf *mysqlField) typeDatabaseName() string {
}
var (
scanTypeFloat32 = reflect.TypeOf(float32(0))
scanTypeFloat64 = reflect.TypeOf(float64(0))
scanTypeInt8 = reflect.TypeOf(int8(0))
scanTypeInt16 = reflect.TypeOf(int16(0))
scanTypeInt32 = reflect.TypeOf(int32(0))
scanTypeInt64 = reflect.TypeOf(int64(0))
scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{})
scanTypeNullInt = reflect.TypeOf(sql.NullInt64{})
scanTypeNullTime = reflect.TypeOf(sql.NullTime{})
scanTypeUint8 = reflect.TypeOf(uint8(0))
scanTypeUint16 = reflect.TypeOf(uint16(0))
scanTypeUint32 = reflect.TypeOf(uint32(0))
scanTypeUint64 = reflect.TypeOf(uint64(0))
scanTypeString = reflect.TypeOf("")
scanTypeNullString = reflect.TypeOf(sql.NullString{})
scanTypeBytes = reflect.TypeOf([]byte{})
scanTypeUnknown = reflect.TypeOf(new(any))
scanTypeFloat32 = reflect.TypeFor[float32]()
scanTypeFloat64 = reflect.TypeFor[float64]()
scanTypeInt8 = reflect.TypeFor[int8]()
scanTypeInt16 = reflect.TypeFor[int16]()
scanTypeInt32 = reflect.TypeFor[int32]()
scanTypeInt64 = reflect.TypeFor[int64]()
scanTypeNullFloat = reflect.TypeFor[sql.NullFloat64]()
scanTypeNullInt = reflect.TypeFor[sql.NullInt64]()
scanTypeNullUint = reflect.TypeFor[sql.Null[uint64]]()
scanTypeNullTime = reflect.TypeFor[sql.NullTime]()
scanTypeUint8 = reflect.TypeFor[uint8]()
scanTypeUint16 = reflect.TypeFor[uint16]()
scanTypeUint32 = reflect.TypeFor[uint32]()
scanTypeUint64 = reflect.TypeFor[uint64]()
scanTypeString = reflect.TypeFor[string]()
scanTypeNullString = reflect.TypeFor[sql.NullString]()
scanTypeBytes = reflect.TypeFor[[]byte]()
scanTypeUnknown = reflect.TypeFor[*any]()
)
type mysqlField struct {
@@ -185,6 +186,9 @@ func (mf *mysqlField) scanType() reflect.Type {
}
return scanTypeInt64
}
if mf.flags&flagUnsigned != 0 {
return scanTypeNullUint
}
return scanTypeNullInt
case fieldTypeFloat:
+1 -4
View File
@@ -95,10 +95,7 @@ const defaultPacketSize = 16 * 1024 // 16KB is small enough for disk readahead a
func (mc *okHandler) handleInFileRequest(name string) (err error) {
var rdr io.Reader
packetSize := defaultPacketSize
if mc.maxWriteSize < packetSize {
packetSize = mc.maxWriteSize
}
packetSize := min(mc.maxWriteSize, defaultPacketSize)
if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader
// The server might return an an absolute path. See issue #355.
+205 -138
View File
@@ -179,20 +179,22 @@ func (mc *mysqlConn) writePacket(data []byte) error {
******************************************************************************/
// Handshake Initialization Packet
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake
func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) {
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_v10.html
// https://mariadb.com/kb/en/connection/#initial-handshake-packet
func (mc *mysqlConn) readHandshakePacket() (data []byte, capabilities capabilityFlag, extendedCapabilities extendedCapabilityFlag, plugin string, err error) {
data, err = mc.readPacket()
if err != nil {
return
}
if data[0] == iERR {
return nil, "", mc.handleErrorPacket(data)
err = mc.handleErrorPacket(data)
return
}
// protocol version [1 byte]
if data[0] < minProtocolVersion {
return nil, "", fmt.Errorf(
return nil, 0, 0, "", fmt.Errorf(
"unsupported protocol version %d. Version %d or higher is required",
data[0],
minProtocolVersion,
@@ -210,15 +212,15 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
pos += 8 + 1
// capability flags (lower 2 bytes) [2 bytes]
mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2]))
if mc.flags&clientProtocol41 == 0 {
return nil, "", ErrOldProtocol
capabilities = capabilityFlag(binary.LittleEndian.Uint16(data[pos : pos+2]))
if capabilities&clientProtocol41 == 0 {
return nil, capabilities, 0, "", ErrOldProtocol
}
if mc.flags&clientSSL == 0 && mc.cfg.TLS != nil {
if capabilities&clientSSL == 0 && mc.cfg.TLS != nil {
if mc.cfg.AllowFallbackToPlaintext {
mc.cfg.TLS = nil
} else {
return nil, "", ErrNoTLS
return nil, capabilities, 0, "", ErrNoTLS
}
}
pos += 2
@@ -228,11 +230,16 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
// status flags [2 bytes]
pos += 3
// capability flags (upper 2 bytes) [2 bytes]
mc.flags |= clientFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16
capabilities |= capabilityFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16
pos += 2
// length of auth-plugin-data [1 byte]
// reserved (all [00]) [10 bytes]
pos += 11
// reserved (all [00]) [6 bytes]
pos += 7
if capabilities&clientMySQL == 0 {
// MariaDB server extended flag
extendedCapabilities = extendedCapabilityFlag(binary.LittleEndian.Uint32(data[pos : pos+4]))
}
pos += 4
// second part of the password cipher [minimum 13 bytes],
// where len=MAX(13, length of auth-plugin-data - 8)
@@ -260,82 +267,72 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
// make a memory safe copy of the cipher slice
var b [20]byte
copy(b[:], authData)
return b[:], plugin, nil
return b[:], capabilities, extendedCapabilities, plugin, nil
}
// make a memory safe copy of the cipher slice
var b [8]byte
copy(b[:], authData)
return b[:], plugin, nil
return b[:], capabilities, 0, plugin, nil
}
// Client Authentication Packet
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse
func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error {
// Adjust client flags based on server support
clientFlags := clientProtocol41 |
clientSecureConn |
clientLongPassword |
clientTransactions |
clientLocalFiles |
clientPluginAuth |
clientMultiResults |
mc.flags&clientConnectAttrs |
mc.flags&clientLongFlag
// initCapabilities initializes the capabilities based on server support and configuration
func (mc *mysqlConn) initCapabilities(serverCapabilities capabilityFlag, serverExtCapabilities extendedCapabilityFlag, cfg *Config) {
clientCapabilities :=
clientMySQL |
clientLongFlag |
clientProtocol41 |
clientSecureConn |
clientTransactions |
clientPluginAuthLenEncClientData |
clientLocalFiles |
clientPluginAuth |
clientMultiResults |
clientConnectAttrs |
clientDeprecateEOF
sendConnectAttrs := mc.flags&clientConnectAttrs != 0
if mc.cfg.ClientFoundRows {
clientFlags |= clientFoundRows
if cfg.ClientFoundRows {
clientCapabilities |= clientFoundRows
}
if mc.cfg.compress && mc.flags&clientCompress == clientCompress {
clientFlags |= clientCompress
if cfg.compress {
clientCapabilities |= clientCompress
}
// To enable TLS / SSL
if mc.cfg.TLS != nil {
clientFlags |= clientSSL
clientCapabilities |= clientSSL
}
if mc.cfg.MultiStatements {
clientFlags |= clientMultiStatements
clientCapabilities |= clientMultiStatements
}
if n := len(cfg.DBName); n > 0 {
clientCapabilities |= clientConnectWithDB
}
// encode length of the auth plugin data
var authRespLEIBuf [9]byte
authRespLen := len(authResp)
authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen))
if len(authRespLEI) > 1 {
// if the length can not be written in 1 byte, it must be written as a
// length encoded integer
clientFlags |= clientPluginAuthLenEncClientData
}
// only keep client capabilities that server have
mc.capabilities = clientCapabilities & serverCapabilities
pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1
// set MariaDB extended clientCacheMetadata capability if server support it
mc.extCapabilities = clientCacheMetadata & serverExtCapabilities
}
// To specify a db name
if n := len(mc.cfg.DBName); n > 0 {
clientFlags |= clientConnectWithDB
pktLen += n + 1
}
// encode length of the connection attributes
var connAttrsLEI []byte
if sendConnectAttrs {
var connAttrsLEIBuf [9]byte
connAttrsLen := len(mc.connector.encodedAttributes)
connAttrsLEI = appendLengthEncodedInteger(connAttrsLEIBuf[:0], uint64(connAttrsLen))
pktLen += len(connAttrsLEI) + len(mc.connector.encodedAttributes)
}
// Calculate packet length and get buffer with that size
data, err := mc.buf.takeBuffer(pktLen + 4)
// Client Authentication Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_response.html
func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error {
// packet header 4
// capabilities 4
// maxPacketSize 4
// collation id 1
// filler 23
data, err := mc.buf.takeSmallBuffer(4*3 + 24)
if err != nil {
mc.cleanup()
return err
}
_ = data[4*3+23] // boundery check
// ClientFlags [32 bit]
binary.LittleEndian.PutUint32(data[4:], uint32(clientFlags))
// clientCapabilities [32 bit]
binary.LittleEndian.PutUint32(data[4:], uint32(mc.capabilities))
// MaxPacketSize [32 bit] (none)
binary.LittleEndian.PutUint32(data[8:], 0)
@@ -353,16 +350,26 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
}
// Filler [23 bytes] (all 0x00)
// or filler 19bytes + mariadb extCapabilities
pos := 13
for ; pos < 13+23; pos++ {
data[pos] = 0
if mc.capabilities&clientMySQL == 0 {
for ; pos < 13+19; pos++ {
data[pos] = 0
}
// MariaDB Extended Capabilities
binary.LittleEndian.PutUint32(data[13+19:], uint32(mc.extCapabilities))
} else {
for ; pos < 13+23; pos++ {
data[pos] = 0
}
}
// SSL Connection Request Packet
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_ssl_request.html
// https://mariadb.com/kb/en/connection/#sslrequest-packet
if mc.cfg.TLS != nil {
// Send TLS / SSL request packet
if err := mc.writePacket(data[:(4+4+1+23)+4]); err != nil {
if err := mc.writePacket(data); err != nil {
return err
}
@@ -379,37 +386,35 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
// User [null terminated string]
if len(mc.cfg.User) > 0 {
pos += copy(data[pos:], mc.cfg.User)
data = append(data, mc.cfg.User...)
}
data[pos] = 0x00
pos++
data = append(data, 0)
// Auth Data [length encoded integer]
pos += copy(data[pos:], authRespLEI)
pos += copy(data[pos:], authResp)
data = appendLengthEncodedInteger(data, uint64(len(authResp)))
data = append(data, authResp...)
// Databasename [null terminated string]
if len(mc.cfg.DBName) > 0 {
pos += copy(data[pos:], mc.cfg.DBName)
data[pos] = 0x00
pos++
// Database name [null terminated string]
if mc.capabilities&clientConnectWithDB != 0 {
data = append(data, mc.cfg.DBName...)
data = append(data, 0)
}
pos += copy(data[pos:], plugin)
data[pos] = 0x00
pos++
data = append(data, plugin...)
data = append(data, 0)
// Connection Attributes
if sendConnectAttrs {
pos += copy(data[pos:], connAttrsLEI)
pos += copy(data[pos:], []byte(mc.connector.encodedAttributes))
if mc.capabilities&clientConnectAttrs != 0 {
connAttrsLen := len(mc.connector.encodedAttributes)
data = appendLengthEncodedInteger(data, uint64(connAttrsLen))
data = append(data, mc.connector.encodedAttributes...)
}
// Send Auth packet
return mc.writePacket(data[:pos])
return mc.writePacket(data)
}
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_auth_switch_response.html
func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error {
pktLen := 4 + len(authData)
data, err := mc.buf.takeBuffer(pktLen)
@@ -511,7 +516,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) {
case iEOF:
if len(data) == 1 {
// https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_old_auth_switch_request.html
return nil, "mysql_old_password", nil
}
pluginEndIndex := bytes.IndexByte(data, 0x00)
@@ -545,36 +550,41 @@ func (mc *okHandler) readResultOK() error {
// Result Set Header Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response.html
func (mc *okHandler) readResultSetHeaderPacket() (int, error) {
func (mc *okHandler) readResultSetHeaderPacket() (int, bool, error) {
// handleOkPacket replaces both values; other cases leave the values unchanged.
mc.result.affectedRows = append(mc.result.affectedRows, 0)
mc.result.insertIds = append(mc.result.insertIds, 0)
data, err := mc.conn().readPacket()
if err != nil {
return 0, err
return 0, false, err
}
switch data[0] {
case iOK:
return 0, mc.handleOkPacket(data)
return 0, false, mc.handleOkPacket(data)
case iERR:
return 0, mc.conn().handleErrorPacket(data)
return 0, false, mc.conn().handleErrorPacket(data)
case iLocalInFile:
return 0, mc.handleInFileRequest(string(data[1:]))
return 0, false, mc.handleInFileRequest(string(data[1:]))
}
// column count
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html
num, _, _ := readLengthEncodedInteger(data)
// https://mariadb.com/kb/en/result-set-packets/#column-count-packet
num, _, len := readLengthEncodedInteger(data)
if mc.extCapabilities&clientCacheMetadata != 0 {
return int(num), data[len] == 0x01, nil
}
// ignore remaining data in the packet. see #1478.
return int(num), nil
return int(num), true, nil
}
// Error Packet
// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-ERR_Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html
func (mc *mysqlConn) handleErrorPacket(data []byte) error {
if data[0] != iERR {
return ErrMalformPkt
@@ -656,7 +666,7 @@ func (mc *mysqlConn) clearResult() *okHandler {
}
// Ok Packet
// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_ok_packet.html
func (mc *okHandler) handleOkPacket(data []byte) error {
var n, m int
var affectedRows, insertId uint64
@@ -690,24 +700,19 @@ func (mc *okHandler) handleOkPacket(data []byte) error {
}
// Read Packets as Field Packets until EOF-Packet or an Error appears
// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41
func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_column_definition.html#sect_protocol_com_query_response_text_resultset_column_definition_41
func (mc *mysqlConn) readColumns(count int, old []mysqlField) ([]mysqlField, error) {
columns := make([]mysqlField, count)
if len(old) != count {
old = nil
}
for i := 0; ; i++ {
for i := range count {
data, err := mc.readPacket()
if err != nil {
return nil, err
}
// EOF Packet
if data[0] == iEOF && (len(data) == 5 || len(data) == 1) {
if i == count {
return columns, nil
}
return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns))
}
// Catalog
pos, err := skipLengthEncodedString(data)
if err != nil {
@@ -728,7 +733,12 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
return nil, err
}
pos += n
columns[i].tableName = string(tableName)
if old != nil && old[i].tableName == string(tableName) {
// avoid allocating new string
columns[i].tableName = old[i].tableName
} else {
columns[i].tableName = string(tableName)
}
} else {
n, err = skipLengthEncodedString(data[pos:])
if err != nil {
@@ -749,7 +759,12 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
if err != nil {
return nil, err
}
columns[i].name = string(name)
if old != nil && old[i].name == string(name) {
// avoid allocating new string
columns[i].name = old[i].name
} else {
columns[i].name = string(name)
}
pos += n
// Original name [len coded string]
@@ -780,17 +795,17 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
// Decimals [uint8]
columns[i].decimals = data[pos]
//pos++
// Default value [len coded binary]
//if pos < len(data) {
// defaultVal, _, err = bytesToLengthCodedBinary(data[pos:])
//}
}
// skip EOF packet if client does not support deprecateEOF
if err := mc.skipEof(); err != nil {
return nil, err
}
return columns, nil
}
// Read Packets as Field Packets until EOF-Packet or an Error appears
// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::ResultsetRow
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_row.html
func (rows *textRows) readRow(dest []driver.Value) error {
mc := rows.mc
@@ -804,9 +819,20 @@ func (rows *textRows) readRow(dest []driver.Value) error {
}
// EOF Packet
if data[0] == iEOF && len(data) == 5 {
// server_status [2 bytes]
rows.mc.status = readStatus(data[3:])
// text row packets may starts with LengthEncodedString.
// In such case, 0xFE can mean string larger than 0xffffff.
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_dt_integers.html#sect_protocol_basic_dt_int_le
if data[0] == iEOF && len(data) <= 0xffffff {
if mc.capabilities&clientDeprecateEOF == 0 {
// Deprecated EOF packet
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_eof_packet.html
mc.status = readStatus(data[3:])
} else {
// Ok Packet with an 0xFE header
_, _, n := readLengthEncodedInteger(data[1:]) // affected_rows
_, _, m := readLengthEncodedInteger(data[1+n:]) // last_insert_id
mc.status = readStatus(data[1+n+m:])
}
rows.rs.done = true
if !rows.HasNextResultSet() {
rows.mc = nil
@@ -880,8 +906,34 @@ func (rows *textRows) readRow(dest []driver.Value) error {
return nil
}
// Reads Packets until EOF-Packet or an Error appears. Returns count of Packets read
func (mc *mysqlConn) readUntilEOF() error {
func (mc *mysqlConn) skipPackets(n int) error {
for range n {
if _, err := mc.readPacket(); err != nil {
return err
}
}
return nil
}
// skips EOF packet after n * ColumnDefinition packets when clientDeprecateEOF is not set
func (mc *mysqlConn) skipEof() error {
if mc.capabilities&clientDeprecateEOF == 0 {
if _, err := mc.readPacket(); err != nil {
return err
}
}
return nil
}
func (mc *mysqlConn) skipColumns(n int) error {
if err := mc.skipPackets(n); err != nil {
return err
}
return mc.skipEof()
}
// Reads Packets until EOF-Packet or an Error appears.
func (mc *mysqlConn) skipRows() error {
for {
data, err := mc.readPacket()
if err != nil {
@@ -892,10 +944,20 @@ func (mc *mysqlConn) readUntilEOF() error {
case iERR:
return mc.handleErrorPacket(data)
case iEOF:
if len(data) == 5 {
mc.status = readStatus(data[3:])
// text row packets may starts with LengthEncodedString.
// In such case, 0xFE can mean string larger than 0xffffff.
if len(data) <= 0xffffff {
if mc.capabilities&clientDeprecateEOF == 0 {
// EOF packet
mc.status = readStatus(data[3:])
} else {
// OK packet with an 0xFE header
_, _, n := readLengthEncodedInteger(data[1:]) // affected_rows
_, _, m := readLengthEncodedInteger(data[1+n:]) // last_insert_id
mc.status = readStatus(data[1+n+m:])
}
return nil
}
return nil
}
}
}
@@ -905,7 +967,7 @@ func (mc *mysqlConn) readUntilEOF() error {
******************************************************************************/
// Prepare Result Packets
// http://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_prepare.html#sect_protocol_com_stmt_prepare_response
func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) {
data, err := stmt.mc.readPacket()
if err == nil {
@@ -932,7 +994,7 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) {
return 0, err
}
// http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_send_long_data.html
func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error {
maxLen := stmt.mc.maxAllowedPacket - 1
pktLen := maxLen
@@ -979,7 +1041,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error {
}
// Execute Prepared Statement
// http://dev.mysql.com/doc/internals/en/com-stmt-execute.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html
func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
if len(args) != stmt.paramCount {
return fmt.Errorf(
@@ -993,10 +1055,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
mc := stmt.mc
// Determine threshold dynamically to avoid packet size shortage.
longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1)
if longDataSize < 64 {
longDataSize = 64
}
longDataSize := max(mc.maxAllowedPacket/(stmt.paramCount+1), 64)
// Reset packet-sequence
mc.resetSequence()
@@ -1185,17 +1244,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
// mc.affectedRows and mc.insertIds.
func (mc *okHandler) discardResults() error {
for mc.status&statusMoreResultsExists != 0 {
resLen, err := mc.readResultSetHeaderPacket()
resLen, _, err := mc.readResultSetHeaderPacket()
if err != nil {
return err
}
if resLen > 0 {
// columns
if err := mc.conn().readUntilEOF(); err != nil {
if err := mc.conn().skipColumns(resLen); err != nil {
return err
}
// rows
if err := mc.conn().readUntilEOF(); err != nil {
if err := mc.conn().skipRows(); err != nil {
return err
}
}
@@ -1203,7 +1262,7 @@ func (mc *okHandler) discardResults() error {
return nil
}
// http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row
func (rows *binaryRows) readRow(dest []driver.Value) error {
data, err := rows.mc.readPacket()
if err != nil {
@@ -1212,9 +1271,17 @@ func (rows *binaryRows) readRow(dest []driver.Value) error {
// packet indicator [1 byte]
if data[0] != iOK {
// EOF Packet
if data[0] == iEOF && len(data) == 5 {
rows.mc.status = readStatus(data[3:])
// EOF/OK Packet
if data[0] == iEOF {
if rows.mc.capabilities&clientDeprecateEOF == 0 {
// EOF packet
rows.mc.status = readStatus(data[3:])
} else {
// OK Packet with an 0xFE header
_, _, n := readLengthEncodedInteger(data[1:])
_, _, m := readLengthEncodedInteger(data[1+n:])
rows.mc.status = readStatus(data[1+n+m:])
}
rows.rs.done = true
if !rows.HasNextResultSet() {
rows.mc = nil
+4 -2
View File
@@ -8,6 +8,8 @@
package mysql
import "slices"
import "database/sql/driver"
// Result exposes data not available through *connection.Result.
@@ -42,9 +44,9 @@ func (res *mysqlResult) RowsAffected() (int64, error) {
}
func (res *mysqlResult) AllLastInsertIds() []int64 {
return append([]int64{}, res.insertIds...) // defensive copy
return slices.Clone(res.insertIds) // defensive copy
}
func (res *mysqlResult) AllRowsAffected() []int64 {
return append([]int64{}, res.affectedRows...) // defensive copy
return slices.Clone(res.affectedRows) // defensive copy
}
+5 -5
View File
@@ -113,7 +113,7 @@ func (rows *mysqlRows) Close() (err error) {
// Remove unread packets from stream
if !rows.rs.done {
err = mc.readUntilEOF()
err = mc.skipRows()
}
if err == nil {
handleOk := mc.clearResult()
@@ -143,7 +143,7 @@ func (rows *mysqlRows) nextResultSet() (int, error) {
// Remove unread packets from stream
if !rows.rs.done {
if err := rows.mc.readUntilEOF(); err != nil {
if err := rows.mc.skipRows(); err != nil {
return 0, err
}
rows.rs.done = true
@@ -156,7 +156,7 @@ func (rows *mysqlRows) nextResultSet() (int, error) {
rows.rs = resultSet{}
// rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to
// nextResultSet.
resLen, err := rows.mc.resultUnchanged().readResultSetHeaderPacket()
resLen, _, err := rows.mc.resultUnchanged().readResultSetHeaderPacket()
if err != nil {
// Clean up about multi-results flag
rows.rs.done = true
@@ -186,7 +186,7 @@ func (rows *binaryRows) NextResultSet() error {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
rows.rs.columns, err = rows.mc.readColumns(resLen, nil)
return err
}
@@ -208,7 +208,7 @@ func (rows *textRows) NextResultSet() (err error) {
return err
}
rows.rs.columns, err = rows.mc.readColumns(resLen)
rows.rs.columns, err = rows.mc.readColumns(resLen, nil)
return err
}
+26 -8
View File
@@ -20,6 +20,7 @@ type mysqlStmt struct {
mc *mysqlConn
id uint32
paramCount int
columns []mysqlField
}
func (stmt *mysqlStmt) Close() error {
@@ -64,19 +65,26 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) {
handleOk := stmt.mc.clearResult()
// Read Result
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, metadataFollows, err := handleOk.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
if resLen > 0 {
// Columns
if err = mc.readUntilEOF(); err != nil {
return nil, err
if metadataFollows && stmt.mc.extCapabilities&clientCacheMetadata != 0 {
// we can not skip column metadata because next stmt.Query() may use it.
if stmt.columns, err = mc.readColumns(resLen, stmt.columns); err != nil {
return nil, err
}
} else {
if err = mc.skipColumns(resLen); err != nil {
return nil, err
}
}
// Rows
if err := mc.readUntilEOF(); err != nil {
if err = mc.skipRows(); err != nil {
return nil, err
}
}
@@ -107,7 +115,7 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
// Read Result
handleOk := stmt.mc.clearResult()
resLen, err := handleOk.readResultSetHeaderPacket()
resLen, metadataFollows, err := handleOk.readResultSetHeaderPacket()
if err != nil {
return nil, err
}
@@ -116,7 +124,17 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
if resLen > 0 {
rows.mc = mc
rows.rs.columns, err = mc.readColumns(resLen)
if metadataFollows {
if rows.rs.columns, err = mc.readColumns(resLen, stmt.columns); err != nil {
return nil, err
}
stmt.columns = rows.rs.columns
} else {
if err = mc.skipEof(); err != nil {
return nil, err
}
rows.rs.columns = stmt.columns
}
} else {
rows.rs.done = true
@@ -131,7 +149,7 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
return rows, err
}
var jsonType = reflect.TypeOf(json.RawMessage{})
var jsonType = reflect.TypeFor[json.RawMessage]()
type converter struct{}
@@ -193,7 +211,7 @@ func (c converter) ConvertValue(v any) (driver.Value, error) {
return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind())
}
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
var valuerReflectType = reflect.TypeFor[driver.Valuer]()
// callValuerValue returns vr.Value(), with one exception:
// If vr.Value is an auto-generated method on a pointer type and the
+65 -90
View File
@@ -182,7 +182,7 @@ func parseDateTime(b []byte, loc *time.Location) (time.Time, error) {
func parseByteYear(b []byte) (int, error) {
year, n := 0, 1000
for i := 0; i < 4; i++ {
for i := range 4 {
v, err := bToi(b[i])
if err != nil {
return 0, err
@@ -207,7 +207,7 @@ func parseByte2Digits(b1, b2 byte) (int, error) {
func parseByteNanoSec(b []byte) (int, error) {
ns, digit := 0, 100000 // max is 6-digits
for i := 0; i < len(b); i++ {
for i := range b {
v, err := bToi(b[i])
if err != nil {
return 0, err
@@ -625,108 +625,80 @@ func reserveBuffer(buf []byte, appendSize int) []byte {
return buf[:newSize]
}
// escapeBytesBackslash escapes []byte with backslashes (\)
// This escapes the contents of a string (provided as []byte) by adding backslashes before special
// characters, and turning others into specific escape sequences, such as
// turning newlines into \n and null bytes into \0.
// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L823-L932
func escapeBytesBackslash(buf, v []byte) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
// Lookup table for backslash escapes (used for both string and bytes)
var backslashEscapeTable [256]byte
for _, c := range v {
switch c {
case '\x00':
buf[pos+1] = '0'
buf[pos] = '\\'
pos += 2
case '\n':
buf[pos+1] = 'n'
buf[pos] = '\\'
pos += 2
case '\r':
buf[pos+1] = 'r'
buf[pos] = '\\'
pos += 2
case '\x1a':
buf[pos+1] = 'Z'
buf[pos] = '\\'
pos += 2
case '\'':
buf[pos+1] = '\''
buf[pos] = '\\'
pos += 2
case '"':
buf[pos+1] = '"'
buf[pos] = '\\'
pos += 2
case '\\':
buf[pos+1] = '\\'
buf[pos] = '\\'
pos += 2
default:
buf[pos] = c
pos++
}
}
return buf[:pos]
func init() {
backslashEscapeTable['\x00'] = '0'
backslashEscapeTable['\n'] = 'n'
backslashEscapeTable['\r'] = 'r'
backslashEscapeTable['\x1a'] = 'Z'
backslashEscapeTable['\''] = '\''
backslashEscapeTable['"'] = '"'
backslashEscapeTable['\\'] = '\\'
}
// escapeStringBackslash is similar to escapeBytesBackslash but for string.
func escapeStringBackslash(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
for i := 0; i < len(v); i++ {
c := v[i]
switch c {
case '\x00':
buf[pos+1] = '0'
if esc := backslashEscapeTable[c]; esc != 0 {
buf[pos+1] = esc
buf[pos] = '\\'
pos += 2
case '\n':
buf[pos+1] = 'n'
buf[pos] = '\\'
pos += 2
case '\r':
buf[pos+1] = 'r'
buf[pos] = '\\'
pos += 2
case '\x1a':
buf[pos+1] = 'Z'
buf[pos] = '\\'
pos += 2
case '\'':
buf[pos+1] = '\''
buf[pos] = '\\'
pos += 2
case '"':
buf[pos+1] = '"'
buf[pos] = '\\'
pos += 2
case '\\':
buf[pos+1] = '\\'
buf[pos] = '\\'
pos += 2
default:
} else {
buf[pos] = c
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
// escapeBytesQuotes escapes apostrophes in []byte by doubling them up.
// This escapes the contents of a string by doubling up any apostrophes that
// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in
// effect on the server.
// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L963-L1038
func escapeBytesQuotes(buf, v []byte) []byte {
// escapeBytesBackslash appends _binary'...' or '...' with backslash escaping for bytes.
func escapeBytesBackslash(buf, v []byte, binary bool) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
if binary {
buf = reserveBuffer(buf, len(v)*2+9)
copy(buf[pos:], []byte("_binary'"))
pos += 8
} else {
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
}
for _, c := range v {
if esc := backslashEscapeTable[c]; esc != 0 {
buf[pos+1] = esc
buf[pos] = '\\'
pos += 2
} else {
buf[pos] = c
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
// escapeBytesQuotes appends _binary'...' or '...' with single-quote escaping for bytes.
func escapeBytesQuotes(buf, v []byte, binary bool) []byte {
pos := len(buf)
if binary {
buf = reserveBuffer(buf, len(v)*2+9)
copy(buf[pos:], []byte("_binary'"))
pos += 8
} else {
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
}
for _, c := range v {
if c == '\'' {
buf[pos+1] = '\''
@@ -737,16 +709,18 @@ func escapeBytesQuotes(buf, v []byte) []byte {
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
// escapeStringQuotes is similar to escapeBytesQuotes but for string.
func escapeStringQuotes(buf []byte, v string) []byte {
pos := len(buf)
buf = reserveBuffer(buf, len(v)*2)
for i := 0; i < len(v); i++ {
buf = reserveBuffer(buf, len(v)*2+2)
buf[pos] = '\''
pos++
for i := range len(v) {
c := v[i]
if c == '\'' {
buf[pos+1] = '\''
@@ -757,7 +731,8 @@ func escapeStringQuotes(buf []byte, v string) []byte {
pos++
}
}
buf[pos] = '\''
pos++
return buf[:pos]
}
+1 -1
View File
@@ -1,4 +1,4 @@
Copyright (c) 2016 HashiCorp, Inc.
Copyright IBM Corp. 2016, 2025
Mozilla Public License, version 2.0
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) HashiCorp, Inc.
# Copyright IBM Corp. 2016, 2025
# SPDX-License-Identifier: MPL-2.0
version: v1
+1 -1
View File
@@ -1,4 +1,4 @@
# Copyright (c) HashiCorp, Inc.
# Copyright IBM Corp. 2016, 2025
# SPDX-License-Identifier: MPL-2.0
version: v1
+6 -4
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
@@ -669,7 +669,7 @@ func (c *Client) Start() (addr net.Addr, err error) {
// Setup a temporary certificate for client/server mtls, and send the public
// certificate to the plugin.
if c.config.AutoMTLS {
c.logger.Info("configuring client automatic mTLS")
c.logger.Debug("configuring client automatic mTLS")
certPEM, keyPEM, err := generateCert()
if err != nil {
c.logger.Error("failed to generate client certificate", "error", err)
@@ -753,9 +753,12 @@ func (c *Client) Start() (addr net.Addr, err error) {
// Create a context for when we kill
c.doneCtx, c.ctxCancel = context.WithCancel(context.Background())
// Add two to pipesWaitGroup: one for logStderr, one for the goroutine
// below that consumes Stdout. We mustn't continue to Add once we might Wait.
c.pipesWaitGroup.Add(2)
// Start goroutine that logs the stderr
c.clientWaitGroup.Add(1)
c.pipesWaitGroup.Add(1)
// logStderr calls c.pipesWaitGroup.Done()
go c.logStderr(runner.Name(), runner.Stderr())
@@ -791,7 +794,6 @@ func (c *Client) Start() (addr net.Addr, err error) {
// out of stdout
linesCh := make(chan string)
c.clientWaitGroup.Add(1)
c.pipesWaitGroup.Add(1)
go func() {
defer c.clientWaitGroup.Done()
defer c.pipesWaitGroup.Done()
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+1 -2
View File
@@ -1,8 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
//go:build !windows
// +build !windows
package cmdrunner
+1 -2
View File
@@ -1,8 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
//go:build windows
// +build windows
package cmdrunner
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
+6 -2
View File
@@ -1,8 +1,7 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
//go:build !windows
// +build !windows
package cmdrunner
@@ -16,6 +15,11 @@ import (
func _pidAlive(pid int) bool {
proc, err := os.FindProcess(pid)
if err == nil {
// On Linux with Go 1.23+, FindProcess opens a pidfd which must be
// released or it leaks an FD on every call. Release errors are
// intentionally ignored; the handle is short-lived and there's
// nothing actionable to recover from a release failure.
defer func() { _ = proc.Release() }()
err = proc.Signal(syscall.Signal(0))
}
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package cmdrunner
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package grpcmux
+2 -2
View File
@@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
syntax = "proto3";
package plugin;
option go_package = "./plugin";
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
message ConnInfo {
uint32 service_id = 1;
@@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
syntax = "proto3";
package plugin;
option go_package = "./plugin";
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
message Empty {
}
+2 -2
View File
@@ -1,9 +1,9 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
syntax = "proto3";
package plugin;
option go_package = "./plugin";
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
import "google/protobuf/empty.proto";
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
// The plugin package exposes functions and helpers for communicating to
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package runner
+2 -2
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
@@ -307,7 +307,7 @@ func Serve(opts *ServeConfig) {
// If the client is configured using AutoMTLS, the certificate will be here,
// and we need to generate our own in response.
if tlsConfig == nil && clientCert != "" {
logger.Info("configuring server automatic mTLS")
logger.Debug("configuring server automatic mTLS")
clientCertPool := x509.NewCertPool()
if !clientCertPool.AppendCertsFromPEM([]byte(clientCert)) {
logger.Error("client cert provided but failed to parse", "cert", clientCert)
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright (c) HashiCorp, Inc.
// Copyright IBM Corp. 2016, 2025
// SPDX-License-Identifier: MPL-2.0
package plugin
+5
View File
@@ -175,6 +175,11 @@ func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName stri
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+17 -5
View File
@@ -150,6 +150,11 @@ type UploadInfo struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string
ChecksumXXHash3 string
ChecksumXXHash128 string
ChecksumMode string
}
@@ -168,11 +173,12 @@ type ObjectInfo struct {
// each parts concatenated into one string.
ETag string `json:"etag"`
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
Key string `json:"name"` // Name of the object
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
Size int64 `json:"size"` // Size in bytes of the object.
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
ContentEncoding string `json:"contentEncoding"` // A standard MIME type describing encoding of the object data.
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
// Collection of additional metadata on the object.
// eg: x-amz-meta-*, content-encoding etc.
@@ -226,6 +232,12 @@ type ObjectInfo struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string
ChecksumXXHash3 string
ChecksumXXHash128 string
ChecksumAlgorithm string
ChecksumMode string `xml:"ChecksumType"`
Internal *struct {
+96 -10
View File
@@ -23,6 +23,7 @@ import (
"net/http"
"net/url"
"strconv"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/encrypt"
@@ -88,10 +89,17 @@ type ObjectAttributesResponse struct {
StorageClass string
ObjectSize int
Checksum struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
ChecksumType string `xml:",omitempty"`
}
ObjectParts struct {
PartsCount int
@@ -105,12 +113,90 @@ type ObjectAttributesResponse struct {
// ObjectAttributePart is used by ObjectAttributesResponse to describe an object part
type ObjectAttributePart struct {
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
PartNumber int
Size int
ChecksumCRC32 string `xml:",omitempty"`
ChecksumCRC32C string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
PartNumber int
Size int
}
// ChecksumMap returns a map of checksums for the object.
func (o *ObjectAttributesResponse) ChecksumMap() map[string]string {
res := make(map[string]string)
setif := func(typ ChecksumType, value string) {
if value != "" {
res[typ.Key()] = value
}
}
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
return res
}
// ChecksumMode returns the checksum mode of the object.
// If unable to determine, returns ChecksumUnknownMode.
func (o *ObjectAttributesResponse) ChecksumMode() ChecksumMode {
t := o.ChecksumType()
if !t.IsSet() {
return ChecksumUnknownMode
}
switch o.Checksum.ChecksumType {
case amzChecksumModeComposite:
return ChecksumCompositeMode
case amzChecksumModeFullObject:
return ChecksumFullObjectMode
case "":
// Likely not supported by the server.
if o.Checksum.ChecksumCRC64NVME != "" || !strings.ContainsRune(o.ETag, '-') {
// Always full object.
return ChecksumFullObjectMode
}
if !t.CanMergeCRC() {
// Only composite possible.
return ChecksumCompositeMode
}
}
return ChecksumUnknownMode
}
// ChecksumType returns the checksum type of the object.
// If none is set, returns ChecksumNone.
func (o *ObjectAttributesResponse) ChecksumType() ChecksumType {
t := ChecksumNone
setif := func(typ ChecksumType, value string) {
if value != "" {
t = typ
}
}
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
if t.IsSet() && o.Checksum.ChecksumType == amzChecksumModeFullObject {
t |= ChecksumFullObject
}
return t
}
func (o *ObjectAttributes) parseResponse(resp *http.Response) (err error) {
+7 -1
View File
@@ -404,7 +404,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
}
var (
keyMarker = ""
keyMarker = opts.StartAfter
versionIDMarker = ""
preName = ""
preKey = ""
@@ -432,12 +432,18 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
UserMetadata: version.UserMetadata,
Internal: version.Internal,
NumVersions: numVersions,
ChecksumAlgorithm: version.ChecksumAlgorithm,
ChecksumMode: version.ChecksumType,
ChecksumCRC32: version.ChecksumCRC32,
ChecksumCRC32C: version.ChecksumCRC32C,
ChecksumSHA1: version.ChecksumSHA1,
ChecksumSHA256: version.ChecksumSHA256,
ChecksumCRC64NVME: version.ChecksumCRC64NVME,
ChecksumMD5: version.ChecksumMD5,
ChecksumSHA512: version.ChecksumSHA512,
ChecksumXXHash64: version.ChecksumXXHash64,
ChecksumXXHash3: version.ChecksumXXHash3,
ChecksumXXHash128: version.ChecksumXXHash128,
}
if !yield(info) {
return false
+15
View File
@@ -192,6 +192,11 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -353,6 +358,11 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
}
objPart.Size = p.size
objPart.PartNumber = p.partNumber
@@ -458,6 +468,11 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME,
ChecksumMD5: completeMultipartUploadResult.ChecksumMD5,
ChecksumSHA512: completeMultipartUploadResult.ChecksumSHA512,
ChecksumXXHash64: completeMultipartUploadResult.ChecksumXXHash64,
ChecksumXXHash3: completeMultipartUploadResult.ChecksumXXHash3,
ChecksumXXHash128: completeMultipartUploadResult.ChecksumXXHash128,
ChecksumMode: completeMultipartUploadResult.ChecksumType,
}, nil
}
+20
View File
@@ -259,6 +259,11 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME,
ChecksumMD5: uploadRes.Part.ChecksumMD5,
ChecksumSHA512: uploadRes.Part.ChecksumSHA512,
ChecksumXXHash64: uploadRes.Part.ChecksumXXHash64,
ChecksumXXHash3: uploadRes.Part.ChecksumXXHash3,
ChecksumXXHash128: uploadRes.Part.ChecksumXXHash128,
})
}
}
@@ -418,6 +423,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -617,6 +627,11 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
@@ -796,6 +811,11 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+5
View File
@@ -502,6 +502,11 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
ChecksumSHA1: part.ChecksumSHA1,
ChecksumSHA256: part.ChecksumSHA256,
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
ChecksumMD5: part.ChecksumMD5,
ChecksumSHA512: part.ChecksumSHA512,
ChecksumXXHash64: part.ChecksumXXHash64,
ChecksumXXHash3: part.ChecksumXXHash3,
ChecksumXXHash128: part.ChecksumXXHash128,
})
}
+41
View File
@@ -113,6 +113,12 @@ type Version struct {
ChecksumSHA1 string `xml:",omitempty"`
ChecksumSHA256 string `xml:",omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
ChecksumAlgorithm string `xml:",omitempty"`
ChecksumType string `xml:",omitempty"`
isDeleteMarker bool
@@ -297,6 +303,11 @@ type ObjectPart struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
}
// Checksum will return the checksum for the given type.
@@ -313,6 +324,16 @@ func (c ObjectPart) Checksum(t ChecksumType) string {
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
case t.Is(ChecksumMD5):
return c.ChecksumMD5
case t.Is(ChecksumSHA512):
return c.ChecksumSHA512
case t.Is(ChecksumXXHash64):
return c.ChecksumXXHash64
case t.Is(ChecksumXXHash3):
return c.ChecksumXXHash3
case t.Is(ChecksumXXHash128):
return c.ChecksumXXHash128
}
return ""
}
@@ -382,6 +403,11 @@ type completeMultipartUploadResult struct {
ChecksumSHA1 string
ChecksumSHA256 string
ChecksumCRC64NVME string
ChecksumMD5 string
ChecksumSHA512 string
ChecksumXXHash64 string `xml:"ChecksumXXHASH64"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128"`
ChecksumType string
}
@@ -398,6 +424,11 @@ type CompletePart struct {
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
ChecksumCRC64NVME string `xml:",omitempty"`
ChecksumMD5 string `xml:",omitempty"`
ChecksumSHA512 string `xml:",omitempty"`
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
}
// Checksum will return the checksum for the given type.
@@ -414,6 +445,16 @@ func (c CompletePart) Checksum(t ChecksumType) string {
return c.ChecksumSHA256
case t.Is(ChecksumCRC64NVME):
return c.ChecksumCRC64NVME
case t.Is(ChecksumMD5):
return c.ChecksumMD5
case t.Is(ChecksumSHA512):
return c.ChecksumSHA512
case t.Is(ChecksumXXHash64):
return c.ChecksumXXHash64
case t.Is(ChecksumXXHash3):
return c.ChecksumXXHash3
case t.Is(ChecksumXXHash128):
return c.ChecksumXXHash128
}
return ""
}
+17 -9
View File
@@ -52,6 +52,12 @@ type UpdateObjectEncryptionOptions struct {
VersionID string
}
// UpdateObjectEncryptionResult holds the result of an UpdateObjectEncryption call.
type UpdateObjectEncryptionResult struct {
// VersionID is the version ID of the object that was updated, if versioning is enabled.
VersionID string
}
// UpdateObjectEncryption changes the encryption configuration of an existing object in-place.
// The object must already be encrypted with SSE-S3 or SSE-KMS. SSE-C objects are not supported.
// This operation rotates the data encryption key envelope without re-reading/re-writing object data.
@@ -62,19 +68,19 @@ type UpdateObjectEncryptionOptions struct {
// - objectName: Name of the object
// - opts: Options including KMSKeyArn (required), optional BucketKeyEnabled, and optional VersionID
//
// Returns an error if the operation fails.
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) error {
// Returns the version ID of the updated object (if versioning is enabled) and an error if the operation fails.
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) (UpdateObjectEncryptionResult, error) {
// Input validation.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if err := s3utils.CheckValidObjectName(objectName); err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if opts.KMSKeyArn == "" {
return errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
return UpdateObjectEncryptionResult{}, errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
}
// Get resources properly escaped and lined up before
@@ -96,7 +102,7 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
bodyData, err := xml.Marshal(reqBody)
if err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
reqMetadata := requestMetadata{
@@ -113,10 +119,12 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
defer closeResponse(resp)
if err != nil {
return err
return UpdateObjectEncryptionResult{}, err
}
if resp.StatusCode != http.StatusOK {
return httpRespToErrorResponse(resp, bucketName, objectName)
return UpdateObjectEncryptionResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
}
return nil
return UpdateObjectEncryptionResult{
VersionID: resp.Header.Get(amzVersionID),
}, nil
}
+13 -1
View File
@@ -198,6 +198,9 @@ func New(endpoint string, opts *Options) (*Client, error) {
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
// for backwards compatibility.
clnt.s3DualstackEnabled = true
} else if s3utils.IsAmazonOutpostsEndpoint(*clnt.endpointURL) {
// S3 on Outposts uses signature v4 with service name s3-outposts.
clnt.overrideSignerType = credentials.SignatureV4
}
return clnt, nil
@@ -912,7 +915,11 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
} else if signerType.IsV4() {
// Presign URL with signature v4.
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.PreSignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
} else {
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
}
}
return req, nil
}
@@ -971,6 +978,9 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
req = signer.StreamingSignV4Express(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.StreamingSignV4Outposts(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
} else {
req = signer.StreamingSignV4(req, accessKeyID,
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
@@ -991,6 +1001,8 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
req = signer.SignV4TrailerOutposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
} else {
// Add signature version '4' authorization header.
req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
+6 -1
View File
@@ -209,6 +209,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
}
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
region := getDefaultLocation(*c.endpointURL, c.region)
req = signer.SignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, region)
} else {
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
}
return req, nil
}
+67 -4
View File
@@ -18,8 +18,10 @@
package minio
import (
"crypto/md5"
"crypto/sha1"
"crypto/sha256"
"crypto/sha512"
"encoding/base64"
"encoding/binary"
"errors"
@@ -30,8 +32,10 @@ import (
"sort"
"strings"
"github.com/cespare/xxhash/v2"
"github.com/klauspost/crc32"
"github.com/minio/crc64nvme"
"github.com/zeebo/xxh3"
)
// ChecksumMode contains information about the checksum mode on the object
@@ -49,6 +53,9 @@ const (
// checksumModeMask is a mask for valid checksum mode types.
checksumModeMask = checksumLastMode - 1
// ChecksumUnknownMode indicates no or unknown checksum mode.
ChecksumUnknownMode ChecksumMode = 0
)
// Is returns if c is all of t.
@@ -64,9 +71,9 @@ func (c ChecksumMode) Key() string {
func (c ChecksumMode) String() string {
switch c & checksumModeMask {
case ChecksumFullObjectMode:
return "FULL_OBJECT"
return amzChecksumModeFullObject
case ChecksumCompositeMode:
return "COMPOSITE"
return amzChecksumModeComposite
}
return ""
}
@@ -86,6 +93,16 @@ const (
ChecksumCRC32C
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
ChecksumCRC64NVME
// ChecksumMD5 indicates an MD5 checksum.
ChecksumMD5
// ChecksumSHA512 indicates a SHA-512 checksum.
ChecksumSHA512
// ChecksumXXHash64 indicates an XXHash64 checksum.
ChecksumXXHash64
// ChecksumXXHash3 indicates an XXH3-64 checksum.
ChecksumXXHash3
// ChecksumXXHash128 indicates an XXH3-128 checksum.
ChecksumXXHash128
// Keep after all valid checksums
checksumLast
@@ -112,7 +129,15 @@ const (
amzChecksumSHA1 = "x-amz-checksum-sha1"
amzChecksumSHA256 = "x-amz-checksum-sha256"
amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme"
amzChecksumMD5 = "x-amz-checksum-md5"
amzChecksumSHA512 = "x-amz-checksum-sha512"
amzChecksumXXHash64 = "x-amz-checksum-xxhash64"
amzChecksumXXHash3 = "x-amz-checksum-xxhash3"
amzChecksumXXHash128 = "x-amz-checksum-xxhash128"
amzChecksumMode = "x-amz-checksum-type"
amzChecksumModeComposite = "COMPOSITE"
amzChecksumModeFullObject = "FULL_OBJECT"
)
// Base returns the base type, without modifiers.
@@ -139,6 +164,16 @@ func (c ChecksumType) Key() string {
return amzChecksumSHA256
case ChecksumCRC64NVME:
return amzChecksumCRC64NVME
case ChecksumMD5:
return amzChecksumMD5
case ChecksumSHA512:
return amzChecksumSHA512
case ChecksumXXHash64:
return amzChecksumXXHash64
case ChecksumXXHash3:
return amzChecksumXXHash3
case ChecksumXXHash128:
return amzChecksumXXHash128
}
return ""
}
@@ -146,7 +181,8 @@ func (c ChecksumType) Key() string {
// CanComposite will return if the checksum type can be used for composite multipart upload on AWS.
func (c ChecksumType) CanComposite() bool {
switch c & checksumMask {
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C:
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C,
ChecksumMD5, ChecksumSHA512, ChecksumXXHash64, ChecksumXXHash3, ChecksumXXHash128:
return true
}
return false
@@ -186,6 +222,14 @@ func (c ChecksumType) RawByteLen() int {
return sha256.Size
case ChecksumCRC64NVME:
return crc64nvme.Size
case ChecksumXXHash64, ChecksumXXHash3:
return 8
case ChecksumMD5:
return md5.Size
case ChecksumSHA512:
return sha512.Size
case ChecksumXXHash128:
return 16
}
return 0
}
@@ -206,6 +250,16 @@ func (c ChecksumType) Hasher() hash.Hash {
return sha256.New()
case ChecksumCRC64NVME:
return crc64nvme.New()
case ChecksumMD5:
return md5.New()
case ChecksumSHA512:
return sha512.New()
case ChecksumXXHash64:
return xxhash.New()
case ChecksumXXHash3:
return xxh3.New()
case ChecksumXXHash128:
return xxh3.New128()
}
return nil
}
@@ -233,7 +287,6 @@ func (c ChecksumType) EncodeToString(b []byte) string {
}
// String returns the type as a string.
// CRC32, CRC32C, SHA1, and SHA256 for valid values.
// Empty string for unset and "<invalid>" if not valid.
func (c ChecksumType) String() string {
switch c & checksumMask {
@@ -249,6 +302,16 @@ func (c ChecksumType) String() string {
return ""
case ChecksumCRC64NVME:
return "CRC64NVME"
case ChecksumMD5:
return "MD5"
case ChecksumSHA512:
return "SHA512"
case ChecksumXXHash64:
return "XXHASH64"
case ChecksumXXHash3:
return "XXHASH3"
case ChecksumXXHash128:
return "XXHASH128"
}
return "<invalid>"
}
+4 -4
View File
@@ -94,15 +94,15 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
},
"us-iso-east-1": {
"s3.us-iso-east-1.c2s.ic.gov",
"s3.dualstack.us-iso-east-1.c2s.ic.gov",
"", // dualstack endpoint doesn't exist
},
"us-isob-east-1": {
"s3.us-isob-east-1.sc2s.sgov.gov",
"s3.dualstack.us-isob-east-1.sc2s.sgov.gov",
"", // dualstack endpoint doesn't exist
},
"us-iso-west-1": {
"s3.us-iso-west-1.c2s.ic.gov",
"s3.dualstack.us-iso-west-1.c2s.ic.gov",
"", // dualstack endpoint doesn't exist
},
"us-west-2": {
"s3.us-west-2.amazonaws.com",
@@ -269,7 +269,7 @@ func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
}
return "s3.us-east-1.amazonaws.com"
}
if useDualstack {
if useDualstack && s3Endpoint.dualstackEndpoint != "" {
return s3Endpoint.dualstackEndpoint
}
return s3Endpoint.endpoint
+160 -4
View File
@@ -73,8 +73,25 @@ const (
enableKMS = "ENABLE_KMS"
appVersion = "0.1.0"
skipCERTValidation = "SKIP_CERT_VALIDATION"
// TODO: remove when server supports the 2026 checksum types.
ignore2026Checksums = true
)
func ignore2026ChecksumError(cs minio.ChecksumType, err error) bool {
if !ignore2026Checksums {
return false
}
switch cs.Base() {
case minio.ChecksumMD5, minio.ChecksumSHA512,
minio.ChecksumXXHash64, minio.ChecksumXXHash3, minio.ChecksumXXHash128:
default:
return false
}
var er minio.ErrorResponse
return errors.As(err, &er) && er.Code == "InvalidArgument"
}
func createHTTPTransport() (transport *http.Transport) {
var err error
transport, err = minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
@@ -2096,6 +2113,11 @@ func testPutObjectWithChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
{cs: minio.ChecksumSHA1},
@@ -2160,6 +2182,10 @@ func testPutObjectWithChecksums() {
UserMetadata: meta,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2168,6 +2194,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
@@ -2191,6 +2222,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if st.ChecksumMode != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String()))
}
@@ -2238,6 +2274,11 @@ func testPutObjectWithChecksums() {
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
cmpChecksum(st.ChecksumMD5, "")
cmpChecksum(st.ChecksumSHA512, "")
cmpChecksum(st.ChecksumXXHash64, "")
cmpChecksum(st.ChecksumXXHash3, "")
cmpChecksum(st.ChecksumXXHash128, "")
delete(args, "range")
delete(args, "metadata")
@@ -2283,6 +2324,11 @@ func testPutObjectWithTrailingChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumCRC64NVME},
{cs: minio.ChecksumCRC32C},
{cs: minio.ChecksumCRC32},
@@ -2329,6 +2375,10 @@ func testPutObjectWithTrailingChecksums() {
Checksum: test.cs,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2341,6 +2391,11 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
// Read the data back
gopts := minio.GetObjectOptions{Checksum: true}
@@ -2361,7 +2416,12 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if st.Size != int64(bufSize) {
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
@@ -2407,6 +2467,11 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(st.ChecksumCRC32, "")
cmpChecksum(st.ChecksumCRC32C, "")
cmpChecksum(st.ChecksumCRC64NVME, "")
cmpChecksum(st.ChecksumMD5, "")
cmpChecksum(st.ChecksumSHA512, "")
cmpChecksum(st.ChecksumXXHash64, "")
cmpChecksum(st.ChecksumXXHash3, "")
cmpChecksum(st.ChecksumXXHash128, "")
function = "GetObjectAttributes(...)"
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
@@ -2418,6 +2483,16 @@ func testPutObjectWithTrailingChecksums() {
cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"])
cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"])
cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
cmpChecksum(s.Checksum.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
cmpChecksum(s.Checksum.ChecksumMD5, meta["x-amz-checksum-md5"])
cmpChecksum(s.Checksum.ChecksumSHA512, meta["x-amz-checksum-sha512"])
cmpChecksum(s.Checksum.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
cmpChecksum(s.Checksum.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
cmpChecksum(s.Checksum.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
if s.Checksum.ChecksumType != "" && s.Checksum.ChecksumType != minio.ChecksumFullObjectMode.String() {
logError(testName, function, args, startTime, "", "ChecksumType mismatch in GetObjectAttributes", fmt.Errorf("want %s, got %s", minio.ChecksumFullObjectMode.String(), s.Checksum.ChecksumType))
return
}
delete(args, "range")
delete(args, "metadata")
@@ -2495,6 +2570,11 @@ func testPutMultipartObjectWithChecksums() {
tests := []struct {
cs minio.ChecksumType
}{
{cs: minio.ChecksumMD5},
{cs: minio.ChecksumSHA512},
{cs: minio.ChecksumXXHash64},
{cs: minio.ChecksumXXHash3},
{cs: minio.ChecksumXXHash128},
{cs: minio.ChecksumFullObjectCRC32},
{cs: minio.ChecksumFullObjectCRC32C},
{cs: minio.ChecksumCRC64NVME},
@@ -2552,6 +2632,10 @@ func testPutMultipartObjectWithChecksums() {
Checksum: cs,
})
if err != nil {
if ignore2026ChecksumError(test.cs, err) {
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
continue
}
logError(testName, function, args, startTime, "", "PutObject failed", err)
return
}
@@ -2567,6 +2651,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(resp.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(resp.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(resp.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(resp.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(resp.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(resp.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(resp.ChecksumXXHash128, wantChksm)
}
args["section"] = "HeadObject"
@@ -2586,6 +2680,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(st.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantChksm)
}
// Use the CopyObject API to make a copy, in the case it was a composite checksum,
@@ -2622,6 +2726,16 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm)
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantFullObjectChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantFullObjectChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantFullObjectChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantFullObjectChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantFullObjectChksm)
}
args["section"] = "GetObjectAttributes"
@@ -2644,6 +2758,35 @@ func testPutMultipartObjectWithChecksums() {
cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm)
case minio.ChecksumSHA256:
cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm)
case minio.ChecksumCRC64NVME:
cmpChecksum(s.Checksum.ChecksumCRC64NVME, wantChksm)
case minio.ChecksumMD5:
cmpChecksum(s.Checksum.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(s.Checksum.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(s.Checksum.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(s.Checksum.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(s.Checksum.ChecksumXXHash128, wantChksm)
}
if s.Checksum.ChecksumType != "" {
var wantType string
if test.cs.FullObjectRequested() {
wantType = minio.ChecksumFullObjectMode.String()
} else {
wantType = minio.ChecksumCompositeMode.String()
}
cmpChecksum(s.Checksum.ChecksumType, wantType)
}
for _, part := range s.ObjectParts.Parts {
if test.cs == minio.ChecksumCRC64NVME && part.ChecksumCRC64NVME == "" {
logError(testName, function, args, startTime, "", "Part missing CRC64NVME checksum in GetObjectAttributes", fmt.Errorf("part %d", part.PartNumber))
return
}
}
// Read the data back
@@ -2684,6 +2827,16 @@ func testPutMultipartObjectWithChecksums() {
if st.ChecksumCRC64NVME != "" {
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
}
case minio.ChecksumMD5:
cmpChecksum(st.ChecksumMD5, wantChksm)
case minio.ChecksumSHA512:
cmpChecksum(st.ChecksumSHA512, wantChksm)
case minio.ChecksumXXHash64:
cmpChecksum(st.ChecksumXXHash64, wantChksm)
case minio.ChecksumXXHash3:
cmpChecksum(st.ChecksumXXHash3, wantChksm)
case minio.ChecksumXXHash128:
cmpChecksum(st.ChecksumXXHash128, wantChksm)
}
delete(args, "metadata")
@@ -3475,6 +3628,8 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
checksumFound = true
} else if v.ChecksumCRC32C != "" {
checksumFound = true
} else if v.ChecksumCRC64NVME != "" {
checksumFound = true
}
if !checksumFound {
partsMissingChecksum = true
@@ -3497,6 +3652,7 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" ||
OA.Checksum.ChecksumCRC32C != "" ||
OA.Checksum.ChecksumCRC64NVME != "" ||
OA.Checksum.ChecksumSHA1 != "" ||
OA.Checksum.ChecksumSHA256 != "")
@@ -14789,6 +14945,9 @@ func main() {
// execute tests
if isFullMode() {
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums()
testCopyObjectWithChecksums()
testReplaceObjectWithChecksums()
testCorsSetGetDelete()
@@ -14808,9 +14967,6 @@ func main() {
testComposeObjectErrorCasesV2()
testCompose10KSourcesV2()
testUserMetadataCopyingV2()
testPutObjectWithChecksums()
testPutObjectWithTrailingChecksums()
testPutMultipartObjectWithChecksums()
testPutObject0ByteV2()
testPutObjectMetadataNonUSASCIIV2()
testPutObjectNoLengthV2()
+21
View File
@@ -119,6 +119,10 @@ var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`)
// amazonS3HostPrivateLink - regular expression used to determine if an arg is s3 host in AWS PrivateLink interface endpoints style
var amazonS3HostPrivateLink = regexp.MustCompile(`^(?:bucket|accesspoint).vpce-.*?.s3.(.*?).vpce.amazonaws.com$`)
// amazonS3HostOutposts - regular expression used to determine if an arg is S3 on Outposts endpoint.
// Pattern: <something>.s3-outposts.<region>.amazonaws.com
var amazonS3HostOutposts = regexp.MustCompile(`^(.+)\.s3-outposts\.([a-z0-9-]+)\.amazonaws\.com$`)
// GetRegionFromURL - returns a region from url host.
func GetRegionFromURL(endpointURL url.URL) string {
if endpointURL == sentinelURL {
@@ -181,6 +185,11 @@ func GetRegionFromURL(endpointURL url.URL) string {
return parts[1]
}
parts = amazonS3HostOutposts.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 2 {
return parts[2]
}
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
if len(parts) > 1 {
if strings.HasPrefix(parts[1], "xpress-") {
@@ -210,8 +219,20 @@ func IsAmazonExpressZonalEndpoint(endpointURL url.URL) bool {
return amazonS3HostExpress.MatchString(endpointURL.Hostname())
}
// IsAmazonOutpostsEndpoint - Match if the endpoint is S3 on Outposts endpoint.
func IsAmazonOutpostsEndpoint(endpointURL url.URL) bool {
if endpointURL == sentinelURL {
return false
}
return amazonS3HostOutposts.MatchString(endpointURL.Hostname())
}
// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint.
// S3 on Outposts is not treated as Amazon S3 here so that the client keeps path-style and does not replace the host.
func IsAmazonEndpoint(endpointURL url.URL) bool {
if IsAmazonOutpostsEndpoint(endpointURL) {
return false
}
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" || endpointURL.Hostname() == "s3.amazonaws.com" {
return true
}
@@ -90,32 +90,34 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
return streamLen
}
// buildChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
// buildChunkStringToSignWithService - like buildChunkStringToSign but with configurable service type.
func buildChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
if serviceType == "" {
serviceType = ServiceTypeS3
}
stringToSignParts := []string{
streamingPayloadHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
getScope(region, t, serviceType),
previousSig,
emptySHA256,
chunkChecksum,
}
return strings.Join(stringToSignParts, "\n")
}
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
// and previous signature.
func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
// buildTrailerChunkStringToSignWithService - like buildTrailerChunkStringToSign but with configurable service type.
func buildTrailerChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
if serviceType == "" {
serviceType = ServiceTypeS3
}
stringToSignParts := []string{
streamingTrailerHdr,
t.Format(iso8601DateFormat),
getScope(region, t, ServiceTypeS3),
getScope(region, t, serviceType),
previousSig,
chunkChecksum,
}
return strings.Join(stringToSignParts, "\n")
}
@@ -150,36 +152,42 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
// serviceType defaults to ServiceTypeS3 when empty.
func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
previousSignature, secretAccessKey string,
previousSignature, secretAccessKey, serviceType string,
) string {
chunkStringToSign := buildChunkStringToSign(reqTime, region,
previousSignature, chunkCheckSum)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
if serviceType == "" {
serviceType = ServiceTypeS3
}
chunkStringToSign := buildChunkStringToSignWithService(reqTime, region,
previousSignature, chunkCheckSum, serviceType)
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
return getSignature(signingKey, chunkStringToSign)
}
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
// buildTrailerChunkSignature - returns chunk signature for trailer chunk.
// serviceType defaults to ServiceTypeS3 when empty.
func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
previousSignature, secretAccessKey string,
previousSignature, secretAccessKey, serviceType string,
) string {
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
previousSignature, chunkChecksum)
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
if serviceType == "" {
serviceType = ServiceTypeS3
}
chunkStringToSign := buildTrailerChunkStringToSignWithService(reqTime, region,
previousSignature, chunkChecksum, serviceType)
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
return getSignature(signingKey, chunkStringToSign)
}
// getSeedSignature - returns the seed signature for a given request.
func (s *StreamingReader) setSeedSignature(req *http.Request) {
// Get canonical request
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req))
// Get string to sign from canonical request.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3)
// Calculate signature.
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, serviceType)
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, serviceType)
s.seedSignature = getSignature(signingKey, stringToSign)
}
@@ -190,6 +198,7 @@ type StreamingReader struct {
secretAccessKey string
sessionToken string
region string
serviceType string // e.g. ServiceTypeS3, ServiceTypeS3Outposts; empty means S3
prevSignature string
seedSignature string
contentLen int64 // Content-Length from req header
@@ -214,8 +223,12 @@ func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
s.sh256.Write(s.chunkBuf[:chunkLen])
chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
signature := buildChunkSignature(chunckChecksum, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
s.region, s.prevSignature, s.secretAccessKey, serviceType)
// For next chunk signature computation
s.prevSignature = signature
@@ -249,9 +262,12 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
s.sh256.Reset()
s.sh256.Write(s.chunkBuf)
chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
// Compute chunk signature
serviceType := s.serviceType
if serviceType == "" {
serviceType = ServiceTypeS3
}
signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
s.region, s.prevSignature, s.secretAccessKey)
s.region, s.prevSignature, s.secretAccessKey, serviceType)
// For next chunk signature computation
s.prevSignature = signature
@@ -376,6 +392,40 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
return req
}
// StreamingSignV4Outposts - provides chunked upload signatureV4 support for S3 on Outposts (service name s3-outposts).
func StreamingSignV4Outposts(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
) *http.Request {
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
if req.Body == nil {
req.Body = io.NopCloser(bytes.NewReader([]byte("")))
}
stReader := &StreamingReader{
baseReadCloser: req.Body,
accessKeyID: accessKeyID,
secretAccessKey: secretAccessKey,
sessionToken: sessionToken,
region: region,
serviceType: ServiceTypeS3Outposts,
reqTime: reqTime,
chunkBuf: make([]byte, payloadChunkSize),
contentLen: dataLen,
chunkNum: 1,
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
lastChunkSize: int(dataLen % payloadChunkSize),
sh256: sh256,
}
if len(req.Trailer) > 0 {
stReader.trailer = req.Trailer
req.Trailer = nil
}
stReader.setSeedSignature(req)
stReader.setStreamingAuthHeader(req, ServiceTypeS3Outposts)
stReader.prevSignature = stReader.seedSignature
req.Body = stReader
return req
}
// Read - this method performs chunk upload signature providing a
// io.Reader interface.
func (s *StreamingReader) Read(buf []byte) (int, error) {
+51 -3
View File
@@ -38,9 +38,10 @@ const (
// Different service types
const (
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
ServiceTypeS3Express = "s3express"
ServiceTypeS3 = "s3"
ServiceTypeSTS = "sts"
ServiceTypeS3Express = "s3express"
ServiceTypeS3Outposts = "s3-outposts"
)
// Excerpts from @lsegal -
@@ -256,6 +257,38 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
return &req
}
// PreSignV4Outposts presign the request for S3 on Outposts (service name s3-outposts).
func PreSignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
// Presign is not needed for anonymous credentials.
if accessKeyID == "" || secretAccessKey == "" {
return &req
}
t := time.Now().UTC()
credential := GetCredential(accessKeyID, location, t, ServiceTypeS3Outposts)
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
query := req.URL.Query()
query.Set("X-Amz-Algorithm", signV4Algorithm)
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
query.Set("X-Amz-SignedHeaders", signedHeaders)
query.Set("X-Amz-Credential", credential)
if sessionToken != "" {
if v := req.Header.Get("x-amz-s3session-token"); v != "" {
query.Set("X-Amz-S3session-Token", sessionToken)
} else {
query.Set("X-Amz-Security-Token", sessionToken)
}
}
req.URL.RawQuery = query.Encode()
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))
stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3Outposts)
signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3Outposts)
signature := getSignature(signingKey, stringToSign)
req.URL.RawQuery += "&X-Amz-Signature=" + signature
return &req
}
// PostPresignSignatureV4 - presigned signature for PostPolicy
// requests.
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
@@ -393,3 +426,18 @@ func SignV4TrailerExpress(req http.Request, accessKeyID, secretAccessKey, sessio
func SignV4Trailer(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, trailer)
}
// SignV4Outposts sign the request for S3 on Outposts (service name s3-outposts).
func SignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, nil)
}
// SignV4WithServiceType signs a request with AWS Signature Version 4 using a custom service type.
func SignV4WithServiceType(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, serviceType, nil)
}
// SignV4TrailerOutposts sign the request with trailer for S3 on Outposts (service name s3-outposts).
func SignV4TrailerOutposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, trailer)
}
+21
View File
@@ -18,6 +18,7 @@
package tags
import (
"encoding/json"
"encoding/xml"
"io"
"net/url"
@@ -293,6 +294,26 @@ func (tags Tags) ToMap() map[string]string {
return tags.TagSet.toMap()
}
// MarshalJSON encodes Tags as a flat JSON object {"key":"value",...}.
func (tags Tags) MarshalJSON() ([]byte, error) {
return json.Marshal(tags.ToMap())
}
// UnmarshalJSON decodes a flat JSON object {"key":"value",...} into Tags.
func (tags *Tags) UnmarshalJSON(data []byte) error {
var m map[string]string
if err := json.Unmarshal(data, &m); err != nil {
return err
}
if tags.TagSet == nil {
tags.TagSet = &tagSet{
tagMap: make(map[string]string),
}
}
tags.TagSet.tagMap = m
return nil
}
// MapToObjectTags converts an input map of key and value into
// *Tags data structure with validation.
func MapToObjectTags(tagMap map[string]string) (*Tags, error) {
+27
View File
@@ -179,6 +179,11 @@ func isValidEndpointURL(endpointURL url.URL) error {
return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
}
}
if strings.Contains(host, "s3-outposts") {
if !s3utils.IsAmazonOutpostsEndpoint(endpointURL) {
return errInvalidArgument("S3 Outposts endpoint must match <prefix>.s3-outposts.<region>.amazonaws.com")
}
}
return nil
}
@@ -304,6 +309,21 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Region: h.Get("x-amz-bucket-region"),
}
}
mtimeStr := h.Get("X-Minio-Source-Mtime")
if mtimeStr != "" {
mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
if err != nil {
return ObjectInfo{}, ErrorResponse{
Code: InternalError,
Message: fmt.Sprintf("X-Minio-Source-Mtime is not in supported format: %v", err),
BucketName: bucketName,
Key: objectName,
RequestID: h.Get("x-amz-request-id"),
HostID: h.Get("x-amz-id-2"),
Region: h.Get("x-amz-bucket-region"),
}
}
}
// Fetch content type if any present.
contentType := strings.TrimSpace(h.Get("Content-Type"))
@@ -381,6 +401,7 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
Size: size,
LastModified: mtime,
ContentType: contentType,
ContentEncoding: strings.TrimSpace(h.Get("Content-Encoding")),
Expires: expiry,
VersionID: h.Get(amzVersionID),
IsDeleteMarker: deleteMarker,
@@ -402,6 +423,12 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
ChecksumMD5: h.Get(ChecksumMD5.Key()),
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
ChecksumAlgorithm: h.Get(amzChecksumAlgo),
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
}, nil
}
+33 -15
View File
@@ -696,7 +696,7 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
// If we are here we have an auth callout defined and we have failed auth so far
// so we will callout to our auth backend for processing.
if !skip {
authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy)
authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy, ujwt)
}
// Check if we are authorized and in the auth callout account, and if so add in deny publish permissions for the auth subject.
if authorized {
@@ -797,26 +797,42 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
token = opts.Authorization
}
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
if s.trustedKeys != nil {
ujwt = c.opts.JWT
if ujwt == _EMPTY_ && c.isMqtt() {
// For MQTT, we pass the password as the JWT too, but do so here so it's not
// publicly exposed in the client options if it isn't a JWT.
// MQTT can carry JWTs in the password field. Reconstruct it here for auth
// processing and auth callout, but do not populate c.opts.JWT yet or it would
// be exposed through monitoring and advisory paths even when the password is
// not actually a JWT.
if ujwt == _EMPTY_ && c.isMqtt() && c.opts.JWT == _EMPTY_ {
// Don't set juc here, leave that to the next s.trustedKeys != nil block,
// so that we don't try to trust a JWT when we aren't in operator mode. We
// will allow it to be passed through auth callout though.
if _, err := jwt.DecodeUserClaims(c.opts.Password); err == nil {
ujwt = c.opts.Password
}
if ujwt == _EMPTY_ && opts.DefaultSentinel != _EMPTY_ {
c.opts.JWT = opts.DefaultSentinel
ujwt = c.opts.JWT
}
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
if s.trustedKeys != nil {
if ujwt == _EMPTY_ {
// Need to be sure that it's a NATS JWT, otherwise we will not correctly
// attempt the default sentinel below.
if _, err = jwt.DecodeUserClaims(c.opts.JWT); err == nil {
ujwt = c.opts.JWT
}
}
if ujwt == _EMPTY_ {
// Didn't fall through with a valid NATS JWT, so try the default sentinel
// if configured.
if opts.DefaultSentinel != _EMPTY_ {
c.opts.JWT = opts.DefaultSentinel
ujwt = c.opts.JWT
}
}
if ujwt == _EMPTY_ {
s.mu.Unlock()
c.Debugf("Authentication requires a user JWT")
return false
}
// So we have a valid user jwt here.
juc, err = jwt.DecodeUserClaims(ujwt)
if err != nil {
if juc, err = jwt.DecodeUserClaims(ujwt); err != nil {
s.mu.Unlock()
c.Debugf("User JWT not valid: %v", err)
return false
@@ -1015,8 +1031,10 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
c.Debugf("Connection type not allowed")
return false
}
// skip validation of nonce when presented with a bearer token
// FIXME: if BearerToken is only for WSS, need check for server with that port enabled
// Skip validation of nonce when presented with a bearer token.
// While support for bearer tokens was added for WebSockets, there is no
// security benefit in restricting their use to that client protocol: the
// client can just go use the other protocol.
if !juc.BearerToken {
// Verify the signature against the nonce.
if c.opts.Sig == _EMPTY_ {
+10 -4
View File
@@ -41,7 +41,7 @@ func titleCase(m string) string {
}
// Process a callout on this client's behalf.
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool) (authorized bool, errStr string) {
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool, ujwt string) (authorized bool, errStr string) {
isOperatorMode := len(opts.TrustedKeys) > 0
// this is the account the user connected in, or the one running the callout
@@ -374,7 +374,7 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequi
// Grab client info for the request.
c.mu.Lock()
c.fillClientInfo(&claim.ClientInformation)
c.fillConnectOpts(&claim.ConnectOptions)
c.fillConnectOpts(&claim.ConnectOptions, ujwt)
// If we have a sig in the client opts, fill in nonce.
if claim.ConnectOptions.SignedNonce != _EMPTY_ {
claim.ClientInformation.Nonce = string(c.nonce)
@@ -474,16 +474,22 @@ func (c *client) fillClientInfo(ci *jwt.ClientInformation) {
// Fill in client options.
// Lock should be held.
func (c *client) fillConnectOpts(opts *jwt.ConnectOptions) {
func (c *client) fillConnectOpts(opts *jwt.ConnectOptions, ujwt string) {
if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) {
return
}
o := c.opts
if ujwt == _EMPTY_ {
// The caller may supply a reconstructed JWT that should be sent to auth
// callout without storing it in c.opts.JWT. If not, fall back to the client
// option as before.
ujwt = o.JWT
}
// Do it this way to fail to compile if fields are added to jwt.ClientInformation.
*opts = jwt.ConnectOptions{
JWT: o.JWT,
JWT: ujwt,
Nkey: o.Nkey,
SignedNonce: o.Sig,
Token: o.Token,
+2 -2
View File
@@ -239,7 +239,7 @@ func (ss SequenceSet) EncodeLen() int {
return minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2))
}
func (ss SequenceSet) Encode(buf []byte) ([]byte, error) {
func (ss SequenceSet) Encode(buf []byte) []byte {
nn, encLen := ss.Nodes(), ss.EncodeLen()
if cap(buf) < encLen {
@@ -268,7 +268,7 @@ func (ss SequenceSet) Encode(buf []byte) ([]byte, error) {
le.PutUint16(buf[i:], uint16(n.h))
i += 2
})
return buf[:i], nil
return buf[:i]
}
// ErrBadEncoding is returned when we can not decode properly.
+191 -122
View File
@@ -1061,18 +1061,19 @@ func (c *client) setPermissions(perms *Permissions) {
return
}
c.perms = &permissions{}
slcache := c.srv != nil && !c.srv.getOpts().NoSublistCache
// Loop over publish permissions
if perms.Publish != nil {
if perms.Publish.Allow != nil {
c.perms.pub.allow = NewSublistWithCache()
c.perms.pub.allow = NewSublist(slcache)
}
for _, pubSubject := range perms.Publish.Allow {
sub := &subscription{subject: []byte(pubSubject)}
c.perms.pub.allow.Insert(sub)
}
if len(perms.Publish.Deny) > 0 {
c.perms.pub.deny = NewSublistWithCache()
c.perms.pub.deny = NewSublist(slcache)
}
for _, pubSubject := range perms.Publish.Deny {
sub := &subscription{subject: []byte(pubSubject)}
@@ -1091,7 +1092,7 @@ func (c *client) setPermissions(perms *Permissions) {
if perms.Subscribe != nil {
var err error
if len(perms.Subscribe.Allow) > 0 {
c.perms.sub.allow = NewSublistWithCache()
c.perms.sub.allow = NewSublist(slcache)
}
for _, subSubject := range perms.Subscribe.Allow {
sub := &subscription{}
@@ -1103,7 +1104,7 @@ func (c *client) setPermissions(perms *Permissions) {
c.perms.sub.allow.Insert(sub)
}
if len(perms.Subscribe.Deny) > 0 {
c.perms.sub.deny = NewSublistWithCache()
c.perms.sub.deny = NewSublist(slcache)
// Also hold onto this array for later.
c.darray = perms.Subscribe.Deny
}
@@ -1200,6 +1201,7 @@ func (c *client) mergeDenyPermissions(what denyType, denyPubs []string) {
if c.perms == nil {
c.perms = &permissions{}
}
slcache := c.srv != nil && !c.srv.getOpts().NoSublistCache
var perms []*perm
switch what {
case pub:
@@ -1211,7 +1213,7 @@ func (c *client) mergeDenyPermissions(what denyType, denyPubs []string) {
}
for _, p := range perms {
if p.deny == nil {
p.deny = NewSublistWithCache()
p.deny = NewSublist(slcache)
}
FOR_DENY:
for _, subj := range denyPubs {
@@ -2254,12 +2256,20 @@ func (c *client) processConnect(arg []byte) error {
// least ClientProtoInfo, we need to increment the following counter.
// This is decremented when client is removed from the server's
// clients map.
if kind == CLIENT && proto >= ClientProtoInfo {
if kind == CLIENT && proto >= ClientProtoInfo && firstConnect {
srv.mu.Lock()
srv.cproto++
srv.mu.Unlock()
}
// A second CONNECT may move the client into a different account via
// checkAuthentication. Drop any previously-registered subscriptions
// from the current account first so they don't leak in that account's
// sublist after the client switches.
if !firstConnect {
c.clearAccountSubs(false)
}
// Check for Auth
if ok := srv.checkAuthentication(c); !ok {
// We may fail here because we reached max limits on an account.
@@ -3273,19 +3283,20 @@ func (c *client) canSubscribe(subject string, optQueue ...string) bool {
r := c.perms.sub.deny.Match(subject)
allowed = len(r.psubs) == 0
if queue != _EMPTY_ && len(r.qsubs) > 0 {
if allowed && queue != _EMPTY_ && len(r.qsubs) > 0 {
// If the queue appears in the deny list, then DO NOT allow.
allowed = !queueMatches(queue, r.qsubs)
}
// We use the actual subscription to signal us to spin up the deny mperms
// and cache. We check if the subject is a wildcard that contains any of
// and cache. We check if the subject is a wildcard that intersects any of
// the deny clauses.
// FIXME(dlc) - We could be smarter and track when these go away and remove.
if allowed && c.mperms == nil && subjectHasWildcard(subject) {
// Whip through the deny array and check if this wildcard subject is within scope.
// Whip through the deny array and check if this wildcard subject can
// overlap with any denied deliveries.
for _, sub := range c.darray {
if subjectIsSubsetMatch(sub, subject) {
if SubjectsCollide(sub, subject) {
c.loadMsgDenyFilter()
break
}
@@ -3658,14 +3669,7 @@ func (c *client) deliverMsg(prodIsMQTT bool, sub *subscription, acc *Account, su
// Check if we are a leafnode and have perms to check.
if client.kind == LEAF && client.perms != nil {
var subjectToCheck []byte
if subject[0] == '_' && bytes.HasPrefix(subject, []byte(gwReplyPrefix)) {
subjectToCheck = subject[gwSubjectOffset:]
} else if subject[0] == '$' && bytes.HasPrefix(subject, []byte(oldGWReplyPrefix)) {
subjectToCheck = subject[oldGWReplyStart:]
} else {
subjectToCheck = subject
}
subjectToCheck, _ := getGWRoutedSubjectOrSelf(subject)
if !client.pubAllowedFullCheck(string(subjectToCheck), true, true) {
mt.addEgressEvent(client, sub, errMsgTracePubViolation)
client.mu.Unlock()
@@ -4068,7 +4072,7 @@ func (c *client) allowedMsgTraceDest(hdr []byte, hasLock bool) (string, bool) {
return _EMPTY_, true
}
td := sliceHeader(MsgTraceDest, hdr)
if len(td) == 0 {
if len(td) == 0 || bytes.Equal(td, traceDestDisabledAsBytes) {
return _EMPTY_, true
}
dest := bytesToString(td)
@@ -4131,17 +4135,7 @@ func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bo
if !hasLock {
c.mu.Lock()
}
if resp := c.replies[subject]; resp != nil {
resp.n++
// Check if we have sent too many responses.
if c.perms.resp.MaxMsgs > 0 && resp.n > c.perms.resp.MaxMsgs {
delete(c.replies, subject)
} else if c.perms.resp.Expires > 0 && time.Since(resp.t) > c.perms.resp.Expires {
delete(c.replies, subject)
} else {
allowed = true
}
}
allowed = c.responseAllowed(subject)
if !hasLock {
c.mu.Unlock()
}
@@ -4155,6 +4149,25 @@ func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bo
return allowed
}
// Returns true if this subject matches a tracked dynamic reply permission.
// Lock must be held.
func (c *client) responseAllowed(subject string) bool {
if c.perms == nil || c.perms.resp == nil {
return false
}
if resp := c.replies[subject]; resp != nil {
resp.n++
if c.perms.resp.MaxMsgs > 0 && resp.n > c.perms.resp.MaxMsgs {
delete(c.replies, subject)
} else if c.perms.resp.Expires > 0 && time.Since(resp.t) > c.perms.resp.Expires {
delete(c.replies, subject)
} else {
return true
}
}
return false
}
// Test whether a reply subject is a service import reply.
func isServiceReply(reply []byte) bool {
// This function is inlined and checking this way is actually faster
@@ -4162,16 +4175,51 @@ func isServiceReply(reply []byte) bool {
return len(reply) > 3 && bytesToString(reply[:4]) == replyPrefix
}
// Test whether a subject is a JetStream ACK.
func isJSAckSubject(subject []byte) bool {
return len(subject) > jsAckPreLen && bytesToString(subject[:jsAckPreLen]) == jsAckPre
}
// jsAckDeliverIdx returns the byte offset of the `@` separator in an encoded
// `$JS.ACK....@<deliver>` reply, or -1 if reply is not in that form. Stream,
// consumer, and subject tokens may legally contain `@`, so we accept only the
// first `@` that follows the eight dots of the JS ACK token:
//
// $JS.ACK.<stream>.<consumer>.<delivered>.<sseq>.<cseq>.<tm>.<pending>@<deliver>
func jsAckDeliverIdx(reply []byte) int {
if !isJSAckSubject(reply) {
return -1
}
dots := 0
for i, b := range reply {
switch b {
case '.':
dots++
case '@':
if dots >= 8 {
return i
}
}
}
return -1
}
// replyHasJSAckSuffix reports whether reply is already in `$JS.ACK....@<deliver>`
// form, so callers don't double-append the suffix on a re-entrant pass
// (service-import or chained JS push).
func replyHasJSAckSuffix(reply []byte) bool {
return jsAckDeliverIdx(reply) != -1
}
// Test whether a reply subject is a service import or a gateway routed reply.
func isReservedReply(reply []byte) bool {
if isServiceReply(reply) {
return true
}
rLen := len(reply)
// Faster to check with string([:]) than byte-by-byte
if rLen > jsAckPreLen && bytesToString(reply[:jsAckPreLen]) == jsAckPre {
if isJSAckSubject(reply) {
return true
} else if rLen > gwReplyPrefixLen && bytesToString(reply[:gwReplyPrefixLen]) == gwReplyPrefix {
} else if len(reply) > gwReplyPrefixLen && bytesToString(reply[:gwReplyPrefixLen]) == gwReplyPrefix {
return true
}
return false
@@ -4370,7 +4418,7 @@ func (c *client) processInboundClientMsg(msg []byte) (bool, bool) {
// Now deal with gateways
if c.srv.gateway.enabled {
reply := c.pa.reply
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 {
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(reply) > 0 && !replyHasJSAckSuffix(reply) {
reply = append(reply, '@')
reply = append(reply, c.pa.deliver...)
}
@@ -4418,7 +4466,7 @@ func (c *client) handleGWReplyMap(msg []byte) bool {
}
if c.srv.gateway.enabled {
reply := c.pa.reply
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 {
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(reply) > 0 && !replyHasJSAckSuffix(reply) {
reply = append(reply, '@')
reply = append(reply, c.pa.deliver...)
}
@@ -4531,7 +4579,8 @@ func (c *client) setHeader(key, value string, msg []byte) []byte {
// Write original header if present.
if c.pa.hdr > LEN_CR_LF {
omi = c.pa.hdr
hdr := removeHeaderIfPresent(msg[:c.pa.hdr-LEN_CR_LF], key)
// Need to copy since we're removing the header in place.
hdr := removeHeaderIfPresent(copyBytes(msg[:c.pa.hdr-LEN_CR_LF]), key)
if len(hdr) == 0 {
bb.WriteString(hdrLine)
} else {
@@ -4825,6 +4874,12 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
// but the local server must replace it with the identity of the
// authenticated leaf connection instead of trusting forwarded values.
ci = c.getClientInfo(share)
if hadPrevSi && cis != nil && cis.Reply != _EMPTY_ {
ci.Reply = cis.Reply
} else if bytes.HasSuffix(c.pa.reply, []byte(FastBatchSuffix)) {
// Fast batch requires knowledge of the original reply subject.
ci.Reply = bytesToString(c.pa.reply)
}
if hadPrevSi {
ci.Service = acc.Name
if !share && (si.share || isSysImport) {
@@ -4843,6 +4898,10 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
}
} else if c.kind != LEAF || c.pa.hdr < 0 || len(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])) == 0 {
ci = c.getClientInfo(share)
// Fast batch requires knowledge of the original reply subject.
if bytes.HasSuffix(c.pa.reply, []byte(FastBatchSuffix)) {
ci.Reply = bytesToString(c.pa.reply)
}
// If we did not share but the imports destination is the system account add in the server and cluster info.
if !share && isSysImport {
c.addServerAndClusterInfo(ci)
@@ -4902,8 +4961,7 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
// We also need to disable the message trace headers so that
// if the message is routed, it does not initialize tracing in the
// remote.
positions := disableTraceHeaders(c, msg)
defer enableTraceHeaders(msg, positions)
msg = c.setHeader(MsgTraceDest, MsgTraceDestDisabled, msg)
}
}
}
@@ -5037,21 +5095,9 @@ func (c *client) processMsgResults(acc *Account, r *SublistResult, msg, deliver,
// Check for JetStream encoded reply subjects.
// For now these will only be on $JS.ACK prefixed reply subjects.
var remapped bool
if len(creply) > 0 && c.kind != CLIENT && !isInternalClient(c.kind) && bytes.HasPrefix(creply, []byte(jsAckPre)) {
if len(creply) > 0 && c.kind != CLIENT && !isInternalClient(c.kind) {
// We need to rewrite the subject and the reply.
// But, we must be careful that the stream name, consumer name, and subject can contain '@' characters.
// JS ACK contains at least 8 dots, find the first @ after this prefix.
// - $JS.ACK.<stream>.<consumer>.<delivered>.<sseq>.<cseq>.<tm>.<pending>
counter := 0
li := bytes.IndexFunc(creply, func(rn rune) bool {
if rn == '.' {
counter++
} else if rn == '@' {
return counter >= 8
}
return false
})
if li != -1 && li < len(creply)-1 {
if li := jsAckDeliverIdx(creply); li != -1 && li < len(creply)-1 {
remapped = true
subj, creply = creply[li+1:], creply[:li]
}
@@ -5471,7 +5517,7 @@ sendToRoutesOrLeafs:
// at the end of the reply subject if it exists. But only if this wasn't
// already performed, otherwise we'd end up with a duplicate '@' suffix
// resulting in a protocol error.
if len(deliver) > 0 && len(reply) > 0 && !remapped {
if len(deliver) > 0 && len(reply) > 0 && !remapped && !replyHasJSAckSuffix(reply) {
reply = append(reply, '@')
reply = append(reply, deliver...)
}
@@ -5754,11 +5800,12 @@ func (c *client) clearAuthTimer() bool {
return stopped
}
// We may reuse atmr for expiring user jwts,
// so check connectReceived.
// Track whether the parser should still enforce pre-CONNECT rules.
// This is handshake state, not timer state, since some handshakes
// use a different timer while still expecting CONNECT.
// Lock assume held on entry.
func (c *client) awaitingAuth() bool {
return !c.flags.isSet(connectReceived) && c.atmr != nil
return c.flags.isSet(expectConnect) && !c.flags.isSet(connectReceived)
}
// This will set the atmr for the JWT expiration time.
@@ -5987,37 +6034,12 @@ func (c *client) closeConnection(reason ClosedState) {
srv = c.srv
noReconnect = c.flags.isSet(noReconnect)
acc = c.acc
spoke bool
)
// Snapshot for use if we are a client connection.
// FIXME(dlc) - we can just stub in a new one for client
// and reference existing one.
var subs []*subscription
if kind == CLIENT || kind == LEAF || kind == JETSTREAM {
var _subs [32]*subscription
subs = _subs[:0]
// Do not set c.subs to nil or delete the sub from c.subs here because
// it will be needed in saveClosedClient (which has been started as a
// go routine in markConnAsClosed). Cleanup will be done there.
for _, sub := range c.subs {
// Auto-unsubscribe subscriptions must be unsubscribed forcibly.
sub.max = 0
sub.close()
subs = append(subs, sub)
}
spoke = c.isSpokeLeafNode()
}
c.mu.Unlock()
// Remove client's or leaf node or jetstream subscriptions.
if acc != nil && (kind == CLIENT || kind == LEAF || kind == JETSTREAM) {
acc.sl.RemoveBatch(subs)
} else if kind == ROUTER {
if kind == ROUTER {
c.removeRemoteSubs()
}
if srv != nil {
// Unregister
srv.removeClient(c)
@@ -6025,45 +6047,11 @@ func (c *client) closeConnection(reason ClosedState) {
if acc != nil {
// Update remote subscriptions.
if kind == CLIENT || kind == LEAF || kind == JETSTREAM {
qsubs := map[string]*qsub{}
for _, sub := range subs {
// Call unsubscribe here to cleanup shadow subscriptions and such.
c.unsubscribe(acc, sub, true, false)
// Update route as normal for a normal subscriber.
if sub.queue == nil {
if !spoke {
srv.updateRouteSubscriptionMap(acc, sub, -1)
if srv.gateway.enabled {
srv.gatewayUpdateSubInterest(acc.Name, sub, -1)
}
}
acc.updateLeafNodes(sub, -1)
} else {
// We handle queue subscribers special in case we
// have a bunch we can just send one update to the
// connected routes.
num := int32(1)
if kind == LEAF {
num = sub.qw
}
key := keyFromSub(sub)
if esub, ok := qsubs[key]; ok {
esub.n += num
} else {
qsubs[key] = &qsub{sub, num}
}
}
}
// Process any qsubs here.
for _, esub := range qsubs {
if !spoke {
srv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n))
if srv.gateway.enabled {
srv.gatewayUpdateSubInterest(acc.Name, esub.sub, -(esub.n))
}
}
acc.updateLeafNodes(esub.sub, -(esub.n))
}
// Remove client's subscriptions from the account and unregister
// client from that account. Keep c.subs populated because
// saveClosedClient (started as a goroutine in markConnAsClosed)
// still needs to read it.
c.clearAccountSubs(true)
}
// Always remove from the account, otherwise we can leak clients.
// Note that SYSTEM and ACCOUNT types from above cleanup their own subs.
@@ -6090,6 +6078,87 @@ func (c *client) closeConnection(reason ClosedState) {
c.reconnect()
}
// clearAccountSubs removes the client's subscriptions from its current account
// and unregisters it from that account. If close is true, c.subs is left
// populated for saveClosedClient; otherwise c.subs is cleared and c.acc
// registered back to the global account.
// Client lock MUST NOT be held on entry.
func (c *client) clearAccountSubs(close bool) {
c.mu.Lock()
kind := c.kind
srv := c.srv
acc := c.acc
if acc == nil || (kind != CLIENT && kind != LEAF && kind != JETSTREAM) {
c.mu.Unlock()
return
}
var _subs [32]*subscription
subs := _subs[:0]
// Do not set c.subs to nil or delete the sub from c.subs here because
// it will be needed in saveClosedClient (which has been started as a
// go routine in markConnAsClosed). Cleanup will be done there.
for _, sub := range c.subs {
// Auto-unsubscribe subscriptions must be unsubscribed forcibly.
sub.max = 0
sub.close()
subs = append(subs, sub)
if !close {
delete(c.subs, string(sub.sid))
}
}
spoke := c.isSpokeLeafNode()
c.mu.Unlock()
acc.sl.RemoveBatch(subs)
if srv != nil {
qsubs := map[string]*qsub{}
for _, sub := range subs {
// Call unsubscribe here to cleanup shadow subscriptions and such.
c.unsubscribe(acc, sub, true, false)
// Update route as normal for a normal subscriber.
if sub.queue == nil {
if !spoke {
srv.updateRouteSubscriptionMap(acc, sub, -1)
if srv.gateway.enabled {
srv.gatewayUpdateSubInterest(acc.Name, sub, -1)
}
}
acc.updateLeafNodes(sub, -1)
} else {
// We handle queue subscribers special in case we
// have a bunch we can just send one update to the
// connected routes.
num := int32(1)
if kind == LEAF {
num = sub.qw
}
key := keyFromSub(sub)
if esub, ok := qsubs[key]; ok {
esub.n += num
} else {
qsubs[key] = &qsub{sub, num}
}
}
}
// Process any qsubs here.
for _, esub := range qsubs {
if !spoke {
srv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n))
if srv.gateway.enabled {
srv.gatewayUpdateSubInterest(acc.Name, esub.sub, -(esub.n))
}
}
acc.updateLeafNodes(esub.sub, -(esub.n))
}
}
if !close {
// Register back to global account, mimicking the state after client initialization.
c.registerWithAccount(srv.globalAccount())
}
}
// Depending on the kind of connections, this may attempt to recreate a connection.
// The actual reconnect attempt will be started in a go routine.
func (c *client) reconnect() {
@@ -6180,7 +6249,7 @@ func (c *client) reconnect() {
srv.Debugf("Gateway %q not in configuration, not attempting reconnect", gwName)
}
} else if leafCfg != nil {
// Check if this is a solicited leaf node. Start up a reconnect.
// This is a solicited leaf node. Start up a reconnect.
srv.startGoRoutine(func() { srv.reConnectToRemoteLeafNode(leafCfg) })
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ func init() {
const (
// VERSION is the current version for the server.
VERSION = "2.12.6"
VERSION = "2.14.0"
// PROTO is the currently supported protocol.
// 0 was the original
File diff suppressed because it is too large Load Diff
+327
View File
@@ -0,0 +1,327 @@
// Copyright 2025 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Based on code from https://github.com/robfig/cron
// Copyright (C) 2012 Rob Figueiredo
// All Rights Reserved.
//
// MIT LICENSE
//
// Permission is hereby granted, free of charge, to any person obtaining a copy of
// this software and associated documentation files (the "Software"), to deal in
// the Software without restriction, including without limitation the rights to
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
// the Software, and to permit persons to whom the Software is furnished to do so,
// subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
package server
import (
"errors"
"fmt"
"math"
"strconv"
"strings"
"time"
)
// parseCron parses the given cron pattern and returns the next time it will fire based on the provided ts.
func parseCron(pattern string, loc *time.Location, ts int64) (time.Time, error) {
fields := strings.Fields(pattern)
if len(fields) != 6 {
return time.Time{}, fmt.Errorf("pattern requires 6 fields, got %d", len(fields))
}
// If no time zone is passed, default to UTC.
if loc == nil {
loc = time.UTC
}
// Parse each field.
var err error
var second, minute, hour, dayOfMonth, month, dayOfWeek uint64
if second, err = getField(fields[0], seconds); err != nil {
return time.Time{}, err
}
if minute, err = getField(fields[1], minutes); err != nil {
return time.Time{}, err
}
if hour, err = getField(fields[2], hours); err != nil {
return time.Time{}, err
}
if dayOfMonth, err = getField(fields[3], dom); err != nil {
return time.Time{}, err
}
if month, err = getField(fields[4], months); err != nil {
return time.Time{}, err
}
if dayOfWeek, err = getField(fields[5], dow); err != nil {
return time.Time{}, err
}
// General approach
//
// For Month, Day, Hour, Minute, Second:
// Check if the time value matches. If yes, continue to the next field.
// If the field doesn't match the schedule, then increment the field until it matches.
// While incrementing the field, a wrap-around brings it back to the beginning
// of the field list (since it is necessary to re-verify previous field values)
next := time.Unix(0, ts).In(loc)
// Start at the earliest possible time (the upcoming second).
next = next.Truncate(time.Second).Add(time.Second)
// This flag indicates whether a field has been truncated at one point.
truncated := false
// If no time is found within five years, return error.
yearLimit := next.Year() + 5
WRAP:
if next.Year() > yearLimit {
return time.Time{}, errors.New("pattern exceeds maximum range")
}
for 1<<uint(next.Month())&month == 0 {
if !truncated {
truncated = true
next = time.Date(next.Year(), next.Month(), 1, 0, 0, 0, 0, loc)
}
if next = next.AddDate(0, 1, 0); next.Month() == time.January {
goto WRAP
}
}
for !dayMatches(dayOfMonth, dayOfWeek, next) {
if !truncated {
truncated = true
next = time.Date(next.Year(), next.Month(), next.Day(), 0, 0, 0, 0, loc)
}
if next = next.AddDate(0, 0, 1); next.Day() == 1 {
goto WRAP
}
}
for 1<<uint(next.Hour())&hour == 0 {
if !truncated {
truncated = true
next = time.Date(next.Year(), next.Month(), next.Day(), next.Hour(), 0, 0, 0, loc)
}
if next = next.Add(time.Hour); next.Hour() == 0 {
goto WRAP
}
}
for 1<<uint(next.Minute())&minute == 0 {
if !truncated {
truncated = true
next = next.Truncate(time.Minute)
}
if next = next.Add(time.Minute); next.Minute() == 0 {
goto WRAP
}
}
for 1<<uint(next.Second())&second == 0 {
if !truncated {
truncated = true
next = next.Truncate(time.Second)
}
if next = next.Add(time.Second); next.Second() == 0 {
goto WRAP
}
}
return next, nil
}
// getField returns an Int with the bits set representing all of the times that
// the field represents or error parsing field value. A "field" is a comma-separated
// list of "ranges".
func getField(field string, r bounds) (uint64, error) {
var bits uint64
ranges := strings.FieldsFuncSeq(field, func(r rune) bool { return r == ',' })
for expr := range ranges {
bit, err := getRange(expr, r)
if err != nil {
return bits, err
}
bits |= bit
}
return bits, nil
}
// getRange returns the bits indicated by the given expression: number | number [ "-" number ] [ "/" number ]
// or error parsing range.
func getRange(expr string, r bounds) (uint64, error) {
var (
start, end, step uint
rangeAndStep = strings.Split(expr, "/")
lowAndHigh = strings.Split(rangeAndStep[0], "-")
singleDigit = len(lowAndHigh) == 1
err error
)
var extra uint64
if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
start = r.min
end = r.max
extra = starBit
} else {
start, err = parseIntOrName(lowAndHigh[0], r.names)
if err != nil {
return 0, err
}
switch len(lowAndHigh) {
case 1:
end = start
case 2:
end, err = parseIntOrName(lowAndHigh[1], r.names)
if err != nil {
return 0, err
}
default:
return 0, fmt.Errorf("too many hyphens: %s", expr)
}
}
switch len(rangeAndStep) {
case 1:
step = 1
case 2:
step, err = mustParseInt(rangeAndStep[1])
if err != nil {
return 0, err
}
// Special handling: "N/step" means "N-max/step".
if singleDigit {
end = r.max
}
if step > 1 {
extra = 0
}
default:
return 0, fmt.Errorf("too many slashes: %s", expr)
}
if start < r.min {
return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
}
if end > r.max {
return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
}
if start > end {
return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
}
if step == 0 {
return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
}
return getBits(start, end, step) | extra, nil
}
// parseIntOrName returns the (possibly-named) integer contained in expr.
func parseIntOrName(expr string, names map[string]uint) (uint, error) {
if names != nil {
if namedInt, ok := names[strings.ToLower(expr)]; ok {
return namedInt, nil
}
}
return mustParseInt(expr)
}
// mustParseInt parses the given expression as an int or returns an error.
func mustParseInt(expr string) (uint, error) {
num, err := strconv.Atoi(expr)
if err != nil {
return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
}
if num < 0 {
return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
}
return uint(num), nil
}
// getBits sets all bits in the range [min, max], modulo the given step size.
func getBits(min, max, step uint) uint64 {
var bits uint64
// If step is 1, use shifts.
if step == 1 {
return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
}
// Else, use a simple loop.
for i := min; i <= max; i += step {
bits |= 1 << i
}
return bits
}
// bounds provides a range of acceptable values (plus a map of name to value).
type bounds struct {
min, max uint
names map[string]uint
}
// The bounds for each field.
var (
seconds = bounds{0, 59, nil}
minutes = bounds{0, 59, nil}
hours = bounds{0, 23, nil}
dom = bounds{1, 31, nil}
months = bounds{1, 12, map[string]uint{
"jan": 1,
"feb": 2,
"mar": 3,
"apr": 4,
"may": 5,
"jun": 6,
"jul": 7,
"aug": 8,
"sep": 9,
"oct": 10,
"nov": 11,
"dec": 12,
}}
dow = bounds{0, 6, map[string]uint{
"sun": 0,
"mon": 1,
"tue": 2,
"wed": 3,
"thu": 4,
"fri": 5,
"sat": 6,
}}
)
const (
// Set the top bit if a star was included in the expression.
starBit = 1 << 63
)
// dayMatches returns true if the schedule's day-of-week and day-of-month
// restrictions are satisfied by the given time.
func dayMatches(dayOfMonth, dayOfWeek uint64, t time.Time) bool {
var (
domMatch = 1<<uint(t.Day())&dayOfMonth > 0
dowMatch = 1<<uint(t.Weekday())&dayOfWeek > 0
)
if dayOfMonth&starBit > 0 || dayOfWeek&starBit > 0 {
return domMatch && dowMatch
}
return domMatch || dowMatch
}
+211 -1
View File
@@ -2008,5 +2008,215 @@
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSMessageSchedulesSourceInvalidErr",
"code": 400,
"error_code": 10203,
"description": "message schedules source is invalid",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSConsumerInvalidResetErr",
"code": 400,
"error_code": 10204,
"description": "invalid reset: {err}",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSBatchPublishDisabledErr",
"code": 400,
"error_code": 10205,
"description": "batch publish is disabled",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSBatchPublishInvalidPatternErr",
"code": 400,
"error_code": 10206,
"description": "batch publish pattern is invalid",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSBatchPublishInvalidBatchIDErr",
"code": 400,
"error_code": 10207,
"description": "batch publish ID is invalid",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSBatchPublishUnknownBatchIDErr",
"code": 400,
"error_code": 10208,
"description": "batch publish ID unknown",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSMirrorWithBatchPublishErr",
"code": 400,
"error_code": 10209,
"description": "stream mirrors can not also use batch publishing",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSAtomicPublishTooManyInflight",
"code": 429,
"error_code": 10210,
"description": "atomic publish too many inflight",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSBatchPublishTooManyInflight",
"code": 429,
"error_code": 10211,
"description": "batch publish too many inflight",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSMessageSchedulesSchedulerInvalidErr",
"code": 400,
"error_code": 10212,
"description": "message schedules invalid scheduler",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSMirrorDurableConsumerCfgInvalid",
"code": 400,
"error_code": 10213,
"description": "stream mirror consumer config is invalid",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSMirrorConsumerRequiresAckFCErr",
"code": 400,
"error_code": 10214,
"description": "stream mirror consumer requires flow control ack policy",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSSourceDurableConsumerCfgInvalid",
"code": 400,
"error_code": 10215,
"description": "stream source consumer config is invalid",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSSourceDurableConsumerDuplicateDetected",
"code": 400,
"error_code": 10216,
"description": "duplicate stream source consumer detected",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSSourceConsumerRequiresAckFCErr",
"code": 400,
"error_code": 10217,
"description": "stream source consumer requires flow control ack policy",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSConsumerAckFCRequiresPushErr",
"code": 400,
"error_code": 10218,
"description": "flow control ack policy requires a push based consumer",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSConsumerAckFCRequiresFCErr",
"code": 400,
"error_code": 10219,
"description": "flow control ack policy requires flow control",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSConsumerAckFCRequiresMaxAckPendingErr",
"code": 400,
"error_code": 10220,
"description": "flow control ack policy requires max ack pending",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSConsumerAckFCRequiresNoAckWaitErr",
"code": 400,
"error_code": 10221,
"description": "flow control ack policy requires unset ack wait",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSConsumerAckFCRequiresNoMaxDeliverErr",
"code": 400,
"error_code": 10222,
"description": "flow control ack policy requires unset max deliver",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSMessageSchedulesTimeZoneInvalidErr",
"code": 400,
"error_code": 10223,
"description": "message schedules time zone is invalid",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
}
]
]
+21 -11
View File
@@ -247,14 +247,15 @@ type ServerCapability uint64
// ServerInfo identifies remote servers.
type ServerInfo struct {
Name string `json:"name"`
Host string `json:"host"`
ID string `json:"id"`
Cluster string `json:"cluster,omitempty"`
Domain string `json:"domain,omitempty"`
Version string `json:"ver"`
Tags []string `json:"tags,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
Name string `json:"name"`
Host string `json:"host"`
ID string `json:"id"`
Cluster string `json:"cluster,omitempty"`
Domain string `json:"domain,omitempty"`
Version string `json:"ver"`
Tags []string `json:"tags,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
FeatureFlags map[string]bool `json:"feature_flags,omitempty"`
// Whether JetStream is enabled (deprecated in favor of the `ServerCapability`).
JetStream bool `json:"jetstream"`
// Generic capability flags
@@ -328,6 +329,7 @@ type ClientInfo struct {
ClientType string `json:"client_type,omitempty"`
MQTTClient string `json:"client_id,omitempty"` // This is the MQTT client ID
Nonce string `json:"nonce,omitempty"`
Reply string `json:"reply,omitempty"` // Original reply subject after a service import (only when needed).
}
// forAssignmentSnap returns the minimum amount of ClientInfo we need for assignment snapshots.
@@ -518,7 +520,7 @@ RESET:
// Grab tags and metadata.
opts := s.getOpts()
tags, metadata := opts.Tags, opts.Metadata
tags, metadata, featureFlags := opts.Tags, opts.Metadata, opts.getMergedFeatureFlags()
for s.eventsRunning() {
select {
@@ -536,6 +538,7 @@ RESET:
si.Time = time.Now().UTC()
si.Tags = tags
si.Metadata = metadata
si.FeatureFlags = featureFlags
si.Flags = 0
if js {
// New capability based flags.
@@ -1052,8 +1055,15 @@ func (s *Server) sendStatsz(subj string) {
Size: mg.ClusterSize(),
}
}
if ipq := s.jsAPIRoutedReqs; ipq != nil && jStat.Meta != nil {
jStat.Meta.Pending = ipq.len()
if jStat.Meta != nil {
if ipq := s.jsAPIRoutedReqs; ipq != nil {
jStat.Meta.PendingRequests = ipq.len()
}
if ipq := s.jsAPIRoutedInfoReqs; ipq != nil {
jStat.Meta.PendingInfos = ipq.len()
}
jStat.Meta.Pending = jStat.Meta.PendingRequests + jStat.Meta.PendingInfos
jStat.Meta.Snapshot = s.metaClusterSnapshotStats(js, mg)
}
}
jStat.Limits = &s.getOpts().JetStreamLimits
+130
View File
@@ -0,0 +1,130 @@
// Copyright 2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package server
import (
"maps"
"slices"
"strings"
)
const (
FeatureFlagJsAckFormatV2 = "js_ack_fc_v2"
FeatureFlagJsRaftDeleteRange = "js_raft_delete_range"
)
var featureFlags = map[string]bool{
// Use v2 format for `$JS.ACK.>` and `$JS.FC.>`.
// - Introduced: 2.14.0, both v1 and v2 supported, only using v1.
// - Enabled: TBD, both supported, v2 becomes the default.
//
// - v1: $JS.ACK.<stream name>.<consumer name>.<num delivered>.<stream sequence>.<consumer sequence>.<timestamp>.<num pending>
// - v2: $JS.ACK.<domain>.<account hash>.<stream name>.<consumer name>.<num delivered>.<stream sequence>.<consumer sequence>.<timestamp>.<num pending>
// See also: https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-15.md#jsack
FeatureFlagJsAckFormatV2: false,
// Propose delete range gaps as a single `deleteRangeOp` Raft append entry
// instead of one entry per deleted sequence. Dramatically reduces Raft cost
// on mirrors whose origin has a large number of interior deletes.
// - Introduced: 2.14.0, apply-side always supports receiving `deleteRangeOp`.
// - Enabled: TBD, once all supported versions carry the apply-side.
//
// WARNING: Only enable once every peer in the cluster is on a version that
// supports receiving `deleteRangeOp`. Older peers panic on apply of an
// unknown stream entry operation.
FeatureFlagJsRaftDeleteRange: false,
}
// getFeatureFlag is used to retrieve either the default or overwritten value for a feature flag.
// The user's value takes precedence over the system's default. However, if the flag doesn't exist, it's disabled.
// The *Options returned by Server.getOpts() is treated as immutable, mutations go through setOpts,
// so no lock is required on the map read here.
func (o *Options) getFeatureFlag(k string) bool {
defaultValue, ok := featureFlags[k]
if !ok {
return false // Not supported.
}
if userValue, ok := o.FeatureFlags[k]; ok {
return userValue
}
return defaultValue
}
// getMergedFeatureFlags returns a merged map of feature flags, with the user's values taking precedence.
func (o *Options) getMergedFeatureFlags() map[string]bool {
merged := make(map[string]bool)
for k, v := range featureFlags {
merged[k] = v
}
for k, v := range o.FeatureFlags {
if _, ok := featureFlags[k]; !ok {
continue
}
merged[k] = v
}
return merged
}
// printFeatureFlags logs the currently used feature flags on server startup.
func (s *Server) printFeatureFlags(o *Options) {
if len(o.FeatureFlags) == 0 {
return
}
keys := slices.Sorted(maps.Keys(o.FeatureFlags))
var (
configured strings.Builder
unsupported strings.Builder
)
for _, k := range keys {
// Unsupported
defaultValue, ok := featureFlags[k]
if !ok {
if unsupported.Len() > 0 {
unsupported.WriteString(", ")
}
unsupported.WriteString(k)
continue
}
v := o.FeatureFlags[k]
if configured.Len() > 0 {
configured.WriteString(", ")
}
configured.WriteString(k)
configured.WriteString(" (")
if defaultValue {
if v {
configured.WriteString("enabled")
} else {
configured.WriteString("opt-out")
}
} else if v {
configured.WriteString("opt-in")
} else {
configured.WriteString("disabled")
}
configured.WriteString(")")
}
if configured.Len() == 0 {
configured.WriteString("none")
}
s.Noticef(" Feature flags:")
s.Noticef(" Configured: %s", configured.String())
if unsupported.Len() > 0 {
s.Noticef(" Unsupported: %s", unsupported.String())
}
}
File diff suppressed because it is too large Load Diff
+14 -6
View File
@@ -1156,9 +1156,7 @@ func (c *client) processGatewayInfo(info *Info) {
// defensive code above that if we did not register this connection
// because we already have an outbound for this name, then
// close this connection (and make sure it does not try to reconnect)
c.mu.Lock()
c.flags.set(noReconnect)
c.mu.Unlock()
c.setNoReconnect()
c.closeConnection(WrongGateway)
return
}
@@ -1981,7 +1979,7 @@ func (c *client) processGatewayRUnsub(arg []byte) error {
return nil
} else {
// Plain sub, assume optimistic sends, create entry.
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()}
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistForServer(c.srv)}
newe = true
}
// This is when a sub or queue sub is supposed to be in
@@ -2090,7 +2088,7 @@ func (c *client) processGatewayRSub(arg []byte) error {
} else if queue == nil {
return nil
} else {
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()}
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistForServer(c.srv)}
newe = true
useSl = true
}
@@ -2952,6 +2950,16 @@ func getSubjectFromGWRoutedReply(reply []byte, isOldPrefix bool) []byte {
return reply[gwSubjectOffset:]
}
// Returns the subject embedded in the given routed
// reply subject and whether the prefix was stripped.
// If the subject is not routed, returns it unchanged.
func getGWRoutedSubjectOrSelf(subject []byte) ([]byte, bool) {
if isGWPrefix, oldPrefix := isGWRoutedSubjectAndIsOldPrefix(subject); isGWPrefix {
return getSubjectFromGWRoutedReply(subject, oldPrefix), true
}
return subject, false
}
// This should be invoked only from processInboundGatewayMsg() or
// processInboundRoutedMsg() and is checking if the subject
// (c.pa.subject) has the _GR_ prefix. If so, this is processed
@@ -3201,7 +3209,7 @@ func (c *client) gatewayAllSubsReceiveStart(info *Info) {
e.mode = Transitioning
e.Unlock()
} else {
e := &outsie{sl: NewSublistWithCache()}
e := &outsie{sl: NewSublistForServer(c.srv)}
e.mode = Transitioning
c.mu.Lock()
c.gw.outsim.Store(account, e)
+60
View File
@@ -170,6 +170,66 @@ func (s *GenericSublist[T]) NumInterest(subject string) (np int) {
return
}
// MatchesFullWildcard returns true if there is top-level ">" interest.
func (s *GenericSublist[T]) MatchesFullWildcard() bool {
if s == nil {
return false
}
s.RLock()
defer s.RUnlock()
return s.root.fwc != nil
}
// MatchesSingleFilter returns the filter when the sublist contains exactly one unique subject.
func (s *GenericSublist[T]) MatchesSingleFilter() (string, bool) {
if s == nil {
return _EMPTY_, false
}
s.RLock()
defer s.RUnlock()
return singleFilter(s.root, _EMPTY_)
}
func singleFilter[T comparable](l *level[T], filter string) (string, bool) {
if l == nil {
return filter, filter != _EMPTY_
}
if len(l.nodes) > 1 {
return _EMPTY_, false
}
var next *node[T]
branches := 0
if l.pwc != nil {
next = l.pwc
branches++
}
if l.fwc != nil {
next = l.fwc
branches++
}
for _, n := range l.nodes {
next = n
branches++
}
if branches != 1 {
return _EMPTY_, false
}
for _, subj := range next.subs {
filter = subj
break
}
if next.next == nil {
return filter, filter != _EMPTY_
}
if filter != _EMPTY_ {
if next.next.numNodes() > 0 {
return _EMPTY_, false
}
return filter, true
}
return singleFilter(next.next, filter)
}
func (s *GenericSublist[T]) match(subject string, cb func(T), doLock bool) {
tsa := [32]string{}
tokens := tsa[:0]
+63 -423
View File
@@ -25,13 +25,13 @@ import (
"os"
"path/filepath"
"runtime/debug"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/minio/highwayhash"
"github.com/nats-io/nats-server/v2/server/gsl"
"github.com/nats-io/nats-server/v2/server/sysmem"
"github.com/nats-io/nats-server/v2/server/tpm"
"github.com/nats-io/nkeys"
@@ -103,22 +103,26 @@ type JetStreamAPIStats struct {
// This is for internal accounting for JetStream for this server.
type jetStream struct {
// These are here first because of atomics on 32bit systems.
apiInflight int64
apiTotal int64
apiErrors int64
memReserved int64
storeReserved int64
memUsed int64
storeUsed int64
queueLimit int64
clustered int32
mu sync.RWMutex
srv *Server
config JetStreamConfig
cluster *jetStreamCluster
accounts map[string]*jsAccount
apiSubs *Sublist
started time.Time
apiInflight int64
apiTotal int64
apiErrors int64
memMax int64
memReserved int64 // Requires JS lock to be held.
memUsed int64
storeMax int64
storeReserved int64 // Requires JS lock to be held.
storeUsed int64
queueLimit int64
infoQueueLimit int64
clustered int32
mu sync.RWMutex
srv *Server
config JetStreamConfig
cluster *jetStreamCluster
accounts map[string]*jsAccount
apiSubs *Sublist
infoSubs *gsl.SimpleSublist // Subjects for info-specific queue.
started time.Time
// System level request to purge a stream move
accountPurge *subscription
@@ -150,14 +154,12 @@ type jsaStorage struct {
// an internal sub for a stream, so we will direct link to the stream
// and walk backwards as needed vs multiple hash lookups and locks, etc.
type jsAccount struct {
mu sync.RWMutex
js *jetStream
account *Account
storeDir string
inflight sync.Map
streams map[string]*stream
templates map[string]*streamTemplate // Deprecated: stream templates are deprecated and will be removed in a future version.
store TemplateStore // Deprecated: stream templates are deprecated and will be removed in a future version.
mu sync.RWMutex
js *jetStream
account *Account
storeDir string
inflight sync.Map
streams map[string]*stream
// From server
sendq *ipQueue[*pubMsg]
@@ -415,15 +417,19 @@ func (s *Server) initJetStreamEncryption() (err error) {
// enableJetStream will start up the JetStream subsystem.
func (s *Server) enableJetStream(cfg JetStreamConfig) error {
js := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache()}
js := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache(), infoSubs: gsl.NewSimpleSublist()}
s.gcbMu.Lock()
if s.gcbOutMax = s.getOpts().JetStreamMaxCatchup; s.gcbOutMax == 0 {
s.gcbOutMax = defaultMaxTotalCatchupOutBytes
}
s.gcbMu.Unlock()
atomic.StoreInt64(&js.memMax, cfg.MaxMemory)
atomic.StoreInt64(&js.storeMax, cfg.MaxStore)
// TODO: Not currently reloadable.
atomic.StoreInt64(&js.queueLimit, s.getOpts().JetStreamRequestQueueLimit)
atomic.StoreInt64(&js.infoQueueLimit, s.getOpts().JetStreamInfoQueueLimit)
s.js.Store(js)
@@ -1058,8 +1064,10 @@ func (s *Server) shutdownJetStream() {
func (s *Server) JetStreamConfig() *JetStreamConfig {
var c *JetStreamConfig
if js := s.getJetStream(); js != nil {
js.mu.RLock()
copy := js.config
c = &(copy)
js.mu.RUnlock()
}
return c
}
@@ -1219,54 +1227,6 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
s.Debugf("Recovering JetStream state for account %q", a.Name)
}
// Check templates first since messsage sets will need proper ownership.
// FIXME(dlc) - Make this consistent.
tdir := filepath.Join(jsa.storeDir, tmplsDir)
if stat, err := os.Stat(tdir); err == nil && stat.IsDir() {
key := sha256.Sum256([]byte("templates"))
hh, err := highwayhash.NewDigest64(key[:])
if err != nil {
return err
}
fis, _ := os.ReadDir(tdir)
for _, fi := range fis {
metafile := filepath.Join(tdir, fi.Name(), JetStreamMetaFile)
metasum := filepath.Join(tdir, fi.Name(), JetStreamMetaFileSum)
buf, err := os.ReadFile(metafile)
if err != nil {
s.Warnf(" Error reading StreamTemplate metafile %q: %v", metasum, err)
continue
}
if _, err := os.Stat(metasum); os.IsNotExist(err) {
s.Warnf(" Missing StreamTemplate checksum for %q", metasum)
continue
}
sum, err := os.ReadFile(metasum)
if err != nil {
s.Warnf(" Error reading StreamTemplate checksum %q: %v", metasum, err)
continue
}
hh.Reset()
hh.Write(buf)
var hb [highwayhash.Size64]byte
checksum := hex.EncodeToString(hh.Sum(hb[:0]))
if checksum != string(sum) {
s.Warnf(" StreamTemplate checksums do not match %q vs %q", sum, checksum)
continue
}
var cfg StreamTemplateConfig
if err := json.Unmarshal(buf, &cfg); err != nil {
s.Warnf(" Error unmarshalling StreamTemplate metafile: %v", err)
continue
}
cfg.Config.Name = _EMPTY_
if _, err := a.addStreamTemplate(&cfg); err != nil {
s.Warnf(" Error recreating StreamTemplate %q: %v", cfg.Name, err)
continue
}
}
}
// Remember if we should be encrypted and what cipher we think we should use.
encrypted := s.getOpts().JetStreamKey != _EMPTY_
sc := s.getOpts().JetStreamCipher
@@ -1510,15 +1470,6 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
return nil
}
if cfg.Template != _EMPTY_ {
jsa.mu.Lock()
err := jsa.addStreamNameToTemplate(cfg.Template, cfg.Name)
jsa.mu.Unlock()
if err != nil {
s.Warnf(" Error adding stream %q to template %q: %v", cfg.Name, cfg.Template, err)
}
}
// We had a bug that set a default de dupe window on mirror, despite that being not a valid config
fixCfgMirrorWithDedupWindow(&cfg.StreamConfig)
@@ -1587,6 +1538,7 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
batchId string
batchSeq uint64
commit bool
commitEob bool
batchStoreDir string
store StreamStore
state StreamState
@@ -1604,19 +1556,30 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
}
// We've observed a partial batch write. Write the remainder of the batch.
batchSeq++
_, batchStoreDir = getBatchStoreDir(mset, batchId)
_, batchStoreDir = getBatchStoreDir(jsa.storeDir, cfg.Name, batchId)
if _, err = os.Stat(batchStoreDir); err != nil {
s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v",
mset.accName(), mset.name(), batchSeq, err)
goto SKIP
}
store, err = newBatchStore(mset, batchId)
store, err = newBatchStore(mset, batchId, cfg.Replicas, cfg.Storage, jsa.storeDir, cfg.Name)
if err != nil {
s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v",
mset.accName(), mset.name(), batchSeq, err)
goto SKIP
}
store.FastState(&state)
sm, err = store.LoadMsg(state.LastSeq, &smv)
if err != nil || sm == nil {
s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: last msg not found %d",
mset.accName(), mset.name(), batchSeq, state.LastSeq)
goto SKIP
}
commitEob = bytes.Equal(sliceHeader(JSBatchCommit, sm.hdr), []byte("eob"))
// If the commit ends with an "End Of Batch" message, we don't store this.
if commitEob {
state.LastSeq--
}
s.Noticef(" Restoring partial batch write for stream '%s > %s' (seq %d to %d)",
mset.accName(), mset.name(), batchSeq, state.LastSeq)
// Loop through items that weren't persisted yet.
@@ -1627,7 +1590,12 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
mset.accName(), mset.name(), seq, err)
break
}
mset.processJetStreamMsg(sm.subj, _EMPTY_, sm.hdr, sm.msg, 0, 0, nil, false, true)
hdr := sm.hdr
// If committed by EOB, the last message must get the normal commit header.
if commitEob && seq == state.LastSeq {
hdr = genHeader(hdr, JSBatchCommit, "1")
}
mset.processJetStreamMsg(sm.subj, _EMPTY_, hdr, sm.msg, 0, 0, nil, false, true)
}
store.Delete(true)
SKIP:
@@ -2342,14 +2310,14 @@ func (jsa *jsAccount) sendClusterUsageUpdate() {
func (js *jetStream) wouldExceedLimits(storeType StorageType, sz int) bool {
var (
total *int64
max int64
max *int64
)
if storeType == MemoryStorage {
total, max = &js.memUsed, js.config.MaxMemory
total, max = &js.memUsed, &js.memMax
} else {
total, max = &js.storeUsed, js.config.MaxStore
total, max = &js.storeUsed, &js.storeMax
}
return (atomic.LoadInt64(total) + int64(sz)) > max
return (atomic.LoadInt64(total) + int64(sz)) > atomic.LoadInt64(max)
}
func (js *jetStream) limitsExceeded(storeType StorageType) bool {
@@ -2519,7 +2487,6 @@ func (jsa *jsAccount) acc() *Account {
// Delete the JetStream resources.
func (jsa *jsAccount) delete() {
var streams []*stream
var ts []string
jsa.mu.Lock()
// The update timer and subs need to be protected by usageMu lock
@@ -2538,20 +2505,11 @@ func (jsa *jsAccount) delete() {
for _, ms := range jsa.streams {
streams = append(streams, ms)
}
acc := jsa.account
for _, t := range jsa.templates {
ts = append(ts, t.Name)
}
jsa.templates = nil
jsa.mu.Unlock()
for _, mset := range streams {
mset.stop(false, false)
}
for _, t := range ts {
acc.deleteStreamTemplate(t)
}
}
// Lookup the jetstream account for a given account.
@@ -2763,325 +2721,6 @@ func (a *Account) checkForJetStream() (*Server, *jsAccount, error) {
return s, jsa, nil
}
// StreamTemplateConfig allows a configuration to auto-create streams based on this template when a message
// is received that matches. Each new stream will use the config as the template config to create them.
// Deprecated: stream templates are deprecated and will be removed in a future version.
type StreamTemplateConfig struct {
Name string `json:"name"`
Config *StreamConfig `json:"config"`
MaxStreams uint32 `json:"max_streams"`
}
// StreamTemplateInfo
// Deprecated: stream templates are deprecated and will be removed in a future version.
type StreamTemplateInfo struct {
Config *StreamTemplateConfig `json:"config"`
Streams []string `json:"streams"`
}
// streamTemplate
// Deprecated: stream templates are deprecated and will be removed in a future version.
type streamTemplate struct {
mu sync.Mutex
tc *client
jsa *jsAccount
*StreamTemplateConfig
streams []string
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (t *StreamTemplateConfig) deepCopy() *StreamTemplateConfig {
copy := *t
cfg := *t.Config
copy.Config = &cfg
return &copy
}
// addStreamTemplate will add a stream template to this account that allows auto-creation of streams.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (a *Account) addStreamTemplate(tc *StreamTemplateConfig) (*streamTemplate, error) {
s, jsa, err := a.checkForJetStream()
if err != nil {
return nil, err
}
if tc.Config.Name != "" {
return nil, fmt.Errorf("template config name should be empty")
}
if len(tc.Name) > JSMaxNameLen {
return nil, fmt.Errorf("template name is too long, maximum allowed is %d", JSMaxNameLen)
}
// FIXME(dlc) - Hacky
tcopy := tc.deepCopy()
tcopy.Config.Name = "_"
cfg, apiErr := s.checkStreamCfg(tcopy.Config, a, false)
if apiErr != nil {
return nil, apiErr
}
tcopy.Config = &cfg
t := &streamTemplate{
StreamTemplateConfig: tcopy,
tc: s.createInternalJetStreamClient(),
jsa: jsa,
}
t.tc.registerWithAccount(a)
jsa.mu.Lock()
if jsa.templates == nil {
jsa.templates = make(map[string]*streamTemplate)
// Create the appropriate store
if cfg.Storage == FileStorage {
jsa.store = newTemplateFileStore(jsa.storeDir)
} else {
jsa.store = newTemplateMemStore()
}
} else if _, ok := jsa.templates[tcopy.Name]; ok {
jsa.mu.Unlock()
return nil, fmt.Errorf("template with name %q already exists", tcopy.Name)
}
jsa.templates[tcopy.Name] = t
jsa.mu.Unlock()
// FIXME(dlc) - we can not overlap subjects between templates. Need to have test.
// Setup the internal subscriptions to trap the messages.
if err := t.createTemplateSubscriptions(); err != nil {
return nil, err
}
if err := jsa.store.Store(t); err != nil {
t.delete()
return nil, err
}
return t, nil
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (t *streamTemplate) createTemplateSubscriptions() error {
if t == nil {
return fmt.Errorf("no template")
}
if t.tc == nil {
return fmt.Errorf("template not enabled")
}
c := t.tc
if !c.srv.EventsEnabled() {
return ErrNoSysAccount
}
sid := 1
for _, subject := range t.Config.Subjects {
// Now create the subscription
if _, err := c.processSub([]byte(subject), nil, []byte(strconv.Itoa(sid)), t.processInboundTemplateMsg, false); err != nil {
c.acc.deleteStreamTemplate(t.Name)
return err
}
sid++
}
return nil
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (t *streamTemplate) processInboundTemplateMsg(_ *subscription, pc *client, acc *Account, subject, reply string, msg []byte) {
if t == nil || t.jsa == nil {
return
}
jsa := t.jsa
cn := canonicalName(subject)
jsa.mu.Lock()
// If we already are registered then we can just return here.
if _, ok := jsa.streams[cn]; ok {
jsa.mu.Unlock()
return
}
jsa.mu.Unlock()
// Check if we are at the maximum and grab some variables.
t.mu.Lock()
c := t.tc
cfg := *t.Config
cfg.Template = t.Name
atLimit := len(t.streams) >= int(t.MaxStreams)
if !atLimit {
t.streams = append(t.streams, cn)
}
t.mu.Unlock()
if atLimit {
c.RateLimitWarnf("JetStream could not create stream for account %q on subject %q, at limit", acc.Name, subject)
return
}
// We need to create the stream here.
// Change the config from the template and only use literal subject.
cfg.Name = cn
cfg.Subjects = []string{subject}
mset, err := acc.addStream(&cfg)
if err != nil {
acc.validateStreams(t)
c.RateLimitWarnf("JetStream could not create stream for account %q on subject %q: %v", acc.Name, subject, err)
return
}
// Process this message directly by invoking mset.
mset.processInboundJetStreamMsg(nil, pc, acc, subject, reply, msg)
}
// lookupStreamTemplate looks up the names stream template.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (a *Account) lookupStreamTemplate(name string) (*streamTemplate, error) {
_, jsa, err := a.checkForJetStream()
if err != nil {
return nil, err
}
jsa.mu.Lock()
defer jsa.mu.Unlock()
if jsa.templates == nil {
return nil, fmt.Errorf("template not found")
}
t, ok := jsa.templates[name]
if !ok {
return nil, fmt.Errorf("template not found")
}
return t, nil
}
// This function will check all named streams and make sure they are valid.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (a *Account) validateStreams(t *streamTemplate) {
t.mu.Lock()
var vstreams []string
for _, sname := range t.streams {
if _, err := a.lookupStream(sname); err == nil {
vstreams = append(vstreams, sname)
}
}
t.streams = vstreams
t.mu.Unlock()
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (t *streamTemplate) delete() error {
if t == nil {
return fmt.Errorf("nil stream template")
}
t.mu.Lock()
jsa := t.jsa
c := t.tc
t.tc = nil
defer func() {
if c != nil {
c.closeConnection(ClientClosed)
}
}()
t.mu.Unlock()
if jsa == nil {
return NewJSNotEnabledForAccountError()
}
jsa.mu.Lock()
if jsa.templates == nil {
jsa.mu.Unlock()
return fmt.Errorf("template not found")
}
if _, ok := jsa.templates[t.Name]; !ok {
jsa.mu.Unlock()
return fmt.Errorf("template not found")
}
delete(jsa.templates, t.Name)
acc := jsa.account
jsa.mu.Unlock()
// Remove streams associated with this template.
var streams []*stream
t.mu.Lock()
for _, name := range t.streams {
if mset, err := acc.lookupStream(name); err == nil {
streams = append(streams, mset)
}
}
t.mu.Unlock()
if jsa.store != nil {
if err := jsa.store.Delete(t); err != nil {
return fmt.Errorf("error deleting template from store: %v", err)
}
}
var lastErr error
for _, mset := range streams {
if err := mset.delete(); err != nil {
lastErr = err
}
}
return lastErr
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (a *Account) deleteStreamTemplate(name string) error {
t, err := a.lookupStreamTemplate(name)
if err != nil {
return NewJSStreamTemplateNotFoundError()
}
return t.delete()
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (a *Account) templates() []*streamTemplate {
var ts []*streamTemplate
_, jsa, err := a.checkForJetStream()
if err != nil {
return nil
}
jsa.mu.Lock()
for _, t := range jsa.templates {
// FIXME(dlc) - Copy?
ts = append(ts, t)
}
jsa.mu.Unlock()
return ts
}
// Will add a stream to a template, this is for recovery.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (jsa *jsAccount) addStreamNameToTemplate(tname, mname string) error {
if jsa.templates == nil {
return fmt.Errorf("template not found")
}
t, ok := jsa.templates[tname]
if !ok {
return fmt.Errorf("template not found")
}
// We found template.
t.mu.Lock()
t.streams = append(t.streams, mname)
t.mu.Unlock()
return nil
}
// This will check if a template owns this stream.
// jsAccount lock should be held
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (jsa *jsAccount) checkTemplateOwnership(tname, sname string) bool {
if jsa.templates == nil {
return false
}
t, ok := jsa.templates[tname]
if !ok {
return false
}
// We found template, make sure we are in streams.
for _, streamName := range t.streams {
if sname == streamName {
return true
}
}
return false
}
type Number interface {
int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64
}
@@ -3107,10 +2746,11 @@ func isValidName(name string) bool {
return !strings.ContainsAny(name, " \t\r\n\f.*>")
}
// CanonicalName will replace all token separators '.' with '_'.
// This can be used when naming streams or consumers with multi-token subjects.
func canonicalName(name string) string {
return strings.ReplaceAll(name, ".", "_")
func isValidAssetName(name string) bool {
if name == _EMPTY_ {
return false
}
return !strings.ContainsAny(name, " \t\r\n\f.*>\\/")
}
// To throttle the out of resources errors.
+175 -331
View File
@@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"io"
"maps"
"os"
"path/filepath"
"runtime"
@@ -46,29 +47,6 @@ const (
// Will return JSON response.
JSApiAccountInfo = "$JS.API.INFO"
// JSApiTemplateCreate is the endpoint to create new stream templates.
// Will return JSON response.
// Deprecated: stream templates are deprecated and will be removed in a future version.
JSApiTemplateCreate = "$JS.API.STREAM.TEMPLATE.CREATE.*"
JSApiTemplateCreateT = "$JS.API.STREAM.TEMPLATE.CREATE.%s"
// JSApiTemplates is the endpoint to list all stream template names for this account.
// Will return JSON response.
// Deprecated: stream templates are deprecated and will be removed in a future version.
JSApiTemplates = "$JS.API.STREAM.TEMPLATE.NAMES"
// JSApiTemplateInfo is for obtaining general information about a named stream template.
// Will return JSON response.
// Deprecated: stream templates are deprecated and will be removed in a future version.
JSApiTemplateInfo = "$JS.API.STREAM.TEMPLATE.INFO.*"
JSApiTemplateInfoT = "$JS.API.STREAM.TEMPLATE.INFO.%s"
// JSApiTemplateDelete is the endpoint to delete stream templates.
// Will return JSON response.
// Deprecated: stream templates are deprecated and will be removed in a future version.
JSApiTemplateDelete = "$JS.API.STREAM.TEMPLATE.DELETE.*"
JSApiTemplateDeleteT = "$JS.API.STREAM.TEMPLATE.DELETE.%s"
// JSApiStreamCreate is the endpoint to create new streams.
// Will return JSON response.
JSApiStreamCreate = "$JS.API.STREAM.CREATE.*"
@@ -177,6 +155,9 @@ const (
// JSApiRequestNextT is the prefix for the request next message(s) for a consumer in worker/pull mode.
JSApiRequestNextT = "$JS.API.CONSUMER.MSG.NEXT.%s.%s"
// JSApiConsumerResetT is the prefix for resetting a given consumer to a new starting sequence.
JSApiConsumerResetT = "$JS.API.CONSUMER.RESET.%s.%s"
// JSApiConsumerUnpinT is the prefix for unpinning subscription for a given consumer.
JSApiConsumerUnpin = "$JS.API.CONSUMER.UNPIN.*.*"
JSApiConsumerUnpinT = "$JS.API.CONSUMER.UNPIN.%s.%s"
@@ -237,13 +218,15 @@ const (
// jsAckT is the template for the ack message stream coming back from a consumer
// when they ACK/NAK, etc a message.
jsAckT = "$JS.ACK.%s.%s"
jsAckTv2 = "$JS.ACK.%s.%s.%s.%s"
jsAckPre = "$JS.ACK."
jsAckPreLen = len(jsAckPre)
// jsFlowControl is for flow control subjects.
jsFlowControlPre = "$JS.FC."
// jsFlowControl is for FC responses.
jsFlowControl = "$JS.FC.%s.%s.*"
jsFlowControl = "$JS.FC.%s.%s.*"
jsFlowControlV2 = "$JS.FC.%s.%s.%s.%s.*"
// JSAdvisoryPrefix is a prefix for all JetStream advisories.
JSAdvisoryPrefix = "$JS.EVENT.ADVISORY"
@@ -787,50 +770,19 @@ type JSApiConsumerGetNextRequest struct {
PriorityGroup
}
// JSApiStreamTemplateCreateResponse for creating templates.
// Deprecated: stream templates are deprecated and will be removed in a future version.
type JSApiStreamTemplateCreateResponse struct {
// JSApiConsumerResetRequest is for resetting a consumer to a specific sequence.
type JSApiConsumerResetRequest struct {
Seq uint64 `json:"seq,omitempty"`
}
// JSApiConsumerResetResponse is a superset of JSApiConsumerCreateResponse, but including an explicit ResetSeq.
type JSApiConsumerResetResponse struct {
ApiResponse
*StreamTemplateInfo
*ConsumerInfo
ResetSeq uint64 `json:"reset_seq"`
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
const JSApiStreamTemplateCreateResponseType = "io.nats.jetstream.api.v1.stream_template_create_response"
// Deprecated: stream templates are deprecated and will be removed in a future version.
type JSApiStreamTemplateDeleteResponse struct {
ApiResponse
Success bool `json:"success,omitempty"`
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
const JSApiStreamTemplateDeleteResponseType = "io.nats.jetstream.api.v1.stream_template_delete_response"
// JSApiStreamTemplateInfoResponse for information about stream templates.
// Deprecated: stream templates are deprecated and will be removed in a future version.
type JSApiStreamTemplateInfoResponse struct {
ApiResponse
*StreamTemplateInfo
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
const JSApiStreamTemplateInfoResponseType = "io.nats.jetstream.api.v1.stream_template_info_response"
// Deprecated: stream templates are deprecated and will be removed in a future version.
type JSApiStreamTemplatesRequest struct {
ApiPagedRequest
}
// JSApiStreamTemplateNamesResponse list of templates
// Deprecated: stream templates are deprecated and will be removed in a future version.
type JSApiStreamTemplateNamesResponse struct {
ApiResponse
ApiPaged
Templates []string `json:"streams"`
}
// Deprecated: stream templates are deprecated and will be removed in a future version.
const JSApiStreamTemplateNamesResponseType = "io.nats.jetstream.api.v1.stream_template_names_response"
const JSApiConsumerResetResponseType = "io.nats.jetstream.api.v1.consumer_reset_response"
// Structure that holds state for a JetStream API request that is processed
// in a separate long-lived go routine. This is to avoid blocking connections.
@@ -911,11 +863,19 @@ func (js *jetStream) apiDispatch(sub *subscription, c *client, acc *Account, sub
// Copy the state. Note the JSAPI only uses the hdr index to piece apart the
// header from the msg body. No other references are needed.
// Check pending and warn if getting backed up.
pending, _ := s.jsAPIRoutedReqs.push(&jsAPIRoutedReq{jsub, sub, acc, subject, reply, copyBytes(rmsg), c.pa})
limit := atomic.LoadInt64(&js.queueLimit)
var queue *ipQueue[*jsAPIRoutedReq]
var limit int64
if js.infoSubs.HasInterest(subject) {
queue = s.jsAPIRoutedInfoReqs
limit = atomic.LoadInt64(&js.infoQueueLimit)
} else {
queue = s.jsAPIRoutedReqs
limit = atomic.LoadInt64(&js.queueLimit)
}
pending, _ := queue.push(&jsAPIRoutedReq{jsub, sub, acc, subject, reply, copyBytes(rmsg), c.pa})
if pending >= int(limit) {
s.rateLimitFormatWarnf("JetStream API queue limit reached, dropping %d requests", pending)
drained := int64(s.jsAPIRoutedReqs.drain())
s.rateLimitFormatWarnf("%s limit reached, dropping %d requests", queue.name, pending)
drained := int64(queue.drain())
atomic.AddInt64(&js.apiInflight, -drained)
s.publishAdvisory(nil, JSAdvisoryAPILimitReached, JSAPILimitReachedAdvisory{
@@ -935,29 +895,45 @@ func (s *Server) processJSAPIRoutedRequests() {
defer s.grWG.Done()
s.mu.RLock()
queue := s.jsAPIRoutedReqs
queue, infoqueue := s.jsAPIRoutedReqs, s.jsAPIRoutedInfoReqs
client := &client{srv: s, kind: JETSTREAM}
s.mu.RUnlock()
js := s.getJetStream()
processFromQueue := func(ipq *ipQueue[*jsAPIRoutedReq]) {
// Only pop one item at a time here, otherwise if the system is recovering
// from queue buildup, then one worker will pull off all the tasks and the
// others will be starved of work.
if r, ok := ipq.popOne(); ok && r != nil {
client.pa = r.pa
start := time.Now()
r.jsub.icb(r.sub, client, r.acc, r.subject, r.reply, r.msg)
if dur := time.Since(start); dur >= readLoopReportThreshold {
s.Warnf("Internal subscription on %q took too long: %v", r.subject, dur)
}
atomic.AddInt64(&js.apiInflight, -1)
}
}
for {
// First select case is prioritizing queue, we will only fall through
// to the second select case that considers infoqueue if queue is empty.
// This effectively means infos are deprioritized.
select {
case <-queue.ch:
// Only pop one item at a time here, otherwise if the system is recovering
// from queue buildup, then one worker will pull off all the tasks and the
// others will be starved of work.
for r, ok := queue.popOne(); ok && r != nil; r, ok = queue.popOne() {
client.pa = r.pa
start := time.Now()
r.jsub.icb(r.sub, client, r.acc, r.subject, r.reply, r.msg)
if dur := time.Since(start); dur >= readLoopReportThreshold {
s.Warnf("Internal subscription on %q took too long: %v", r.subject, dur)
}
atomic.AddInt64(&js.apiInflight, -1)
}
processFromQueue(queue)
case <-s.quitCh:
return
default:
select {
case <-infoqueue.ch:
processFromQueue(infoqueue)
case <-queue.ch:
processFromQueue(queue)
case <-s.quitCh:
return
}
}
}
}
@@ -976,7 +952,8 @@ func (s *Server) setJetStreamExportSubs() error {
if mp > maxProcs {
mp = maxProcs
}
s.jsAPIRoutedReqs = newIPQueue[*jsAPIRoutedReq](s, "Routed JS API Requests")
s.jsAPIRoutedReqs = newIPQueue[*jsAPIRoutedReq](s, "JetStream API queue")
s.jsAPIRoutedInfoReqs = newIPQueue[*jsAPIRoutedReq](s, "JetStream API info queue")
for i := 0; i < mp; i++ {
s.startGoRoutine(s.processJSAPIRoutedRequests)
}
@@ -992,20 +969,13 @@ func (s *Server) setJetStreamExportSubs() error {
}
// API handles themselves.
// infopairs are deprioritized compared to pairs in processJSAPIRoutedRequests.
pairs := []struct {
subject string
handler msgHandler
}{
{JSApiAccountInfo, s.jsAccountInfoRequest},
{JSApiTemplateCreate, s.jsTemplateCreateRequest},
{JSApiTemplates, s.jsTemplateNamesRequest},
{JSApiTemplateInfo, s.jsTemplateInfoRequest},
{JSApiTemplateDelete, s.jsTemplateDeleteRequest},
{JSApiStreamCreate, s.jsStreamCreateRequest},
{JSApiStreamUpdate, s.jsStreamUpdateRequest},
{JSApiStreams, s.jsStreamNamesRequest},
{JSApiStreamList, s.jsStreamListRequest},
{JSApiStreamInfo, s.jsStreamInfoRequest},
{JSApiStreamDelete, s.jsStreamDeleteRequest},
{JSApiStreamPurge, s.jsStreamPurgeRequest},
{JSApiStreamSnapshot, s.jsStreamSnapshotRequest},
@@ -1018,23 +988,40 @@ func (s *Server) setJetStreamExportSubs() error {
{JSApiConsumerCreateEx, s.jsConsumerCreateRequest},
{JSApiConsumerCreate, s.jsConsumerCreateRequest},
{JSApiDurableCreate, s.jsConsumerCreateRequest},
{JSApiConsumers, s.jsConsumerNamesRequest},
{JSApiConsumerList, s.jsConsumerListRequest},
{JSApiConsumerInfo, s.jsConsumerInfoRequest},
{JSApiConsumerDelete, s.jsConsumerDeleteRequest},
{JSApiConsumerPause, s.jsConsumerPauseRequest},
{JSApiConsumerUnpin, s.jsConsumerUnpinRequest},
}
infopairs := []struct {
subject string
handler msgHandler
}{
{JSApiAccountInfo, s.jsAccountInfoRequest},
{JSApiStreams, s.jsStreamNamesRequest},
{JSApiStreamList, s.jsStreamListRequest},
{JSApiStreamInfo, s.jsStreamInfoRequest},
{JSApiConsumers, s.jsConsumerNamesRequest},
{JSApiConsumerList, s.jsConsumerListRequest},
{JSApiConsumerInfo, s.jsConsumerInfoRequest},
}
js.mu.Lock()
defer js.mu.Unlock()
for _, p := range pairs {
// As well as populating js.apiSubs for the dispatch function to use, we
// will also populate js.infoSubs, so that the dispatch function can
// decide quickly whether or not the request is an info request or not.
for _, p := range append(infopairs, pairs...) {
sub := &subscription{subject: []byte(p.subject), icb: p.handler}
if err := js.apiSubs.Insert(sub); err != nil {
return err
}
}
for _, p := range infopairs {
if err := js.infoSubs.Insert(p.subject, struct{}{}); err != nil {
return err
}
}
return nil
}
@@ -1239,7 +1226,7 @@ func (s *Server) unmarshalRequest(c *client, acc *Account, subject string, msg [
c.RateLimitWarnf("Invalid JetStream request '%s > %s': %s", acc, subject, err)
if s.JetStreamConfig().Strict {
if js := s.getJetStream(); js != nil && js.config.Strict {
return err
}
@@ -1345,10 +1332,6 @@ func (s *Server) jsAccountInfoRequest(sub *subscription, c *client, _ *Account,
}
// Helpers for token extraction.
func templateNameFromSubject(subject string) string {
return tokenAt(subject, 6)
}
func streamNameFromSubject(subject string) string {
return tokenAt(subject, 5)
}
@@ -1357,223 +1340,6 @@ func consumerNameFromSubject(subject string) string {
return tokenAt(subject, 6)
}
// Request to create a new template.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (s *Server) jsTemplateCreateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
if c == nil {
return
}
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
if err != nil {
s.Warnf(badAPIRequestT, msg)
return
}
var resp = JSApiStreamTemplateCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateCreateResponseType}}
if errorOnRequiredApiLevel(hdr) {
resp.Error = NewJSRequiredApiLevelError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if !acc.JetStreamEnabled() {
resp.Error = NewJSNotEnabledForAccountError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
// Not supported for now.
if s.JetStreamIsClustered() {
resp.Error = NewJSClusterUnSupportFeatureError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
var cfg StreamTemplateConfig
if err := s.unmarshalRequest(c, acc, subject, msg, &cfg); err != nil {
resp.Error = NewJSInvalidJSONError(err)
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
templateName := templateNameFromSubject(subject)
if templateName != cfg.Name {
resp.Error = NewJSTemplateNameNotMatchSubjectError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
t, err := acc.addStreamTemplate(&cfg)
if err != nil {
resp.Error = NewJSStreamTemplateCreateError(err, Unless(err))
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
t.mu.Lock()
tcfg := t.StreamTemplateConfig.deepCopy()
streams := t.streams
if streams == nil {
streams = []string{}
}
t.mu.Unlock()
resp.StreamTemplateInfo = &StreamTemplateInfo{Config: tcfg, Streams: streams}
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
}
// Request for the list of all template names.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (s *Server) jsTemplateNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
if c == nil {
return
}
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
if err != nil {
s.Warnf(badAPIRequestT, msg)
return
}
var resp = JSApiStreamTemplateNamesResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateNamesResponseType}}
if errorOnRequiredApiLevel(hdr) {
resp.Error = NewJSRequiredApiLevelError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if !acc.JetStreamEnabled() {
resp.Error = NewJSNotEnabledForAccountError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
// Not supported for now.
if s.JetStreamIsClustered() {
resp.Error = NewJSClusterUnSupportFeatureError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
var offset int
if isJSONObjectOrArray(msg) {
var req JSApiStreamTemplatesRequest
if err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {
resp.Error = NewJSInvalidJSONError(err)
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
offset = req.Offset
}
ts := acc.templates()
slices.SortFunc(ts, func(i, j *streamTemplate) int {
return cmp.Compare(i.StreamTemplateConfig.Name, j.StreamTemplateConfig.Name)
})
tcnt := len(ts)
if offset > tcnt {
offset = tcnt
}
for _, t := range ts[offset:] {
t.mu.Lock()
name := t.Name
t.mu.Unlock()
resp.Templates = append(resp.Templates, name)
if len(resp.Templates) >= JSApiNamesLimit {
break
}
}
resp.Total = tcnt
resp.Limit = JSApiNamesLimit
resp.Offset = offset
if resp.Templates == nil {
resp.Templates = []string{}
}
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
}
// Request for information about a stream template.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (s *Server) jsTemplateInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
if c == nil {
return
}
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
if err != nil {
s.Warnf(badAPIRequestT, msg)
return
}
var resp = JSApiStreamTemplateInfoResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateInfoResponseType}}
if errorOnRequiredApiLevel(hdr) {
resp.Error = NewJSRequiredApiLevelError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if !acc.JetStreamEnabled() {
resp.Error = NewJSNotEnabledForAccountError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if !isEmptyRequest(msg) {
resp.Error = NewJSNotEmptyRequestError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
name := templateNameFromSubject(subject)
t, err := acc.lookupStreamTemplate(name)
if err != nil {
resp.Error = NewJSStreamTemplateNotFoundError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
t.mu.Lock()
cfg := t.StreamTemplateConfig.deepCopy()
streams := t.streams
if streams == nil {
streams = []string{}
}
t.mu.Unlock()
resp.StreamTemplateInfo = &StreamTemplateInfo{Config: cfg, Streams: streams}
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
}
// Request to delete a stream template.
// Deprecated: stream templates are deprecated and will be removed in a future version.
func (s *Server) jsTemplateDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
if c == nil {
return
}
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
if err != nil {
s.Warnf(badAPIRequestT, msg)
return
}
var resp = JSApiStreamTemplateDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateDeleteResponseType}}
if errorOnRequiredApiLevel(hdr) {
resp.Error = NewJSRequiredApiLevelError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if !acc.JetStreamEnabled() {
resp.Error = NewJSNotEnabledForAccountError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if !isEmptyRequest(msg) {
resp.Error = NewJSNotEmptyRequestError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
name := templateNameFromSubject(subject)
err = acc.deleteStreamTemplate(name)
if err != nil {
resp.Error = NewJSStreamTemplateDeleteError(err, Unless(err))
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
resp.Success = true
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
}
func (s *Server) jsonResponse(v any) string {
b, err := json.Marshal(v)
if err != nil {
@@ -2109,7 +1875,7 @@ func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, a *Account, s
if cc != nil {
// Check to make sure the stream is assigned.
js.mu.RLock()
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, streamName)
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, streamName)
var offline bool
if sa != nil {
clusterWideConsCount = len(sa.consumers)
@@ -2338,7 +2104,7 @@ func (s *Server) jsStreamLeaderStepDownRequest(sub *subscription, c *client, _ *
}
js.mu.RLock()
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, name)
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, name)
js.mu.RUnlock()
if isLeader && sa == nil {
@@ -2455,7 +2221,7 @@ func (s *Server) jsConsumerLeaderStepDownRequest(sub *subscription, c *client, _
consumer := tokenAt(subject, 7)
js.mu.RLock()
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
js.mu.RUnlock()
if isLeader && sa == nil {
@@ -3456,7 +3222,7 @@ func (s *Server) jsMsgDeleteRequest(sub *subscription, c *client, _ *Account, su
}
js.mu.RLock()
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
js.mu.RUnlock()
if isLeader && sa == nil {
@@ -3581,7 +3347,7 @@ func (s *Server) jsMsgGetRequest(sub *subscription, c *client, _ *Account, subje
}
js.mu.RLock()
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
js.mu.RUnlock()
if isLeader && sa == nil {
@@ -3876,7 +3642,7 @@ func (s *Server) jsStreamPurgeRequest(sub *subscription, c *client, _ *Account,
}
js.mu.RLock()
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
js.mu.RUnlock()
if isLeader && sa == nil {
@@ -4048,6 +3814,13 @@ func (s *Server) jsStreamRestoreRequest(sub *subscription, c *client, _ *Account
return
}
// Check for path like separators in the name.
if strings.ContainsAny(stream, `\/`) {
resp.Error = NewJSStreamNameContainsPathSeparatorsError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
if s.JetStreamIsClustered() {
s.jsClusteredStreamRestoreRequest(ci, acc, &req, subject, reply, rmsg)
return
@@ -4597,11 +4370,49 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
isClustered := s.JetStreamIsClustered()
// Determine if we should proceed here when we are in clustered mode.
direct := req.Config.Direct
if isClustered {
if req.Config.Direct {
// Check to see if we have this stream and are the stream leader.
if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {
return
if direct {
// If it's just a direct consumer, check for stream leader.
if !req.Config.Sourcing {
// Check to see if we have this stream and are the stream leader.
if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {
return
}
} else {
// Otherwise, we either need this to be answered by the stream or meta leader.
var cc *jetStreamCluster
js, cc = s.getJetStreamCluster()
if js == nil || cc == nil {
return
}
js.mu.RLock()
sa := js.streamAssignmentOrInflight(acc.Name, streamNameFromSubject(subject))
if sa == nil {
js.mu.RUnlock()
return
}
// If the stream is WQ or Interest, we need the meta leader to answer.
if sa.Config.Retention != LimitsPolicy {
direct = false
}
js.mu.RUnlock()
if direct {
// Check to see if we have this stream and are the stream leader.
if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {
return
}
} else {
if js.isLeaderless() {
resp.Error = NewJSClusterNotAvailError()
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
return
}
// Make sure we are meta leader.
if !s.JetStreamIsLeader() {
return
}
}
}
} else {
var cc *jetStreamCluster
@@ -4645,6 +4456,7 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
// Legacy ephemeral.
rt = ccLegacyEphemeral
streamName = streamNameFromSubject(subject)
consumerName = req.Config.Name
} else {
// New style and durable legacy.
if tokenAt(subject, 4) == "DURABLE" {
@@ -4736,7 +4548,7 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
return
}
if isClustered && !req.Config.Direct {
if isClustered && !direct {
s.jsClusteredConsumerRequest(ci, acc, subject, reply, rmsg, req.Stream, &req.Config, req.Action, req.Pedantic)
return
}
@@ -4760,6 +4572,23 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
return
}
// If the consumer is a direct sourcing consumer, we need to "upgrade"
// it to be durable without AckNone if not a Limits-based stream.
if req.Config.Direct && req.Config.Sourcing && req.Config.Name != _EMPTY_ {
if !isClustered && stream.isInterestRetention() {
req.Config.Direct = false
req.Config.Durable = req.Config.Name
req.Config.AckPolicy = AckFlowControl
req.Config.AckWait = 0
req.Config.MaxDeliver = 0
req.Config.InactiveThreshold = 0
} else {
// Otherwise, need to append a randomized suffix since the source uses a stable name.
req.Config.Name = fmt.Sprintf("%s-%s", req.Config.Name, createConsumerName())
consumerName = req.Config.Name
}
}
if o := stream.lookupConsumer(consumerName); o != nil {
if o.offlineReason != _EMPTY_ {
resp.Error = NewJSConsumerOfflineReasonError(errors.New(o.offlineReason))
@@ -4770,6 +4599,12 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
// it back to whatever the current configured value is.
o.mu.RLock()
req.Config.PauseUntil = o.cfg.PauseUntil
// If a durable sourcing consumer is used, we need to reset the deliver policy.
if req.Config.Sourcing && req.Config.Durable != _EMPTY_ {
req.Config.DeliverPolicy = o.cfg.DeliverPolicy
req.Config.OptStartSeq = o.cfg.OptStartSeq
req.Config.OptStartTime = o.cfg.OptStartTime
}
o.mu.RUnlock()
}
@@ -5079,7 +4914,7 @@ func (s *Server) jsConsumerInfoRequest(sub *subscription, c *client, _ *Account,
groupCreated := meta.Created()
js.mu.RLock()
isLeader, sa, ca := cc.isLeader(), js.streamAssignment(acc.Name, streamName), js.consumerAssignment(acc.Name, streamName, consumerName)
isLeader, sa, ca := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, streamName), js.consumerAssignmentOrInflight(acc.Name, streamName, consumerName)
var rg *raftGroup
var offline, isMember bool
if ca != nil {
@@ -5404,8 +5239,11 @@ func (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account
return
}
nca := *ca
nca := ca.clone()
// We're only holding the read lock and release below,
// we need a copy to prevent concurrent reads/writes.
ncfg := *ca.Config
ncfg.Metadata = maps.Clone(ncfg.Metadata)
nca.Config = &ncfg
meta := cc.meta
js.mu.RUnlock()
@@ -5420,7 +5258,7 @@ func (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account
// Only PauseUntil is updated above, so reuse config for both.
setStaticConsumerMetadata(nca.Config)
eca := encodeAddConsumerAssignment(&nca)
eca := encodeAddConsumerAssignment(nca)
meta.Propose(eca)
resp.PauseUntil = pauseUTC
@@ -5453,7 +5291,13 @@ func (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account
return
}
// We're only holding the read lock and release below,
// we need a copy to prevent concurrent reads/writes.
obs.mu.RLock()
ncfg := obs.cfg
ncfg.Metadata = maps.Clone(ncfg.Metadata)
obs.mu.RUnlock()
pauseUTC := req.PauseUntil.UTC()
if !pauseUTC.IsZero() {
ncfg.PauseUntil = &pauseUTC
+455 -48
View File
@@ -21,6 +21,7 @@ import (
"math/big"
"path/filepath"
"slices"
"strconv"
"strings"
"sync"
"sync/atomic"
@@ -29,60 +30,105 @@ import (
var (
// Tracks the total inflight batches, across all streams and accounts that enable batching.
globalInflightBatches atomic.Int32
globalInflightAtomicBatches atomic.Int64
globalInflightFastBatches atomic.Int64
)
type batching struct {
mu sync.Mutex
group map[string]*batchGroup
mu sync.Mutex
atomic map[string]*atomicBatch
fast map[string]*fastBatch
}
type batchGroup struct {
lseq uint64
store StreamStore
timer *time.Timer
type atomicBatch struct {
timer *time.Timer // Inactivity timer for the batch.
lseq uint64 // The highest sequence for this batch.
store StreamStore // Where the batch is staged before committing.
}
type fastBatch struct {
timer *time.Timer // Inactivity timer for the batch.
lseq uint64 // The highest sequence for this batch.
sseq uint64 // Last persisted stream sequence.
pseq uint64 // Last persisted batch sequence (is always lower or equal to lseq).
fseq uint64 // Sequence of when we last sent a flow message (is always lower or equal to pseq).
pending uint32 // Number of pending messages in the batch waiting to be persisted.
ackMessages uint16 // Ack will be sent every N messages.
maxAckMessages uint16 // Maximum ackMessages value the client allows.
reply string // The last reply subject seen when persisting a message.
gapOk bool // Whether a gap is okay, if not, the batch would be rejected.
commit bool // If the batch is committed.
}
// newAtomicBatch creates an atomic batch publish object.
// Lock should be held.
func (batches *batching) newBatchGroup(mset *stream, batchId string) (*batchGroup, error) {
store, err := newBatchStore(mset, batchId)
func (batches *batching) newAtomicBatch(mset *stream, batchId string, replicas int, storage StorageType, storeDir, streamName string) (*atomicBatch, error) {
store, err := newBatchStore(mset, batchId, replicas, storage, storeDir, streamName)
if err != nil {
return nil, err
}
b := &batchGroup{store: store}
b := &atomicBatch{store: store}
b.setupCleanupTimer(mset, batchId, batches)
return b, nil
}
// setupCleanupTimer sets up a timer to clean up the batch after a timeout.
func (b *atomicBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) {
// Create a timer to clean up after timeout.
timeout := streamMaxBatchTimeout
if maxBatchTimeout := mset.srv.getOpts().JetStreamLimits.MaxBatchTimeout; maxBatchTimeout > 0 {
timeout = maxBatchTimeout
}
timeout := getCleanupTimeout(mset)
b.timer = time.AfterFunc(timeout, func() {
b.cleanup(batchId, batches)
mset.sendStreamBatchAbandonedAdvisory(batchId, BatchTimeout)
})
return b, nil
}
func getBatchStoreDir(mset *stream, batchId string) (string, string) {
mset.mu.RLock()
jsa, name := mset.jsa, mset.cfg.Name
mset.mu.RUnlock()
// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch.
// Returns whether the timer was reset without it having expired before.
func (b *atomicBatch) resetCleanupTimer(mset *stream) bool {
timeout := getCleanupTimeout(mset)
return b.timer.Reset(timeout)
}
jsa.mu.RLock()
sd := jsa.storeDir
jsa.mu.RUnlock()
// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches.
func (b *atomicBatch) cleanup(batchId string, batches *batching) {
batches.mu.Lock()
defer batches.mu.Unlock()
b.cleanupLocked(batchId, batches)
}
// Lock should be held.
func (b *atomicBatch) cleanupLocked(batchId string, batches *batching) {
if b.timer == nil {
return
}
globalInflightAtomicBatches.Add(-1)
b.timer.Stop()
b.store.Delete(true)
delete(batches.atomic, batchId)
// Reset so that another invocation doesn't double-account.
b.timer = nil
}
// Lock should be held.
func (b *atomicBatch) stopLocked() {
if b.timer == nil {
return
}
globalInflightAtomicBatches.Add(-1)
b.timer.Stop()
b.store.Stop()
// Reset so that another invocation doesn't double-account.
b.timer = nil
}
func getBatchStoreDir(storeDir, streamName, batchId string) (string, string) {
bname := getHash(batchId)
return bname, filepath.Join(sd, streamsDir, name, batchesDir, bname)
return bname, filepath.Join(storeDir, streamsDir, streamName, batchesDir, bname)
}
func newBatchStore(mset *stream, batchId string) (StreamStore, error) {
mset.mu.RLock()
replicas, storage := mset.cfg.Replicas, mset.cfg.Storage
mset.mu.RUnlock()
func newBatchStore(mset *stream, batchId string, replicas int, storage StorageType, storeDir, streamName string) (StreamStore, error) {
if replicas == 1 && storage == FileStorage {
bname, storeDir := getBatchStoreDir(mset, batchId)
bname, storeDir := getBatchStoreDir(storeDir, streamName, batchId)
fcfg := FileStoreConfig{AsyncFlush: true, BlockSize: defaultLargeBlockSize, StoreDir: storeDir}
s := mset.srv
prf := s.jsKeyGen(s.getOpts().JetStreamKey, mset.acc.Name)
@@ -101,34 +147,264 @@ func newBatchStore(mset *stream, batchId string) (StreamStore, error) {
// If the timer has already cleaned up the batch, we can't commit.
// Otherwise, we ensure the timer does not clean up the batch in the meantime.
// Lock should be held.
func (b *batchGroup) readyForCommit() bool {
func (b *atomicBatch) readyForCommit() *BatchAbandonReason {
if !b.timer.Stop() {
return &BatchTimeout
}
if b.store.FlushAllPending() != nil {
return &BatchIncomplete
}
return nil
}
// newFastBatch creates a fast batch publish object and registers it in batches.fast.
// Lock should be held.
func (batches *batching) newFastBatch(mset *stream, batchId string, gapOk bool, maxAckMessages uint16) *fastBatch {
b := &fastBatch{gapOk: gapOk, maxAckMessages: maxAckMessages}
if batches.fast == nil {
batches.fast = make(map[string]*fastBatch, 1)
}
batches.fast[batchId] = b
batches.fastBatchInit(b)
b.setupCleanupTimer(mset, batchId, batches)
return b
}
// fastBatchInit (re)initializes the ackMessages field for a fast batch.
// The batch must already be registered in batches.fast.
// Lock should be held.
func (batches *batching) fastBatchInit(b *fastBatch) {
// If it's the only batch, just allow what the client wants, otherwise we'll
// need to coordinate and slowly ramp up this publisher.
// TODO(mvv): fast ingest's initial flow value improvements?
ackMessages := min(500, b.maxAckMessages)
if len(batches.fast) > 1 {
ackMessages = 1
}
b.ackMessages = ackMessages
}
// fastBatchReset resets the fast batch to an empty state and sends a flow control message.
// Lock should be held.
func (batches *batching) fastBatchReset(mset *stream, batchId string, b *fastBatch) {
// If the timer already stopped before we could commit, we clean it up.
if b.timer == nil || (!b.commit && !b.timer.Stop()) {
b.cleanupLocked(batchId, batches)
return
}
// Otherwise, reset the state.
batches.fastBatchInit(b)
b.timer.Reset(getCleanupTimeout(mset))
b.commit = false
b.pending = 0
b.fseq, b.lseq = b.pseq, b.pseq
b.sendFlowControl(b.fseq, mset, b.reply)
}
// fastBatchRegisterSequences registers the highest stored batch and stream sequence and returns
// whether a PubAck should be sent if the batch has been committed.
// If this is called on a follower, it only registers the highest stream and persisted batch sequences.
// Lock should be held.
func (batches *batching) fastBatchRegisterSequences(mset *stream, reply string, streamSeq uint64, isLeader bool, batch *FastBatch) bool {
b, ok := batches.fast[batch.id]
if !ok || !isLeader {
// If this batch has committed, we can clean it up.
if batch.commit {
if b != nil {
b.cleanupLocked(batch.id, batches)
}
return false
}
// Otherwise, even as a follower, we record the latest state of this batch.
if b == nil || !b.resetCleanupTimer(mset) {
if b != nil {
// The timer couldn't be reset, this means the timer already runs and is likely
// waiting to acquire the lock. We reset the timer here so it doesn't clean up
// this batch that we're about to overwrite.
b.timer = nil
} else {
// If this is a new batch for us, even though we're a follower, we still need
// to account toward the global inflight limit.
globalInflightFastBatches.Add(1)
}
// We'll need a copy as we'll use it as a key and later for cleanup.
batchId := copyString(batch.id)
b = batches.newFastBatch(mset, batchId, batch.gapOk, batch.flow)
}
b.sseq = streamSeq
b.pseq, b.lseq = batch.seq, batch.seq
b.reply = reply
return false
}
b.store.FlushAllPending()
b.reply = reply
if b.pending > 0 {
b.pending--
}
b.sseq = streamSeq
// Store last persisted batch sequence.
// If we have no remaining pending writes, we might have had duplicate messages
// and need to send additional flow control messages.
var skipped bool
if b.pending == 0 {
skipped = true
b.pseq = b.lseq
} else {
b.pseq = batch.seq
}
// If the PubAck needs to be sent now as a result of a commit.
if b.lseq == b.pseq && b.commit {
b.cleanupLocked(batch.id, batches)
// If we skipped ahead due to duplicate messages, send the PubAck with the highest sequence.
if skipped {
var buf [256]byte
pubAck := append(buf[:0], mset.pubAck...)
response := append(pubAck, strconv.FormatUint(b.sseq, 10)...)
response = append(response, fmt.Sprintf(",\"batch\":%q,\"count\":%d}", batch.id, b.lseq)...)
if len(reply) > 0 {
mset.outq.sendMsg(reply, response)
}
return false
}
return true
}
b.checkFlowControl(mset, reply, batches)
return false
}
// checkFlowControl checks whether a flow control message should be sent.
// If so, it updates the flow values to speed up or slow down the publisher if needed.
// Returns whether a flow control message was sent.
// Lock should be held.
func (b *fastBatch) checkFlowControl(mset *stream, reply string, batches *batching) bool {
am := uint64(b.ackMessages)
if b.pseq < b.fseq+am {
return false
}
// Instead of sending multiple flow control messages, skip ahead to only send the last.
steps := (b.pseq - b.fseq) / am
b.fseq += steps * am
// TODO(mvv): fast ingest's dynamic flow value improvements?
// This is currently just a simple value to have a working version. Should take average
// message sizes into account and compare how much this client is contributing to the
// ingest IPQ total size and messages and have publishers share based on that.
maxAckMessages := uint16(500 / len(batches.fast))
if maxAckMessages < 1 {
maxAckMessages = 1
}
// Limit to the client's allowed maximum.
if maxAckMessages > b.maxAckMessages {
maxAckMessages = b.maxAckMessages
}
if b.ackMessages < maxAckMessages {
// Ramp up.
b.ackMessages *= 2
if b.ackMessages > maxAckMessages {
b.ackMessages = maxAckMessages
}
} else if b.ackMessages > maxAckMessages {
// Slow down.
b.ackMessages /= 2
if b.ackMessages <= maxAckMessages {
b.ackMessages = maxAckMessages
}
}
// Finally, send the flow control message.
b.sendFlowControl(b.fseq, mset, reply)
return true
}
// sendFlowControl sends a fast batch flow control message for the current highest sequence.
// Lock should be held.
func (b *fastBatch) sendFlowControl(batchSeq uint64, mset *stream, reply string) {
if len(reply) == 0 {
return
}
response, _ := BatchFlowAck{Sequence: batchSeq, Messages: b.ackMessages}.MarshalJSON()
mset.outq.sendMsg(reply, response)
}
// fastBatchCommit ends the batch and commits the data up to that point. If all messages
// have already been persisted, a PubAck is sent immediately. Otherwise, it will be sent
// after the last message has been persisted.
// Lock should be held.
func (batches *batching) fastBatchCommit(b *fastBatch, batchId string, mset *stream, reply string) bool {
// Either we commit now, or we clean up later, so stop the timer.
if b.timer == nil || (!b.commit && !b.timer.Stop()) {
// Shouldn't be possible for the timer to already be stopped if we haven't committed yet,
// since we pre-check being able to reset the timer. But guard against it anyhow.
return true
}
// Mark that this batch commits.
b.commit = true
// If the whole batch has been persisted, we can respond with the PubAck now.
if b.lseq == b.pseq {
b.cleanupLocked(batchId, batches)
var buf [256]byte
pubAck := append(buf[:0], mset.pubAck...)
response := append(pubAck, strconv.FormatUint(b.sseq, 10)...)
response = append(response, fmt.Sprintf(",\"batch\":%q,\"count\":%d}", batchId, b.lseq)...)
if len(reply) > 0 {
mset.outq.sendMsg(reply, response)
}
return true
}
// Otherwise, we need to wait and the PubAck will be sent when the last message is persisted.
return false
}
// setupCleanupTimer sets up a timer to clean up the batch after a timeout.
func (b *fastBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) {
// Create a timer to clean up after timeout.
timeout := getCleanupTimeout(mset)
b.timer = time.AfterFunc(timeout, func() {
b.cleanup(batchId, batches)
})
}
// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch.
// Returns whether the timer was reset without it having expired before.
func (b *fastBatch) resetCleanupTimer(mset *stream) bool {
if b.commit {
return true
}
if b.timer == nil {
return false
}
timeout := getCleanupTimeout(mset)
return b.timer.Reset(timeout)
}
// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches.
func (b *batchGroup) cleanup(batchId string, batches *batching) {
func (b *fastBatch) cleanup(batchId string, batches *batching) {
batches.mu.Lock()
defer batches.mu.Unlock()
b.cleanupLocked(batchId, batches)
}
// Lock should be held.
func (b *batchGroup) cleanupLocked(batchId string, batches *batching) {
globalInflightBatches.Add(-1)
func (b *fastBatch) cleanupLocked(batchId string, batches *batching) {
// If the timer is nil, it means this batch has been replaced with a new one.
// This can happen on a follower depending on timing.
if b.timer == nil {
return
}
globalInflightFastBatches.Add(-1)
b.timer.Stop()
b.store.Delete(true)
delete(batches.group, batchId)
delete(batches.fast, batchId)
// Reset so that another invocation doesn't double-account.
b.timer = nil
}
// Lock should be held.
func (b *batchGroup) stopLocked() {
globalInflightBatches.Add(-1)
b.timer.Stop()
b.store.Stop()
// getCleanupTimeout returns the timeout for the batch, taking into account the server's limits.
func getCleanupTimeout(mset *stream) time.Duration {
timeout := streamMaxBatchTimeout
if maxBatchTimeout := mset.srv.getOpts().JetStreamLimits.MaxBatchTimeout; maxBatchTimeout > 0 {
timeout = maxBatchTimeout
}
return timeout
}
// batchStagedDiff stages all changes for consistency checks until commit.
@@ -136,6 +412,7 @@ type batchStagedDiff struct {
msgIds map[string]struct{}
counter map[string]*msgCounterRunningTotal
inflight map[string]*inflightSubjectRunningTotal
inflightTransform map[uint64]string
expectedPerSubject map[string]*batchExpectedPerSubject
}
@@ -180,6 +457,16 @@ func (diff *batchStagedDiff) commit(mset *stream) {
}
}
// Track inflight subject transforms.
if len(diff.inflightTransform) > 0 {
if mset.inflightTransform == nil {
mset.inflightTransform = make(map[uint64]string, len(diff.inflightTransform))
}
for clseq, subj := range diff.inflightTransform {
mset.inflightTransform[clseq] = subj
}
}
// Track sequence and subject.
if len(diff.expectedPerSubject) > 0 {
if mset.expectedPerSubjectSequence == nil {
@@ -238,7 +525,7 @@ func (batch *batchApply) rejectBatchState(mset *stream) {
// mset.mu lock must NOT be held or used.
// mset.clMu lock must be held.
func checkMsgHeadersPreClusteredProposal(
diff *batchStagedDiff, mset *stream, subject string, hdr []byte, msg []byte, sourced bool, name string,
diff *batchStagedDiff, mset *stream, subject, rsubject string, hdr []byte, msg []byte, sourced bool, name string,
jsa *jsAccount, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules bool,
discard DiscardPolicy, discardNewPer bool, maxMsgSize int, maxMsgs int64, maxMsgsPer int64, maxBytes int64,
) ([]byte, []byte, uint64, *ApiError, error) {
@@ -515,8 +802,9 @@ func checkMsgHeadersPreClusteredProposal(
}
// Message scheduling.
if schedule, ok := getMessageSchedule(hdr); !ok {
apiErr := NewJSMessageSchedulesPatternInvalidError()
if sourced {
// noop, sourced messages were already validated by the origin stream.
} else if schedule, apiErr := getMessageSchedule(hdr); apiErr != nil {
if !allowMsgSchedules {
apiErr = NewJSMessageSchedulesDisabledError()
}
@@ -528,22 +816,40 @@ func checkMsgHeadersPreClusteredProposal(
} else if scheduleTtl, ok := getMessageScheduleTTL(hdr); !ok {
apiErr := NewJSMessageSchedulesTTLInvalidError()
return hdr, msg, 0, apiErr, apiErr
} else if scheduleRollup := getMessageScheduleRollup(hdr); scheduleRollup != _EMPTY_ && scheduleRollup != JSMsgRollupSubject {
apiErr := NewJSMessageSchedulesRollupInvalidError()
return hdr, msg, 0, apiErr, apiErr
} else if scheduleTtl != _EMPTY_ && !allowTTL {
return hdr, msg, 0, NewJSMessageTTLDisabledError(), errMsgTTLDisabled
} else if scheduleTarget := getMessageScheduleTarget(hdr); scheduleTarget == _EMPTY_ ||
!IsValidPublishSubject(scheduleTarget) || SubjectsCollide(scheduleTarget, subject) {
!IsValidPublishSubject(scheduleTarget) || scheduleTarget == subject {
apiErr := NewJSMessageSchedulesTargetInvalidError()
return hdr, msg, 0, apiErr, apiErr
} else if scheduleSource := getMessageScheduleSource(hdr); scheduleSource != _EMPTY_ &&
(scheduleSource == scheduleTarget || scheduleSource == subject || !IsValidPublishSubject(scheduleSource)) {
apiErr := NewJSMessageSchedulesSourceInvalidError()
return hdr, msg, 0, apiErr, apiErr
} else {
mset.cfgMu.RLock()
match := slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool {
return SubjectsCollide(subj, scheduleTarget)
})
mset.cfgMu.RUnlock()
if !match {
mset.cfgMu.RUnlock()
apiErr := NewJSMessageSchedulesTargetInvalidError()
return hdr, msg, 0, apiErr, apiErr
}
if scheduleSource != _EMPTY_ {
match = slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool {
return SubjectsCollide(subj, scheduleSource)
})
if !match {
mset.cfgMu.RUnlock()
apiErr := NewJSMessageSchedulesSourceInvalidError()
return hdr, msg, 0, apiErr, apiErr
}
}
mset.cfgMu.RUnlock()
// Add a rollup sub header if it doesn't already exist.
// Otherwise, it must exist already as a rollup on the subject.
@@ -555,10 +861,32 @@ func checkMsgHeadersPreClusteredProposal(
}
}
}
if scheduleNext := sliceHeader(JSScheduleNext, hdr); len(scheduleNext) > 0 && !sourced {
// Clients may only use Nats-Schedule-Next to purge a schedule.
if bytesToString(scheduleNext) != JSScheduleNextPurge {
apiErr := NewJSMessageSchedulesSchedulerInvalidError()
return hdr, msg, 0, apiErr, apiErr
}
// Nats-Scheduler must accompany the purge and:
// - it must NOT be empty.
// - it must NOT match the publish subject.
if scheduler := sliceHeader(JSScheduler, hdr); len(scheduler) == 0 ||
bytesToString(scheduler) == subject || !IsValidPublishSubject(bytesToString(scheduler)) {
apiErr := NewJSMessageSchedulesSchedulerInvalidError()
return hdr, msg, 0, apiErr, apiErr
} else if !allowMsgSchedules {
apiErr := NewJSMessageSchedulesDisabledError()
return hdr, msg, 0, apiErr, apiErr
}
} else if !sourced && len(sliceHeader(JSScheduler, hdr)) > 0 {
// Clients may only use Nats-Scheduler alongside Nats-Schedule-Next.
apiErr := NewJSMessageSchedulesSchedulerInvalidError()
return hdr, msg, 0, apiErr, apiErr
}
// Check for any rollups.
if rollup := getRollup(hdr); rollup != _EMPTY_ {
if !allowRollup || denyPurge {
if (!allowRollup || denyPurge) && !sourced {
err := errors.New("rollup not permitted")
return hdr, msg, 0, NewJSStreamRollupFailedError(err), err
}
@@ -607,6 +935,19 @@ func checkMsgHeadersPreClusteredProposal(
diff.inflight[subject] = i
}
// Subject transform.
if subject != rsubject {
// The 'subject' is a transformed subject used for consistency checks.
// But since we propose the original (raw) subject to our peers, we need
// to store the transformed subject separately for when we apply.
// TODO(mvv): since subject transforms are handled by each replica individually, this has a
// potential for desync given out-of-order stream subject transform updates.
if diff.inflightTransform == nil {
diff.inflightTransform = make(map[uint64]string, 1)
}
diff.inflightTransform[mset.clseq] = subject
}
// Check if we have discard new with max msgs or bytes.
// We need to deny here otherwise we'd need to bump CLFS, and it could succeed on some
// peers and not others depending on consumer ack state (if interest policy).
@@ -639,7 +980,8 @@ func checkMsgHeadersPreClusteredProposal(
}
// Similarly, check DiscardNew per-subject threshold to not need to bump CLFS.
if discardNewPer && maxMsgsPer > 0 {
// Allow rollup messages through since they will purge after storing.
if discardNewPer && maxMsgsPer > 0 && len(sliceHeader(JSMsgRollup, hdr)) == 0 {
// Get the current total for this subject.
totalMsgsForSubject := mset.store.SubjectsTotals(subject)[subject]
// Add inflight count in this batch and for this stream.
@@ -656,3 +998,68 @@ func checkMsgHeadersPreClusteredProposal(
return hdr, msg, 0, nil, nil
}
// recalculateClusteredSeq initializes or updates mset.clseq, for example after a leader change.
// This is reused for normal clustered publishing into a stream, and for atomic and fast batch publishing.
// mset.clMu lock must be held.
func recalculateClusteredSeq(mset *stream, needStreamLock bool) (lseq uint64) {
// Need to unlock and re-acquire the locks in the proper order.
mset.clMu.Unlock()
// Locking order is stream -> batchMu -> clMu
if needStreamLock {
mset.mu.RLock()
}
batch := mset.batchApply
var batchCount uint64
if batch != nil {
batch.mu.Lock()
batchCount = batch.count
}
mset.clMu.Lock()
// Re-capture
lseq = mset.lseq
mset.clseq = lseq + mset.clfs + batchCount
// Keep hold of the mset.clMu, but unlock the others.
if batch != nil {
batch.mu.Unlock()
}
if needStreamLock {
mset.mu.RUnlock()
}
return lseq
}
// commitSingleMsg commits and proposes a single message to the node.
// This is reused both for normal publishing into a stream, and for fast batch publishing.
// mset.clMu lock must be held.
func commitSingleMsg(
diff *batchStagedDiff, mset *stream, subject string, reply string, hdr []byte, msg []byte, name string,
jsa *jsAccount, mt *msgTrace, node RaftNode, replicas int, lseq uint64,
) error {
// Do proposal.
esm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), false)
if err := node.Propose(esm); err != nil {
return err
}
var mtKey uint64
if mt != nil {
mtKey = mset.clseq
if mset.mt == nil {
mset.mt = make(map[uint64]*msgTrace)
}
mset.mt[mtKey] = mt
}
diff.commit(mset)
mset.clseq++
mset.trackReplicationTraffic(node, len(esm), replicas)
// Check to see if we are being overrun.
// TODO(dlc) - Make this a limit where we drop messages to protect ourselves, but allow to be configured.
if mset.clseq-(lseq+mset.clfs) > streamLagWarnThreshold {
lerr := fmt.Errorf("JetStream stream '%s > %s' has high message lag", jsa.acc().Name, name)
mset.srv.RateLimitWarnf("%s", lerr.Error())
}
return nil
}
File diff suppressed because it is too large Load Diff
@@ -29,12 +29,30 @@ const (
// JSAtomicPublishTooLargeBatchErrF atomic publish batch is too large: {size}
JSAtomicPublishTooLargeBatchErrF ErrorIdentifier = 10199
// JSAtomicPublishTooManyInflight atomic publish too many inflight
JSAtomicPublishTooManyInflight ErrorIdentifier = 10210
// JSAtomicPublishUnsupportedHeaderBatchErr atomic publish unsupported header used: {header}
JSAtomicPublishUnsupportedHeaderBatchErr ErrorIdentifier = 10177
// JSBadRequestErr bad request
JSBadRequestErr ErrorIdentifier = 10003
// JSBatchPublishDisabledErr batch publish is disabled
JSBatchPublishDisabledErr ErrorIdentifier = 10205
// JSBatchPublishInvalidBatchIDErr batch publish ID is invalid
JSBatchPublishInvalidBatchIDErr ErrorIdentifier = 10207
// JSBatchPublishInvalidPatternErr batch publish pattern is invalid
JSBatchPublishInvalidPatternErr ErrorIdentifier = 10206
// JSBatchPublishTooManyInflight batch publish too many inflight
JSBatchPublishTooManyInflight ErrorIdentifier = 10211
// JSBatchPublishUnknownBatchIDErr batch publish ID unknown
JSBatchPublishUnknownBatchIDErr ErrorIdentifier = 10208
// JSClusterIncompleteErr incomplete results
JSClusterIncompleteErr ErrorIdentifier = 10004
@@ -71,6 +89,21 @@ const (
// JSClusterUnSupportFeatureErr not currently supported in clustered mode
JSClusterUnSupportFeatureErr ErrorIdentifier = 10036
// JSConsumerAckFCRequiresFCErr flow control ack policy requires flow control
JSConsumerAckFCRequiresFCErr ErrorIdentifier = 10219
// JSConsumerAckFCRequiresMaxAckPendingErr flow control ack policy requires max ack pending
JSConsumerAckFCRequiresMaxAckPendingErr ErrorIdentifier = 10220
// JSConsumerAckFCRequiresNoAckWaitErr flow control ack policy requires unset ack wait
JSConsumerAckFCRequiresNoAckWaitErr ErrorIdentifier = 10221
// JSConsumerAckFCRequiresNoMaxDeliverErr flow control ack policy requires unset max deliver
JSConsumerAckFCRequiresNoMaxDeliverErr ErrorIdentifier = 10222
// JSConsumerAckFCRequiresPushErr flow control ack policy requires a push based consumer
JSConsumerAckFCRequiresPushErr ErrorIdentifier = 10218
// JSConsumerAckPolicyInvalidErr consumer ack policy invalid
JSConsumerAckPolicyInvalidErr ErrorIdentifier = 10181
@@ -167,6 +200,9 @@ const (
// JSConsumerInvalidPriorityGroupErr Provided priority group does not exist for this consumer
JSConsumerInvalidPriorityGroupErr ErrorIdentifier = 10160
// JSConsumerInvalidResetErr invalid reset: {err}
JSConsumerInvalidResetErr ErrorIdentifier = 10204
// JSConsumerInvalidSamplingErrF failed to parse consumer sampling configuration: {err}
JSConsumerInvalidSamplingErrF ErrorIdentifier = 10095
@@ -317,21 +353,36 @@ const (
// JSMessageSchedulesRollupInvalidErr message schedules invalid rollup
JSMessageSchedulesRollupInvalidErr ErrorIdentifier = 10192
// JSMessageSchedulesSchedulerInvalidErr message schedules invalid scheduler
JSMessageSchedulesSchedulerInvalidErr ErrorIdentifier = 10212
// JSMessageSchedulesSourceInvalidErr message schedules source is invalid
JSMessageSchedulesSourceInvalidErr ErrorIdentifier = 10203
// JSMessageSchedulesTTLInvalidErr message schedules invalid per-message TTL
JSMessageSchedulesTTLInvalidErr ErrorIdentifier = 10191
// JSMessageSchedulesTargetInvalidErr message schedules target is invalid
JSMessageSchedulesTargetInvalidErr ErrorIdentifier = 10190
// JSMessageSchedulesTimeZoneInvalidErr message schedules time zone is invalid
JSMessageSchedulesTimeZoneInvalidErr ErrorIdentifier = 10223
// JSMessageTTLDisabledErr per-message TTL is disabled
JSMessageTTLDisabledErr ErrorIdentifier = 10166
// JSMessageTTLInvalidErr invalid per-message TTL
JSMessageTTLInvalidErr ErrorIdentifier = 10165
// JSMirrorConsumerRequiresAckFCErr stream mirror consumer requires flow control ack policy
JSMirrorConsumerRequiresAckFCErr ErrorIdentifier = 10214
// JSMirrorConsumerSetupFailedErrF generic mirror consumer setup failure string ({err})
JSMirrorConsumerSetupFailedErrF ErrorIdentifier = 10029
// JSMirrorDurableConsumerCfgInvalid stream mirror consumer config is invalid
JSMirrorDurableConsumerCfgInvalid ErrorIdentifier = 10213
// JSMirrorInvalidStreamName mirrored stream name is invalid
JSMirrorInvalidStreamName ErrorIdentifier = 10142
@@ -353,6 +404,9 @@ const (
// JSMirrorWithAtomicPublishErr stream mirrors can not also use atomic publishing
JSMirrorWithAtomicPublishErr ErrorIdentifier = 10198
// JSMirrorWithBatchPublishErr stream mirrors can not also use batch publishing
JSMirrorWithBatchPublishErr ErrorIdentifier = 10209
// JSMirrorWithCountersErr stream mirrors can not also calculate counters
JSMirrorWithCountersErr ErrorIdentifier = 10173
@@ -416,12 +470,21 @@ const (
// JSSnapshotDeliverSubjectInvalidErr deliver subject not valid
JSSnapshotDeliverSubjectInvalidErr ErrorIdentifier = 10015
// JSSourceConsumerRequiresAckFCErr stream source consumer requires flow control ack policy
JSSourceConsumerRequiresAckFCErr ErrorIdentifier = 10217
// JSSourceConsumerSetupFailedErrF General source consumer setup failure string ({err})
JSSourceConsumerSetupFailedErrF ErrorIdentifier = 10045
// JSSourceDuplicateDetected source stream, filter and transform (plus external if present) must form a unique combination (duplicate source configuration detected)
JSSourceDuplicateDetected ErrorIdentifier = 10140
// JSSourceDurableConsumerCfgInvalid stream source consumer config is invalid
JSSourceDurableConsumerCfgInvalid ErrorIdentifier = 10215
// JSSourceDurableConsumerDuplicateDetected duplicate stream source consumer detected
JSSourceDurableConsumerDuplicateDetected ErrorIdentifier = 10216
// JSSourceInvalidStreamName sourced stream name is invalid
JSSourceInvalidStreamName ErrorIdentifier = 10141
@@ -619,8 +682,14 @@ var (
JSAtomicPublishInvalidBatchIDErr: {Code: 400, ErrCode: 10179, Description: "atomic publish batch ID is invalid"},
JSAtomicPublishMissingSeqErr: {Code: 400, ErrCode: 10175, Description: "atomic publish sequence is missing"},
JSAtomicPublishTooLargeBatchErrF: {Code: 400, ErrCode: 10199, Description: "atomic publish batch is too large: {size}"},
JSAtomicPublishTooManyInflight: {Code: 429, ErrCode: 10210, Description: "atomic publish too many inflight"},
JSAtomicPublishUnsupportedHeaderBatchErr: {Code: 400, ErrCode: 10177, Description: "atomic publish unsupported header used: {header}"},
JSBadRequestErr: {Code: 400, ErrCode: 10003, Description: "bad request"},
JSBatchPublishDisabledErr: {Code: 400, ErrCode: 10205, Description: "batch publish is disabled"},
JSBatchPublishInvalidBatchIDErr: {Code: 400, ErrCode: 10207, Description: "batch publish ID is invalid"},
JSBatchPublishInvalidPatternErr: {Code: 400, ErrCode: 10206, Description: "batch publish pattern is invalid"},
JSBatchPublishTooManyInflight: {Code: 429, ErrCode: 10211, Description: "batch publish too many inflight"},
JSBatchPublishUnknownBatchIDErr: {Code: 400, ErrCode: 10208, Description: "batch publish ID unknown"},
JSClusterIncompleteErr: {Code: 503, ErrCode: 10004, Description: "incomplete results"},
JSClusterNoPeersErrF: {Code: 400, ErrCode: 10005, Description: "{err}"},
JSClusterNotActiveErr: {Code: 500, ErrCode: 10006, Description: "JetStream not in clustered mode"},
@@ -633,6 +702,11 @@ var (
JSClusterServerNotMemberErr: {Code: 400, ErrCode: 10044, Description: "server is not a member of the cluster"},
JSClusterTagsErr: {Code: 400, ErrCode: 10011, Description: "tags placement not supported for operation"},
JSClusterUnSupportFeatureErr: {Code: 503, ErrCode: 10036, Description: "not currently supported in clustered mode"},
JSConsumerAckFCRequiresFCErr: {Code: 400, ErrCode: 10219, Description: "flow control ack policy requires flow control"},
JSConsumerAckFCRequiresMaxAckPendingErr: {Code: 400, ErrCode: 10220, Description: "flow control ack policy requires max ack pending"},
JSConsumerAckFCRequiresNoAckWaitErr: {Code: 400, ErrCode: 10221, Description: "flow control ack policy requires unset ack wait"},
JSConsumerAckFCRequiresNoMaxDeliverErr: {Code: 400, ErrCode: 10222, Description: "flow control ack policy requires unset max deliver"},
JSConsumerAckFCRequiresPushErr: {Code: 400, ErrCode: 10218, Description: "flow control ack policy requires a push based consumer"},
JSConsumerAckPolicyInvalidErr: {Code: 400, ErrCode: 10181, Description: "consumer ack policy invalid"},
JSConsumerAckWaitNegativeErr: {Code: 400, ErrCode: 10183, Description: "consumer ack wait needs to be positive"},
JSConsumerAlreadyExists: {Code: 400, ErrCode: 10148, Description: "consumer already exists"},
@@ -665,6 +739,7 @@ var (
JSConsumerInvalidGroupNameErr: {Code: 400, ErrCode: 10162, Description: "Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters"},
JSConsumerInvalidPolicyErrF: {Code: 400, ErrCode: 10094, Description: "{err}"},
JSConsumerInvalidPriorityGroupErr: {Code: 400, ErrCode: 10160, Description: "Provided priority group does not exist for this consumer"},
JSConsumerInvalidResetErr: {Code: 400, ErrCode: 10204, Description: "invalid reset: {err}"},
JSConsumerInvalidSamplingErrF: {Code: 400, ErrCode: 10095, Description: "failed to parse consumer sampling configuration: {err}"},
JSConsumerMaxDeliverBackoffErr: {Code: 400, ErrCode: 10116, Description: "max deliver is required to be > length of backoff values"},
JSConsumerMaxPendingAckExcessErrF: {Code: 400, ErrCode: 10121, Description: "consumer max ack pending exceeds system limit of {limit}"},
@@ -715,11 +790,16 @@ var (
JSMessageSchedulesDisabledErr: {Code: 400, ErrCode: 10188, Description: "message schedules is disabled"},
JSMessageSchedulesPatternInvalidErr: {Code: 400, ErrCode: 10189, Description: "message schedules pattern is invalid"},
JSMessageSchedulesRollupInvalidErr: {Code: 400, ErrCode: 10192, Description: "message schedules invalid rollup"},
JSMessageSchedulesSchedulerInvalidErr: {Code: 400, ErrCode: 10212, Description: "message schedules invalid scheduler"},
JSMessageSchedulesSourceInvalidErr: {Code: 400, ErrCode: 10203, Description: "message schedules source is invalid"},
JSMessageSchedulesTTLInvalidErr: {Code: 400, ErrCode: 10191, Description: "message schedules invalid per-message TTL"},
JSMessageSchedulesTargetInvalidErr: {Code: 400, ErrCode: 10190, Description: "message schedules target is invalid"},
JSMessageSchedulesTimeZoneInvalidErr: {Code: 400, ErrCode: 10223, Description: "message schedules time zone is invalid"},
JSMessageTTLDisabledErr: {Code: 400, ErrCode: 10166, Description: "per-message TTL is disabled"},
JSMessageTTLInvalidErr: {Code: 400, ErrCode: 10165, Description: "invalid per-message TTL"},
JSMirrorConsumerRequiresAckFCErr: {Code: 400, ErrCode: 10214, Description: "stream mirror consumer requires flow control ack policy"},
JSMirrorConsumerSetupFailedErrF: {Code: 500, ErrCode: 10029, Description: "{err}"},
JSMirrorDurableConsumerCfgInvalid: {Code: 400, ErrCode: 10213, Description: "stream mirror consumer config is invalid"},
JSMirrorInvalidStreamName: {Code: 400, ErrCode: 10142, Description: "mirrored stream name is invalid"},
JSMirrorInvalidSubjectFilter: {Code: 400, ErrCode: 10151, Description: "mirror transform source: {err}"},
JSMirrorInvalidTransformDestination: {Code: 400, ErrCode: 10154, Description: "mirror transform: {err}"},
@@ -727,6 +807,7 @@ var (
JSMirrorMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10150, Description: "mirror with multiple subject transforms cannot also have a single subject filter"},
JSMirrorOverlappingSubjectFilters: {Code: 400, ErrCode: 10152, Description: "mirror subject filters can not overlap"},
JSMirrorWithAtomicPublishErr: {Code: 400, ErrCode: 10198, Description: "stream mirrors can not also use atomic publishing"},
JSMirrorWithBatchPublishErr: {Code: 400, ErrCode: 10209, Description: "stream mirrors can not also use batch publishing"},
JSMirrorWithCountersErr: {Code: 400, ErrCode: 10173, Description: "stream mirrors can not also calculate counters"},
JSMirrorWithFirstSeqErr: {Code: 400, ErrCode: 10143, Description: "stream mirrors can not have first sequence configured"},
JSMirrorWithMsgSchedulesErr: {Code: 400, ErrCode: 10186, Description: "stream mirrors can not also schedule messages"},
@@ -748,8 +829,11 @@ var (
JSRestoreSubscribeFailedErrF: {Code: 500, ErrCode: 10042, Description: "JetStream unable to subscribe to restore snapshot {subject}: {err}"},
JSSequenceNotFoundErrF: {Code: 400, ErrCode: 10043, Description: "sequence {seq} not found"},
JSSnapshotDeliverSubjectInvalidErr: {Code: 400, ErrCode: 10015, Description: "deliver subject not valid"},
JSSourceConsumerRequiresAckFCErr: {Code: 400, ErrCode: 10217, Description: "stream source consumer requires flow control ack policy"},
JSSourceConsumerSetupFailedErrF: {Code: 500, ErrCode: 10045, Description: "{err}"},
JSSourceDuplicateDetected: {Code: 400, ErrCode: 10140, Description: "duplicate source configuration detected"},
JSSourceDurableConsumerCfgInvalid: {Code: 400, ErrCode: 10215, Description: "stream source consumer config is invalid"},
JSSourceDurableConsumerDuplicateDetected: {Code: 400, ErrCode: 10216, Description: "duplicate stream source consumer detected"},
JSSourceInvalidStreamName: {Code: 400, ErrCode: 10141, Description: "sourced stream name is invalid"},
JSSourceInvalidSubjectFilter: {Code: 400, ErrCode: 10145, Description: "source transform source: {err}"},
JSSourceInvalidTransformDestination: {Code: 400, ErrCode: 10146, Description: "source transform: {err}"},
@@ -923,6 +1007,16 @@ func NewJSAtomicPublishTooLargeBatchError(size interface{}, opts ...ErrorOption)
}
}
// NewJSAtomicPublishTooManyInflightError creates a new JSAtomicPublishTooManyInflight error: "atomic publish too many inflight"
func NewJSAtomicPublishTooManyInflightError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSAtomicPublishTooManyInflight]
}
// NewJSAtomicPublishUnsupportedHeaderBatchError creates a new JSAtomicPublishUnsupportedHeaderBatchErr error: "atomic publish unsupported header used: {header}"
func NewJSAtomicPublishUnsupportedHeaderBatchError(header interface{}, opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -949,6 +1043,56 @@ func NewJSBadRequestError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSBadRequestErr]
}
// NewJSBatchPublishDisabledError creates a new JSBatchPublishDisabledErr error: "batch publish is disabled"
func NewJSBatchPublishDisabledError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSBatchPublishDisabledErr]
}
// NewJSBatchPublishInvalidBatchIDError creates a new JSBatchPublishInvalidBatchIDErr error: "batch publish ID is invalid"
func NewJSBatchPublishInvalidBatchIDError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSBatchPublishInvalidBatchIDErr]
}
// NewJSBatchPublishInvalidPatternError creates a new JSBatchPublishInvalidPatternErr error: "batch publish pattern is invalid"
func NewJSBatchPublishInvalidPatternError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSBatchPublishInvalidPatternErr]
}
// NewJSBatchPublishTooManyInflightError creates a new JSBatchPublishTooManyInflight error: "batch publish too many inflight"
func NewJSBatchPublishTooManyInflightError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSBatchPublishTooManyInflight]
}
// NewJSBatchPublishUnknownBatchIDError creates a new JSBatchPublishUnknownBatchIDErr error: "batch publish ID unknown"
func NewJSBatchPublishUnknownBatchIDError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSBatchPublishUnknownBatchIDErr]
}
// NewJSClusterIncompleteError creates a new JSClusterIncompleteErr error: "incomplete results"
func NewJSClusterIncompleteError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -1075,6 +1219,56 @@ func NewJSClusterUnSupportFeatureError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSClusterUnSupportFeatureErr]
}
// NewJSConsumerAckFCRequiresFCError creates a new JSConsumerAckFCRequiresFCErr error: "flow control ack policy requires flow control"
func NewJSConsumerAckFCRequiresFCError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSConsumerAckFCRequiresFCErr]
}
// NewJSConsumerAckFCRequiresMaxAckPendingError creates a new JSConsumerAckFCRequiresMaxAckPendingErr error: "flow control ack policy requires max ack pending"
func NewJSConsumerAckFCRequiresMaxAckPendingError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSConsumerAckFCRequiresMaxAckPendingErr]
}
// NewJSConsumerAckFCRequiresNoAckWaitError creates a new JSConsumerAckFCRequiresNoAckWaitErr error: "flow control ack policy requires unset ack wait"
func NewJSConsumerAckFCRequiresNoAckWaitError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSConsumerAckFCRequiresNoAckWaitErr]
}
// NewJSConsumerAckFCRequiresNoMaxDeliverError creates a new JSConsumerAckFCRequiresNoMaxDeliverErr error: "flow control ack policy requires unset max deliver"
func NewJSConsumerAckFCRequiresNoMaxDeliverError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSConsumerAckFCRequiresNoMaxDeliverErr]
}
// NewJSConsumerAckFCRequiresPushError creates a new JSConsumerAckFCRequiresPushErr error: "flow control ack policy requires a push based consumer"
func NewJSConsumerAckFCRequiresPushError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSConsumerAckFCRequiresPushErr]
}
// NewJSConsumerAckPolicyInvalidError creates a new JSConsumerAckPolicyInvalidErr error: "consumer ack policy invalid"
func NewJSConsumerAckPolicyInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -1419,6 +1613,22 @@ func NewJSConsumerInvalidPriorityGroupError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSConsumerInvalidPriorityGroupErr]
}
// NewJSConsumerInvalidResetError creates a new JSConsumerInvalidResetErr error: "invalid reset: {err}"
func NewJSConsumerInvalidResetError(err error, opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
e := ApiErrors[JSConsumerInvalidResetErr]
args := e.toReplacerArgs([]interface{}{"{err}", err})
return &ApiError{
Code: e.Code,
ErrCode: e.ErrCode,
Description: strings.NewReplacer(args...).Replace(e.Description),
}
}
// NewJSConsumerInvalidSamplingError creates a new JSConsumerInvalidSamplingErrF error: "failed to parse consumer sampling configuration: {err}"
func NewJSConsumerInvalidSamplingError(err error, opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -1967,6 +2177,26 @@ func NewJSMessageSchedulesRollupInvalidError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSMessageSchedulesRollupInvalidErr]
}
// NewJSMessageSchedulesSchedulerInvalidError creates a new JSMessageSchedulesSchedulerInvalidErr error: "message schedules invalid scheduler"
func NewJSMessageSchedulesSchedulerInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSMessageSchedulesSchedulerInvalidErr]
}
// NewJSMessageSchedulesSourceInvalidError creates a new JSMessageSchedulesSourceInvalidErr error: "message schedules source is invalid"
func NewJSMessageSchedulesSourceInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSMessageSchedulesSourceInvalidErr]
}
// NewJSMessageSchedulesTTLInvalidError creates a new JSMessageSchedulesTTLInvalidErr error: "message schedules invalid per-message TTL"
func NewJSMessageSchedulesTTLInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -1987,6 +2217,16 @@ func NewJSMessageSchedulesTargetInvalidError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSMessageSchedulesTargetInvalidErr]
}
// NewJSMessageSchedulesTimeZoneInvalidError creates a new JSMessageSchedulesTimeZoneInvalidErr error: "message schedules time zone is invalid"
func NewJSMessageSchedulesTimeZoneInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSMessageSchedulesTimeZoneInvalidErr]
}
// NewJSMessageTTLDisabledError creates a new JSMessageTTLDisabledErr error: "per-message TTL is disabled"
func NewJSMessageTTLDisabledError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -2007,6 +2247,16 @@ func NewJSMessageTTLInvalidError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSMessageTTLInvalidErr]
}
// NewJSMirrorConsumerRequiresAckFCError creates a new JSMirrorConsumerRequiresAckFCErr error: "stream mirror consumer requires flow control ack policy"
func NewJSMirrorConsumerRequiresAckFCError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSMirrorConsumerRequiresAckFCErr]
}
// NewJSMirrorConsumerSetupFailedError creates a new JSMirrorConsumerSetupFailedErrF error: "{err}"
func NewJSMirrorConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -2023,6 +2273,16 @@ func NewJSMirrorConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiErr
}
}
// NewJSMirrorDurableConsumerCfgInvalidError creates a new JSMirrorDurableConsumerCfgInvalid error: "stream mirror consumer config is invalid"
func NewJSMirrorDurableConsumerCfgInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSMirrorDurableConsumerCfgInvalid]
}
// NewJSMirrorInvalidStreamNameError creates a new JSMirrorInvalidStreamName error: "mirrored stream name is invalid"
func NewJSMirrorInvalidStreamNameError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -2105,6 +2365,16 @@ func NewJSMirrorWithAtomicPublishError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSMirrorWithAtomicPublishErr]
}
// NewJSMirrorWithBatchPublishError creates a new JSMirrorWithBatchPublishErr error: "stream mirrors can not also use batch publishing"
func NewJSMirrorWithBatchPublishError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSMirrorWithBatchPublishErr]
}
// NewJSMirrorWithCountersError creates a new JSMirrorWithCountersErr error: "stream mirrors can not also calculate counters"
func NewJSMirrorWithCountersError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -2339,6 +2609,16 @@ func NewJSSnapshotDeliverSubjectInvalidError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSSnapshotDeliverSubjectInvalidErr]
}
// NewJSSourceConsumerRequiresAckFCError creates a new JSSourceConsumerRequiresAckFCErr error: "stream source consumer requires flow control ack policy"
func NewJSSourceConsumerRequiresAckFCError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSSourceConsumerRequiresAckFCErr]
}
// NewJSSourceConsumerSetupFailedError creates a new JSSourceConsumerSetupFailedErrF error: "{err}"
func NewJSSourceConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
@@ -2365,6 +2645,26 @@ func NewJSSourceDuplicateDetectedError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSSourceDuplicateDetected]
}
// NewJSSourceDurableConsumerCfgInvalidError creates a new JSSourceDurableConsumerCfgInvalid error: "stream source consumer config is invalid"
func NewJSSourceDurableConsumerCfgInvalidError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSSourceDurableConsumerCfgInvalid]
}
// NewJSSourceDurableConsumerDuplicateDetectedError creates a new JSSourceDurableConsumerDuplicateDetected error: "duplicate stream source consumer detected"
func NewJSSourceDurableConsumerDuplicateDetectedError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSSourceDurableConsumerDuplicateDetected]
}
// NewJSSourceInvalidStreamNameError creates a new JSSourceInvalidStreamName error: "sourced stream name is invalid"
func NewJSSourceInvalidStreamNameError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
+7 -7
View File
@@ -71,10 +71,9 @@ const (
// JSStreamActionAdvisory indicates that a stream was created, edited or deleted
type JSStreamActionAdvisory struct {
TypedEvent
Stream string `json:"stream"`
Action ActionAdvisoryType `json:"action"`
Template string `json:"template,omitempty"` // Deprecated: stream templates are deprecated and will be removed in a future version.
Domain string `json:"domain,omitempty"`
Stream string `json:"stream"`
Action ActionAdvisoryType `json:"action"`
Domain string `json:"domain,omitempty"`
}
const JSStreamActionAdvisoryType = "io.nats.jetstream.advisory.v1.stream_action"
@@ -269,9 +268,10 @@ type JSStreamBatchAbandonedAdvisory struct {
type BatchAbandonReason string
var (
BatchTimeout BatchAbandonReason = "timeout"
BatchLarge BatchAbandonReason = "large"
BatchIncomplete BatchAbandonReason = "incomplete"
BatchTimeout BatchAbandonReason = "timeout"
BatchLarge BatchAbandonReason = "large"
BatchIncomplete BatchAbandonReason = "incomplete"
BatchRequirementsNotMet BatchAbandonReason = "unsupported"
)
// JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer.
+11 -1
View File
@@ -17,7 +17,7 @@ import "strconv"
const (
// JSApiLevel is the maximum supported JetStream API level for this server.
JSApiLevel int = 3
JSApiLevel int = 4
JSRequiredLevelMetadataKey = "_nats.req.level"
JSServerVersionMetadataKey = "_nats.ver"
@@ -82,6 +82,11 @@ func setStaticStreamMetadata(cfg *StreamConfig) {
requires(2)
}
// Fast batch publishing was added in v2.14 and requires API level 4.
if cfg.AllowBatchPublish {
requires(4)
}
cfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel)
}
@@ -158,6 +163,11 @@ func setStaticConsumerMetadata(cfg *ConsumerConfig) {
requires(1)
}
// Added in 2.14
if cfg.AckPolicy == AckFlowControl {
requires(4)
}
cfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel)
}
+44 -14
View File
@@ -202,12 +202,15 @@ func validateSrc(claims *jwt.UserClaims, host string) bool {
}
func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {
return validateTimesAt(claims, time.Now())
}
func validateTimesAt(claims *jwt.UserClaims, now time.Time) (bool, time.Duration) {
if claims == nil {
return false, time.Duration(0)
} else if len(claims.Times) == 0 {
return true, time.Duration(0)
}
now := time.Now()
loc := time.Local
if claims.Locale != "" {
var err error
@@ -216,10 +219,11 @@ func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {
}
now = now.In(loc)
}
var ok bool
var validFor time.Duration
for _, timeRange := range claims.Times {
y, m, d := now.Date()
m = m - 1
d = d - 1
start, err := time.ParseInLocation("15:04:05", timeRange.Start, loc)
if err != nil {
return false, time.Duration(0) // parsing not expected to fail at this point
@@ -228,17 +232,43 @@ func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {
if err != nil {
return false, time.Duration(0) // parsing not expected to fail at this point
}
if start.After(end) {
start = start.AddDate(y, int(m), d)
d++ // the intent is to be the next day
} else {
start = start.AddDate(y, int(m), d)
y, m, d := now.Date()
start = time.Date(y, m, d, start.Hour(), start.Minute(), start.Second(), 0, loc)
end = time.Date(y, m, d, end.Hour(), end.Minute(), end.Second(), 0, loc)
inRange, expires := validateTimeRangeAt(start, end, now)
if inRange && (!ok || expires > validFor) {
ok = true
validFor = expires
}
if start.Before(now) {
end = end.AddDate(y, int(m), d)
if end.After(now) {
return true, end.Sub(now)
}
}
return ok, validFor
}
// Returns true if now is within `start` and `end`, and
// how much time is left until `end`.
// False if `now` is not within range.
func validateTimeRangeAt(start, end, now time.Time) (bool, time.Duration) {
// Now falls within range.
// For example 11:00-22:00 at 13:00
if start.Before(now) && end.After(now) {
return true, end.Sub(now)
}
// Range crosses midnight.
if start.After(end) {
// Now is after midnight.
// For example 22:00-06:00 at 05:00.
if end.After(now) {
return true, end.Sub(now)
}
// Now is before midnight.
// For example 22:00-06:00 at 23:30.
end = end.AddDate(0, 0, 1)
if start.Before(now) && end.After(now) {
return true, end.Sub(now)
}
}
return false, time.Duration(0)

Some files were not shown because too many files have changed in this diff Show More