[full-ci] chore: bump reva to v2.44.0 update opencloud version 6.2.0 (#2734)
This commit is contained in:
+2
-2
@@ -14,8 +14,8 @@
|
||||
//
|
||||
// [Antithesis Go SDK]: https://antithesis.com/docs/using_antithesis/sdk/go/
|
||||
// [Antithesis platform]: https://antithesis.com
|
||||
// [test properties]: https://antithesis.com/docs/using_antithesis/properties/
|
||||
// [workload]: https://antithesis.com/docs/getting_started/first_test/
|
||||
// [test properties]: https://antithesis.com/docs/properties_assertions/properties/
|
||||
// [workload]: https://antithesis.com/docs/test_templates/first_test/
|
||||
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
|
||||
// [triage report]: https://antithesis.com/docs/reports/
|
||||
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@ package internal
|
||||
// --------------------------------------------------------------------------------
|
||||
// Versions
|
||||
// --------------------------------------------------------------------------------
|
||||
const SDK_Version = "0.6.0"
|
||||
const SDK_Version = "0.7.0"
|
||||
const Protocol_Version = "1.1.0"
|
||||
|
||||
// --------------------------------------------------------------------------------
|
||||
|
||||
+7
-3
@@ -18,8 +18,8 @@ Alex Snast <alexsn at fb.com>
|
||||
Alexey Palazhchenko <alexey.palazhchenko at gmail.com>
|
||||
Andrew Reid <andrew.reid at tixtrack.com>
|
||||
Animesh Ray <mail.rayanimesh at gmail.com>
|
||||
Arne Hormann <arnehormann at gmail.com>
|
||||
Ariel Mashraki <ariel at mashraki.co.il>
|
||||
Arne Hormann <arnehormann at gmail.com>
|
||||
Artur Melanchyk <artur.melanchyk@gmail.com>
|
||||
Asta Xie <xiemengjun at gmail.com>
|
||||
B Lamarche <blam413 at gmail.com>
|
||||
@@ -38,6 +38,7 @@ Daniel Montoya <dsmontoyam at gmail.com>
|
||||
Daniel Nichter <nil at codenode.com>
|
||||
Daniël van Eeden <git at myname.nl>
|
||||
Dave Protasowski <dprotaso at gmail.com>
|
||||
Demouth <yuya at demouth.net>
|
||||
Diego Dupin <diego.dupin at gmail.com>
|
||||
Dirkjan Bussink <d.bussink at gmail.com>
|
||||
DisposaBoy <disposaboy at dby.me>
|
||||
@@ -66,6 +67,7 @@ Jeff Hodges <jeff at somethingsimilar.com>
|
||||
Jeffrey Charles <jeffreycharles at gmail.com>
|
||||
Jennifer Purevsuren <jennifer at dolthub.com>
|
||||
Jerome Meyer <jxmeyer at gmail.com>
|
||||
Jiabin Zhang <jiabin.z at qq.com>
|
||||
Jiajia Zhong <zhong2plus at gmail.com>
|
||||
Jian Zhen <zhenjl at gmail.com>
|
||||
Joe Mann <contact at joemann.co.uk>
|
||||
@@ -85,10 +87,12 @@ Linh Tran Tuan <linhduonggnu at gmail.com>
|
||||
Lion Yang <lion at aosc.xyz>
|
||||
Luca Looz <luca.looz92 at gmail.com>
|
||||
Lucas Liu <extrafliu at gmail.com>
|
||||
Lunny Xiao <xiaolunwen at gmail.com>
|
||||
Luke Scott <luke at webconnex.com>
|
||||
Lunny Xiao <xiaolunwen at gmail.com>
|
||||
Maciej Zimnoch <maciej.zimnoch at codilime.com>
|
||||
Michael Woolnough <michael.woolnough at gmail.com>
|
||||
Minh Quang <minhquang4334 at gmail.com>
|
||||
Morgan Tocker <tocker at gmail.com>
|
||||
Nao Yokotsuka <yokotukanao at gmail.com>
|
||||
Nathanial Murphy <nathanial.murphy at gmail.com>
|
||||
Nicola Peduzzi <thenikso at gmail.com>
|
||||
@@ -99,7 +103,6 @@ Paul Bonser <misterpib at gmail.com>
|
||||
Paulius Lozys <pauliuslozys at gmail.com>
|
||||
Peter Schultz <peter.schultz at classmarkets.com>
|
||||
Phil Porada <philporada at gmail.com>
|
||||
Minh Quang <minhquang4334 at gmail.com>
|
||||
Rebecca Chin <rchin at pivotal.io>
|
||||
Reed Allman <rdallman10 at gmail.com>
|
||||
Richard Wilkes <wilkes at me.com>
|
||||
@@ -134,6 +137,7 @@ Ziheng Lyu <zihenglv at gmail.com>
|
||||
# Organizations
|
||||
|
||||
Barracuda Networks, Inc.
|
||||
Block, Inc.
|
||||
Counting Ltd.
|
||||
Defined Networking Inc.
|
||||
DigitalOcean Inc.
|
||||
|
||||
+16
-3
@@ -1,13 +1,26 @@
|
||||
# Changelog
|
||||
|
||||
## v1.10.0 (2026-04-28)
|
||||
|
||||
* Fix `getSystemVar("max_allowed_packet")` potentially returned wrong value. (#1754)
|
||||
This affects only when `maxAllowedPacket=0` is set.
|
||||
|
||||
* Bump filippo.io/edwards25519 from 1.1.1 to 1.2.0. (#1756)
|
||||
While older versions have reported CVEs, they do not affect go-mysql.
|
||||
|
||||
* Update Go versions to 1.24-1.26. (#1763)
|
||||
|
||||
* Enhance interpolateParams to correctly handle placeholders. (#1732)
|
||||
The question mark (?) within strings and comments will no longer be treated as a placeholder.
|
||||
|
||||
|
||||
## v1.9.3 (2025-06-13)
|
||||
|
||||
* `tx.Commit()` and `tx.Rollback()` returned `ErrInvalidConn` always.
|
||||
Now they return cached real error if present. (#1690)
|
||||
|
||||
* Optimize reading small resultsets to fix performance regression
|
||||
introduced by compression protocol support. (#1707)
|
||||
|
||||
* Optimize reading small result sets to fix a performance regression
|
||||
introduced by compression protocol support. (`#1707`)
|
||||
* Fix `db.Ping()` on compressed connection. (#1723)
|
||||
|
||||
|
||||
|
||||
+5
-2
@@ -1,5 +1,8 @@
|
||||
# Go-MySQL-Driver
|
||||
|
||||
[](https://deepwiki.com/go-sql-driver/mysql)
|
||||
|
||||
|
||||
A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) package
|
||||
|
||||

|
||||
@@ -42,8 +45,8 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac
|
||||
|
||||
## Requirements
|
||||
|
||||
* Go 1.21 or higher. We aim to support the 3 latest versions of Go.
|
||||
* MySQL (5.7+) and MariaDB (10.5+) are supported.
|
||||
* Go 1.24 or higher. We aim to support the 3 latest versions of Go.
|
||||
* MySQL (5.7+) and MariaDB (10.5+) are supported by maintainers.
|
||||
* [TiDB](https://github.com/pingcap/tidb) is supported by PingCAP.
|
||||
* Do not ask questions about TiDB in our issue tracker or forum.
|
||||
* [Document](https://docs.pingcap.com/tidb/v6.1/dev-guide-sample-application-golang)
|
||||
|
||||
+1
-1
@@ -305,7 +305,7 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) {
|
||||
if !mc.cfg.AllowNativePasswords {
|
||||
return nil, ErrNativePassword
|
||||
}
|
||||
// https://dev.mysql.com/doc/internals/en/secure-password-authentication.html
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/8.4.5/page_protocol_connection_phase_authentication_methods_native_password_authentication.html
|
||||
// Native password authentication only need and will need 20-byte challenge.
|
||||
authResp := scramblePassword(authData[:20], mc.cfg.Passwd)
|
||||
return authResp, nil
|
||||
|
||||
-1
@@ -7,7 +7,6 @@
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd || solaris || illumos
|
||||
// +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos
|
||||
|
||||
package mysql
|
||||
|
||||
|
||||
-1
@@ -7,7 +7,6 @@
|
||||
// You can obtain one at http://mozilla.org/MPL/2.0/.
|
||||
|
||||
//go:build !linux && !darwin && !dragonfly && !freebsd && !netbsd && !openbsd && !solaris && !illumos
|
||||
// +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos
|
||||
|
||||
package mysql
|
||||
|
||||
|
||||
+185
-90
@@ -33,7 +33,8 @@ type mysqlConn struct {
|
||||
connector *connector
|
||||
maxAllowedPacket int
|
||||
maxWriteSize int
|
||||
flags clientFlag
|
||||
capabilities capabilityFlag
|
||||
extCapabilities extendedCapabilityFlag
|
||||
status statusFlag
|
||||
sequence uint8
|
||||
compressSequence uint8
|
||||
@@ -171,7 +172,7 @@ func (mc *mysqlConn) close() {
|
||||
}
|
||||
|
||||
// Closes the network connection and unsets internal variables. Do not call this
|
||||
// function after successfully authentication, call Close instead. This function
|
||||
// function after successful authentication, call Close instead. This function
|
||||
// is called before auth or on auth failure because MySQL will have already
|
||||
// closed the network connection.
|
||||
func (mc *mysqlConn) cleanup() {
|
||||
@@ -223,13 +224,21 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
|
||||
columnCount, err := stmt.readPrepareResultPacket()
|
||||
if err == nil {
|
||||
if stmt.paramCount > 0 {
|
||||
if err = mc.readUntilEOF(); err != nil {
|
||||
if err = mc.skipColumns(stmt.paramCount); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if columnCount > 0 {
|
||||
err = mc.readUntilEOF()
|
||||
if mc.extCapabilities&clientCacheMetadata != 0 {
|
||||
if stmt.columns, err = mc.readColumns(int(columnCount), nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err = mc.skipColumns(int(columnCount)); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -237,100 +246,184 @@ func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) {
|
||||
}
|
||||
|
||||
func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (string, error) {
|
||||
// Number of ? should be same to len(args)
|
||||
if strings.Count(query, "?") != len(args) {
|
||||
return "", driver.ErrSkip
|
||||
}
|
||||
noBackslashEscapes := (mc.status & statusNoBackslashEscapes) != 0
|
||||
const (
|
||||
stateNormal = iota
|
||||
stateString
|
||||
stateEscape
|
||||
stateEOLComment
|
||||
stateSlashStarComment
|
||||
stateBacktick
|
||||
)
|
||||
|
||||
const (
|
||||
QUOTE_BYTE = byte('\'')
|
||||
DBL_QUOTE_BYTE = byte('"')
|
||||
BACKSLASH_BYTE = byte('\\')
|
||||
QUESTION_MARK_BYTE = byte('?')
|
||||
SLASH_BYTE = byte('/')
|
||||
STAR_BYTE = byte('*')
|
||||
HASH_BYTE = byte('#')
|
||||
MINUS_BYTE = byte('-')
|
||||
LINE_FEED_BYTE = byte('\n')
|
||||
BACKTICK_BYTE = byte('`')
|
||||
)
|
||||
|
||||
buf, err := mc.buf.takeCompleteBuffer()
|
||||
if err != nil {
|
||||
// can not take the buffer. Something must be wrong with the connection
|
||||
mc.cleanup()
|
||||
// interpolateParams would be called before sending any query.
|
||||
// So its safe to retry.
|
||||
return "", driver.ErrBadConn
|
||||
}
|
||||
buf = buf[:0]
|
||||
state := stateNormal
|
||||
singleQuotes := false
|
||||
lastChar := byte(0)
|
||||
argPos := 0
|
||||
lenQuery := len(query)
|
||||
lastIdx := 0
|
||||
|
||||
for i := 0; i < len(query); i++ {
|
||||
q := strings.IndexByte(query[i:], '?')
|
||||
if q == -1 {
|
||||
buf = append(buf, query[i:]...)
|
||||
break
|
||||
}
|
||||
buf = append(buf, query[i:i+q]...)
|
||||
i += q
|
||||
|
||||
arg := args[argPos]
|
||||
argPos++
|
||||
|
||||
if arg == nil {
|
||||
buf = append(buf, "NULL"...)
|
||||
for i := range lenQuery {
|
||||
currentChar := query[i]
|
||||
if state == stateEscape && !((currentChar == QUOTE_BYTE && singleQuotes) || (currentChar == DBL_QUOTE_BYTE && !singleQuotes)) {
|
||||
state = stateString
|
||||
lastChar = currentChar
|
||||
continue
|
||||
}
|
||||
|
||||
switch v := arg.(type) {
|
||||
case int64:
|
||||
buf = strconv.AppendInt(buf, v, 10)
|
||||
case uint64:
|
||||
// Handle uint64 explicitly because our custom ConvertValue emits unsigned values
|
||||
buf = strconv.AppendUint(buf, v, 10)
|
||||
case float64:
|
||||
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
|
||||
case bool:
|
||||
if v {
|
||||
buf = append(buf, '1')
|
||||
} else {
|
||||
buf = append(buf, '0')
|
||||
switch currentChar {
|
||||
case STAR_BYTE:
|
||||
if state == stateNormal && lastChar == SLASH_BYTE {
|
||||
state = stateSlashStarComment
|
||||
}
|
||||
case time.Time:
|
||||
if v.IsZero() {
|
||||
buf = append(buf, "'0000-00-00'"...)
|
||||
} else {
|
||||
buf = append(buf, '\'')
|
||||
buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
buf = append(buf, '\'')
|
||||
case SLASH_BYTE:
|
||||
if state == stateSlashStarComment && lastChar == STAR_BYTE {
|
||||
state = stateNormal
|
||||
// Clear lastChar so the '/' that closed the comment isn't
|
||||
// reused to start a new comment with a following '*'.
|
||||
lastChar = 0
|
||||
continue
|
||||
}
|
||||
case json.RawMessage:
|
||||
buf = append(buf, '\'')
|
||||
if mc.status&statusNoBackslashEscapes == 0 {
|
||||
buf = escapeBytesBackslash(buf, v)
|
||||
} else {
|
||||
buf = escapeBytesQuotes(buf, v)
|
||||
case HASH_BYTE:
|
||||
if state == stateNormal {
|
||||
state = stateEOLComment
|
||||
}
|
||||
buf = append(buf, '\'')
|
||||
case []byte:
|
||||
if v == nil {
|
||||
buf = append(buf, "NULL"...)
|
||||
} else {
|
||||
buf = append(buf, "_binary'"...)
|
||||
if mc.status&statusNoBackslashEscapes == 0 {
|
||||
buf = escapeBytesBackslash(buf, v)
|
||||
case MINUS_BYTE:
|
||||
if state == stateNormal && lastChar == MINUS_BYTE {
|
||||
// -- only starts a comment if followed by whitespace or control char
|
||||
if i+1 < lenQuery {
|
||||
nextChar := query[i+1]
|
||||
if nextChar == ' ' || nextChar == '\t' || nextChar == '\n' || nextChar == '\r' {
|
||||
state = stateEOLComment
|
||||
}
|
||||
} else {
|
||||
buf = escapeBytesQuotes(buf, v)
|
||||
state = stateEOLComment
|
||||
}
|
||||
buf = append(buf, '\'')
|
||||
}
|
||||
case string:
|
||||
buf = append(buf, '\'')
|
||||
if mc.status&statusNoBackslashEscapes == 0 {
|
||||
buf = escapeStringBackslash(buf, v)
|
||||
} else {
|
||||
buf = escapeStringQuotes(buf, v)
|
||||
case LINE_FEED_BYTE:
|
||||
if state == stateEOLComment {
|
||||
state = stateNormal
|
||||
}
|
||||
buf = append(buf, '\'')
|
||||
default:
|
||||
return "", driver.ErrSkip
|
||||
}
|
||||
case DBL_QUOTE_BYTE:
|
||||
if state == stateNormal {
|
||||
state = stateString
|
||||
singleQuotes = false
|
||||
} else if state == stateString && !singleQuotes {
|
||||
state = stateNormal
|
||||
} else if state == stateEscape {
|
||||
state = stateString
|
||||
}
|
||||
case QUOTE_BYTE:
|
||||
if state == stateNormal {
|
||||
state = stateString
|
||||
singleQuotes = true
|
||||
} else if state == stateString && singleQuotes {
|
||||
state = stateNormal
|
||||
} else if state == stateEscape {
|
||||
state = stateString
|
||||
}
|
||||
case BACKSLASH_BYTE:
|
||||
if state == stateString && !noBackslashEscapes {
|
||||
state = stateEscape
|
||||
}
|
||||
case QUESTION_MARK_BYTE:
|
||||
if state == stateNormal {
|
||||
if argPos >= len(args) {
|
||||
return "", driver.ErrSkip
|
||||
}
|
||||
buf = append(buf, query[lastIdx:i]...)
|
||||
arg := args[argPos]
|
||||
argPos++
|
||||
|
||||
if len(buf)+4 > mc.maxAllowedPacket {
|
||||
return "", driver.ErrSkip
|
||||
if arg == nil {
|
||||
buf = append(buf, "NULL"...)
|
||||
lastIdx = i + 1
|
||||
break
|
||||
}
|
||||
|
||||
switch v := arg.(type) {
|
||||
case int64:
|
||||
buf = strconv.AppendInt(buf, v, 10)
|
||||
case uint64:
|
||||
buf = strconv.AppendUint(buf, v, 10)
|
||||
case float64:
|
||||
buf = strconv.AppendFloat(buf, v, 'g', -1, 64)
|
||||
case bool:
|
||||
if v {
|
||||
buf = append(buf, '1')
|
||||
} else {
|
||||
buf = append(buf, '0')
|
||||
}
|
||||
case time.Time:
|
||||
if v.IsZero() {
|
||||
buf = append(buf, "'0000-00-00'"...)
|
||||
} else {
|
||||
buf = append(buf, '\'')
|
||||
buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
buf = append(buf, '\'')
|
||||
}
|
||||
case json.RawMessage:
|
||||
if noBackslashEscapes {
|
||||
buf = escapeBytesQuotes(buf, v, false)
|
||||
} else {
|
||||
buf = escapeBytesBackslash(buf, v, false)
|
||||
}
|
||||
case []byte:
|
||||
if v == nil {
|
||||
buf = append(buf, "NULL"...)
|
||||
} else {
|
||||
if noBackslashEscapes {
|
||||
buf = escapeBytesQuotes(buf, v, true)
|
||||
} else {
|
||||
buf = escapeBytesBackslash(buf, v, true)
|
||||
}
|
||||
}
|
||||
case string:
|
||||
if noBackslashEscapes {
|
||||
buf = escapeStringQuotes(buf, v)
|
||||
} else {
|
||||
buf = escapeStringBackslash(buf, v)
|
||||
}
|
||||
default:
|
||||
return "", driver.ErrSkip
|
||||
}
|
||||
|
||||
if len(buf)+4 > mc.maxAllowedPacket {
|
||||
return "", driver.ErrSkip
|
||||
}
|
||||
lastIdx = i + 1
|
||||
}
|
||||
case BACKTICK_BYTE:
|
||||
if state == stateBacktick {
|
||||
state = stateNormal
|
||||
} else if state == stateNormal {
|
||||
state = stateBacktick
|
||||
}
|
||||
}
|
||||
lastChar = currentChar
|
||||
}
|
||||
buf = append(buf, query[lastIdx:]...)
|
||||
if argPos != len(args) {
|
||||
return "", driver.ErrSkip
|
||||
}
|
||||
@@ -370,19 +463,19 @@ func (mc *mysqlConn) exec(query string) error {
|
||||
}
|
||||
|
||||
// Read Result
|
||||
resLen, err := handleOk.readResultSetHeaderPacket()
|
||||
resLen, _, err := handleOk.readResultSetHeaderPacket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if resLen > 0 {
|
||||
// columns
|
||||
if err := mc.readUntilEOF(); err != nil {
|
||||
if err := mc.skipColumns(resLen); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// rows
|
||||
if err := mc.readUntilEOF(); err != nil {
|
||||
if err := mc.skipRows(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -419,7 +512,7 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
|
||||
|
||||
// Read Result
|
||||
var resLen int
|
||||
resLen, err = handleOk.readResultSetHeaderPacket()
|
||||
resLen, _, err = handleOk.readResultSetHeaderPacket()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -439,21 +532,20 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error)
|
||||
}
|
||||
|
||||
// Columns
|
||||
rows.rs.columns, err = mc.readColumns(resLen)
|
||||
rows.rs.columns, err = mc.readColumns(resLen, nil)
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// Gets the value of the given MySQL System Variable
|
||||
// The returned byte slice is only valid until the next read
|
||||
func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
|
||||
func (mc *mysqlConn) getSystemVar(name string) (string, error) {
|
||||
// Send command
|
||||
handleOk := mc.clearResult()
|
||||
if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil {
|
||||
return nil, err
|
||||
return "", err
|
||||
}
|
||||
|
||||
// Read Result
|
||||
resLen, err := handleOk.readResultSetHeaderPacket()
|
||||
resLen, _, err := handleOk.readResultSetHeaderPacket()
|
||||
if err == nil {
|
||||
rows := new(textRows)
|
||||
rows.mc = mc
|
||||
@@ -461,17 +553,20 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) {
|
||||
|
||||
if resLen > 0 {
|
||||
// Columns
|
||||
if err := mc.readUntilEOF(); err != nil {
|
||||
return nil, err
|
||||
if err := mc.skipColumns(resLen); err != nil {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
|
||||
dest := make([]driver.Value, resLen)
|
||||
if err = rows.readRow(dest); err == nil {
|
||||
return dest[0].([]byte), mc.readUntilEOF()
|
||||
// Convert to string before skipRows, which may
|
||||
// overwrite the read buffer that dest[0] points into.
|
||||
val := string(dest[0].([]byte))
|
||||
return val, mc.skipRows()
|
||||
}
|
||||
}
|
||||
return nil, err
|
||||
return "", err
|
||||
}
|
||||
|
||||
// cancel is called when the query has canceled.
|
||||
|
||||
+7
-5
@@ -42,7 +42,7 @@ func encodeConnectionAttributes(cfg *Config) string {
|
||||
}
|
||||
|
||||
// user-defined connection attributes
|
||||
for _, connAttr := range strings.Split(cfg.ConnectionAttributes, ",") {
|
||||
for connAttr := range strings.SplitSeq(cfg.ConnectionAttributes, ",") {
|
||||
k, v, found := strings.Cut(connAttr, ":")
|
||||
if !found {
|
||||
continue
|
||||
@@ -131,7 +131,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
mc.buf = newBuffer()
|
||||
|
||||
// Reading Handshake Initialization Packet
|
||||
authData, plugin, err := mc.readHandshakePacket()
|
||||
authData, serverCapabilities, serverExtCapabilities, plugin, err := mc.readHandshakePacket()
|
||||
if err != nil {
|
||||
mc.cleanup()
|
||||
return nil, err
|
||||
@@ -153,6 +153,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
mc.initCapabilities(serverCapabilities, serverExtCapabilities, mc.cfg)
|
||||
if err = mc.writeHandshakeResponsePacket(authResp, plugin); err != nil {
|
||||
mc.cleanup()
|
||||
return nil, err
|
||||
@@ -161,13 +162,14 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
// Handle response to auth packet, switch methods if possible
|
||||
if err = mc.handleAuthResult(authData, plugin); err != nil {
|
||||
// Authentication failed and MySQL has already closed the connection
|
||||
// (https://dev.mysql.com/doc/internals/en/authentication-fails.html).
|
||||
// (https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase.html#sect_protocol_connection_phase_fast_path_fails).
|
||||
// Do not send COM_QUIT, just cleanup and return the error.
|
||||
mc.cleanup()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if mc.cfg.compress && mc.flags&clientCompress == clientCompress {
|
||||
// compression is enabled after auth, not right after sending handshake response.
|
||||
if mc.capabilities&clientCompress > 0 {
|
||||
mc.compress = true
|
||||
mc.compIO = newCompIO(mc)
|
||||
}
|
||||
@@ -180,7 +182,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) {
|
||||
mc.Close()
|
||||
return nil, err
|
||||
}
|
||||
n, err := strconv.Atoi(string(maxap))
|
||||
n, err := strconv.Atoi(maxap)
|
||||
if err != nil {
|
||||
mc.Close()
|
||||
return nil, fmt.Errorf("invalid max_allowed_packet value (%q): %w", maxap, err)
|
||||
|
||||
+17
-4
@@ -32,7 +32,7 @@ const (
|
||||
)
|
||||
|
||||
// MySQL constants documentation:
|
||||
// http://dev.mysql.com/doc/internals/en/client-server-protocol.html
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_PROTOCOL.html
|
||||
|
||||
const (
|
||||
iOK byte = 0x00
|
||||
@@ -42,11 +42,12 @@ const (
|
||||
iERR byte = 0xff
|
||||
)
|
||||
|
||||
// https://dev.mysql.com/doc/internals/en/capability-flags.html#packet-Protocol::CapabilityFlags
|
||||
type clientFlag uint32
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/group__group__cs__capabilities__flags.html
|
||||
// https://mariadb.com/kb/en/connection/#capabilities
|
||||
type capabilityFlag uint32
|
||||
|
||||
const (
|
||||
clientLongPassword clientFlag = 1 << iota
|
||||
clientMySQL capabilityFlag = 1 << iota
|
||||
clientFoundRows
|
||||
clientLongFlag
|
||||
clientConnectWithDB
|
||||
@@ -73,6 +74,18 @@ const (
|
||||
clientDeprecateEOF
|
||||
)
|
||||
|
||||
// https://mariadb.com/kb/en/connection/#capabilities
|
||||
type extendedCapabilityFlag uint32
|
||||
|
||||
const (
|
||||
progressIndicator extendedCapabilityFlag = 1 << iota
|
||||
clientComMulti
|
||||
clientStmtBulkOperations
|
||||
clientExtendedMetadata
|
||||
clientCacheMetadata
|
||||
clientUnitBulkResult
|
||||
)
|
||||
|
||||
const (
|
||||
comQuit byte = iota + 1
|
||||
comInitDB
|
||||
|
||||
+4
-5
@@ -15,6 +15,7 @@ import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"maps"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/url"
|
||||
@@ -157,9 +158,7 @@ func (cfg *Config) Clone() *Config {
|
||||
}
|
||||
if len(cp.Params) > 0 {
|
||||
cp.Params = make(map[string]string, len(cfg.Params))
|
||||
for k, v := range cfg.Params {
|
||||
cp.Params[k] = v
|
||||
}
|
||||
maps.Copy(cp.Params, cfg.Params)
|
||||
}
|
||||
if cfg.pubKey != nil {
|
||||
cp.pubKey = &rsa.PublicKey{
|
||||
@@ -414,7 +413,7 @@ func ParseDSN(dsn string) (cfg *Config, err error) {
|
||||
if dsn[j] == '@' {
|
||||
// username[:password]
|
||||
// Find the first ':' in dsn[:j]
|
||||
for k = 0; k < j; k++ {
|
||||
for k = 0; k < j; k++ { // We cannot use k = range j here, because we use dsn[:k] below
|
||||
if dsn[k] == ':' {
|
||||
cfg.Passwd = dsn[k+1 : j]
|
||||
break
|
||||
@@ -477,7 +476,7 @@ func ParseDSN(dsn string) (cfg *Config, err error) {
|
||||
// parseDSNParams parses the DSN "query string"
|
||||
// Values must be url.QueryEscape'ed
|
||||
func parseDSNParams(cfg *Config, params string) (err error) {
|
||||
for _, v := range strings.Split(params, "&") {
|
||||
for v := range strings.SplitSeq(params, "&") {
|
||||
key, value, found := strings.Cut(v, "=")
|
||||
if !found {
|
||||
continue
|
||||
|
||||
+21
-17
@@ -120,23 +120,24 @@ func (mf *mysqlField) typeDatabaseName() string {
|
||||
}
|
||||
|
||||
var (
|
||||
scanTypeFloat32 = reflect.TypeOf(float32(0))
|
||||
scanTypeFloat64 = reflect.TypeOf(float64(0))
|
||||
scanTypeInt8 = reflect.TypeOf(int8(0))
|
||||
scanTypeInt16 = reflect.TypeOf(int16(0))
|
||||
scanTypeInt32 = reflect.TypeOf(int32(0))
|
||||
scanTypeInt64 = reflect.TypeOf(int64(0))
|
||||
scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{})
|
||||
scanTypeNullInt = reflect.TypeOf(sql.NullInt64{})
|
||||
scanTypeNullTime = reflect.TypeOf(sql.NullTime{})
|
||||
scanTypeUint8 = reflect.TypeOf(uint8(0))
|
||||
scanTypeUint16 = reflect.TypeOf(uint16(0))
|
||||
scanTypeUint32 = reflect.TypeOf(uint32(0))
|
||||
scanTypeUint64 = reflect.TypeOf(uint64(0))
|
||||
scanTypeString = reflect.TypeOf("")
|
||||
scanTypeNullString = reflect.TypeOf(sql.NullString{})
|
||||
scanTypeBytes = reflect.TypeOf([]byte{})
|
||||
scanTypeUnknown = reflect.TypeOf(new(any))
|
||||
scanTypeFloat32 = reflect.TypeFor[float32]()
|
||||
scanTypeFloat64 = reflect.TypeFor[float64]()
|
||||
scanTypeInt8 = reflect.TypeFor[int8]()
|
||||
scanTypeInt16 = reflect.TypeFor[int16]()
|
||||
scanTypeInt32 = reflect.TypeFor[int32]()
|
||||
scanTypeInt64 = reflect.TypeFor[int64]()
|
||||
scanTypeNullFloat = reflect.TypeFor[sql.NullFloat64]()
|
||||
scanTypeNullInt = reflect.TypeFor[sql.NullInt64]()
|
||||
scanTypeNullUint = reflect.TypeFor[sql.Null[uint64]]()
|
||||
scanTypeNullTime = reflect.TypeFor[sql.NullTime]()
|
||||
scanTypeUint8 = reflect.TypeFor[uint8]()
|
||||
scanTypeUint16 = reflect.TypeFor[uint16]()
|
||||
scanTypeUint32 = reflect.TypeFor[uint32]()
|
||||
scanTypeUint64 = reflect.TypeFor[uint64]()
|
||||
scanTypeString = reflect.TypeFor[string]()
|
||||
scanTypeNullString = reflect.TypeFor[sql.NullString]()
|
||||
scanTypeBytes = reflect.TypeFor[[]byte]()
|
||||
scanTypeUnknown = reflect.TypeFor[*any]()
|
||||
)
|
||||
|
||||
type mysqlField struct {
|
||||
@@ -185,6 +186,9 @@ func (mf *mysqlField) scanType() reflect.Type {
|
||||
}
|
||||
return scanTypeInt64
|
||||
}
|
||||
if mf.flags&flagUnsigned != 0 {
|
||||
return scanTypeNullUint
|
||||
}
|
||||
return scanTypeNullInt
|
||||
|
||||
case fieldTypeFloat:
|
||||
|
||||
+1
-4
@@ -95,10 +95,7 @@ const defaultPacketSize = 16 * 1024 // 16KB is small enough for disk readahead a
|
||||
|
||||
func (mc *okHandler) handleInFileRequest(name string) (err error) {
|
||||
var rdr io.Reader
|
||||
packetSize := defaultPacketSize
|
||||
if mc.maxWriteSize < packetSize {
|
||||
packetSize = mc.maxWriteSize
|
||||
}
|
||||
packetSize := min(mc.maxWriteSize, defaultPacketSize)
|
||||
|
||||
if idx := strings.Index(name, "Reader::"); idx == 0 || (idx > 0 && name[idx-1] == '/') { // io.Reader
|
||||
// The server might return an an absolute path. See issue #355.
|
||||
|
||||
+205
-138
@@ -179,20 +179,22 @@ func (mc *mysqlConn) writePacket(data []byte) error {
|
||||
******************************************************************************/
|
||||
|
||||
// Handshake Initialization Packet
|
||||
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake
|
||||
func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) {
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_v10.html
|
||||
// https://mariadb.com/kb/en/connection/#initial-handshake-packet
|
||||
func (mc *mysqlConn) readHandshakePacket() (data []byte, capabilities capabilityFlag, extendedCapabilities extendedCapabilityFlag, plugin string, err error) {
|
||||
data, err = mc.readPacket()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if data[0] == iERR {
|
||||
return nil, "", mc.handleErrorPacket(data)
|
||||
err = mc.handleErrorPacket(data)
|
||||
return
|
||||
}
|
||||
|
||||
// protocol version [1 byte]
|
||||
if data[0] < minProtocolVersion {
|
||||
return nil, "", fmt.Errorf(
|
||||
return nil, 0, 0, "", fmt.Errorf(
|
||||
"unsupported protocol version %d. Version %d or higher is required",
|
||||
data[0],
|
||||
minProtocolVersion,
|
||||
@@ -210,15 +212,15 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
|
||||
pos += 8 + 1
|
||||
|
||||
// capability flags (lower 2 bytes) [2 bytes]
|
||||
mc.flags = clientFlag(binary.LittleEndian.Uint16(data[pos : pos+2]))
|
||||
if mc.flags&clientProtocol41 == 0 {
|
||||
return nil, "", ErrOldProtocol
|
||||
capabilities = capabilityFlag(binary.LittleEndian.Uint16(data[pos : pos+2]))
|
||||
if capabilities&clientProtocol41 == 0 {
|
||||
return nil, capabilities, 0, "", ErrOldProtocol
|
||||
}
|
||||
if mc.flags&clientSSL == 0 && mc.cfg.TLS != nil {
|
||||
if capabilities&clientSSL == 0 && mc.cfg.TLS != nil {
|
||||
if mc.cfg.AllowFallbackToPlaintext {
|
||||
mc.cfg.TLS = nil
|
||||
} else {
|
||||
return nil, "", ErrNoTLS
|
||||
return nil, capabilities, 0, "", ErrNoTLS
|
||||
}
|
||||
}
|
||||
pos += 2
|
||||
@@ -228,11 +230,16 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
|
||||
// status flags [2 bytes]
|
||||
pos += 3
|
||||
// capability flags (upper 2 bytes) [2 bytes]
|
||||
mc.flags |= clientFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16
|
||||
capabilities |= capabilityFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16
|
||||
pos += 2
|
||||
// length of auth-plugin-data [1 byte]
|
||||
// reserved (all [00]) [10 bytes]
|
||||
pos += 11
|
||||
// reserved (all [00]) [6 bytes]
|
||||
pos += 7
|
||||
if capabilities&clientMySQL == 0 {
|
||||
// MariaDB server extended flag
|
||||
extendedCapabilities = extendedCapabilityFlag(binary.LittleEndian.Uint32(data[pos : pos+4]))
|
||||
}
|
||||
pos += 4
|
||||
|
||||
// second part of the password cipher [minimum 13 bytes],
|
||||
// where len=MAX(13, length of auth-plugin-data - 8)
|
||||
@@ -260,82 +267,72 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro
|
||||
// make a memory safe copy of the cipher slice
|
||||
var b [20]byte
|
||||
copy(b[:], authData)
|
||||
return b[:], plugin, nil
|
||||
return b[:], capabilities, extendedCapabilities, plugin, nil
|
||||
}
|
||||
|
||||
// make a memory safe copy of the cipher slice
|
||||
var b [8]byte
|
||||
copy(b[:], authData)
|
||||
return b[:], plugin, nil
|
||||
return b[:], capabilities, 0, plugin, nil
|
||||
}
|
||||
|
||||
// Client Authentication Packet
|
||||
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::HandshakeResponse
|
||||
func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error {
|
||||
// Adjust client flags based on server support
|
||||
clientFlags := clientProtocol41 |
|
||||
clientSecureConn |
|
||||
clientLongPassword |
|
||||
clientTransactions |
|
||||
clientLocalFiles |
|
||||
clientPluginAuth |
|
||||
clientMultiResults |
|
||||
mc.flags&clientConnectAttrs |
|
||||
mc.flags&clientLongFlag
|
||||
// initCapabilities initializes the capabilities based on server support and configuration
|
||||
func (mc *mysqlConn) initCapabilities(serverCapabilities capabilityFlag, serverExtCapabilities extendedCapabilityFlag, cfg *Config) {
|
||||
clientCapabilities :=
|
||||
clientMySQL |
|
||||
clientLongFlag |
|
||||
clientProtocol41 |
|
||||
clientSecureConn |
|
||||
clientTransactions |
|
||||
clientPluginAuthLenEncClientData |
|
||||
clientLocalFiles |
|
||||
clientPluginAuth |
|
||||
clientMultiResults |
|
||||
clientConnectAttrs |
|
||||
clientDeprecateEOF
|
||||
|
||||
sendConnectAttrs := mc.flags&clientConnectAttrs != 0
|
||||
|
||||
if mc.cfg.ClientFoundRows {
|
||||
clientFlags |= clientFoundRows
|
||||
if cfg.ClientFoundRows {
|
||||
clientCapabilities |= clientFoundRows
|
||||
}
|
||||
if mc.cfg.compress && mc.flags&clientCompress == clientCompress {
|
||||
clientFlags |= clientCompress
|
||||
if cfg.compress {
|
||||
clientCapabilities |= clientCompress
|
||||
}
|
||||
// To enable TLS / SSL
|
||||
if mc.cfg.TLS != nil {
|
||||
clientFlags |= clientSSL
|
||||
clientCapabilities |= clientSSL
|
||||
}
|
||||
|
||||
if mc.cfg.MultiStatements {
|
||||
clientFlags |= clientMultiStatements
|
||||
clientCapabilities |= clientMultiStatements
|
||||
}
|
||||
if n := len(cfg.DBName); n > 0 {
|
||||
clientCapabilities |= clientConnectWithDB
|
||||
}
|
||||
|
||||
// encode length of the auth plugin data
|
||||
var authRespLEIBuf [9]byte
|
||||
authRespLen := len(authResp)
|
||||
authRespLEI := appendLengthEncodedInteger(authRespLEIBuf[:0], uint64(authRespLen))
|
||||
if len(authRespLEI) > 1 {
|
||||
// if the length can not be written in 1 byte, it must be written as a
|
||||
// length encoded integer
|
||||
clientFlags |= clientPluginAuthLenEncClientData
|
||||
}
|
||||
// only keep client capabilities that server have
|
||||
mc.capabilities = clientCapabilities & serverCapabilities
|
||||
|
||||
pktLen := 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + len(authRespLEI) + len(authResp) + 21 + 1
|
||||
// set MariaDB extended clientCacheMetadata capability if server support it
|
||||
mc.extCapabilities = clientCacheMetadata & serverExtCapabilities
|
||||
}
|
||||
|
||||
// To specify a db name
|
||||
if n := len(mc.cfg.DBName); n > 0 {
|
||||
clientFlags |= clientConnectWithDB
|
||||
pktLen += n + 1
|
||||
}
|
||||
|
||||
// encode length of the connection attributes
|
||||
var connAttrsLEI []byte
|
||||
if sendConnectAttrs {
|
||||
var connAttrsLEIBuf [9]byte
|
||||
connAttrsLen := len(mc.connector.encodedAttributes)
|
||||
connAttrsLEI = appendLengthEncodedInteger(connAttrsLEIBuf[:0], uint64(connAttrsLen))
|
||||
pktLen += len(connAttrsLEI) + len(mc.connector.encodedAttributes)
|
||||
}
|
||||
|
||||
// Calculate packet length and get buffer with that size
|
||||
data, err := mc.buf.takeBuffer(pktLen + 4)
|
||||
// Client Authentication Packet
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_handshake_response.html
|
||||
func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string) error {
|
||||
// packet header 4
|
||||
// capabilities 4
|
||||
// maxPacketSize 4
|
||||
// collation id 1
|
||||
// filler 23
|
||||
data, err := mc.buf.takeSmallBuffer(4*3 + 24)
|
||||
if err != nil {
|
||||
mc.cleanup()
|
||||
return err
|
||||
}
|
||||
_ = data[4*3+23] // boundery check
|
||||
|
||||
// ClientFlags [32 bit]
|
||||
binary.LittleEndian.PutUint32(data[4:], uint32(clientFlags))
|
||||
// clientCapabilities [32 bit]
|
||||
binary.LittleEndian.PutUint32(data[4:], uint32(mc.capabilities))
|
||||
|
||||
// MaxPacketSize [32 bit] (none)
|
||||
binary.LittleEndian.PutUint32(data[8:], 0)
|
||||
@@ -353,16 +350,26 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
|
||||
}
|
||||
|
||||
// Filler [23 bytes] (all 0x00)
|
||||
// or filler 19bytes + mariadb extCapabilities
|
||||
pos := 13
|
||||
for ; pos < 13+23; pos++ {
|
||||
data[pos] = 0
|
||||
if mc.capabilities&clientMySQL == 0 {
|
||||
for ; pos < 13+19; pos++ {
|
||||
data[pos] = 0
|
||||
}
|
||||
// MariaDB Extended Capabilities
|
||||
binary.LittleEndian.PutUint32(data[13+19:], uint32(mc.extCapabilities))
|
||||
} else {
|
||||
for ; pos < 13+23; pos++ {
|
||||
data[pos] = 0
|
||||
}
|
||||
}
|
||||
|
||||
// SSL Connection Request Packet
|
||||
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_ssl_request.html
|
||||
// https://mariadb.com/kb/en/connection/#sslrequest-packet
|
||||
if mc.cfg.TLS != nil {
|
||||
// Send TLS / SSL request packet
|
||||
if err := mc.writePacket(data[:(4+4+1+23)+4]); err != nil {
|
||||
if err := mc.writePacket(data); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -379,37 +386,35 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string
|
||||
|
||||
// User [null terminated string]
|
||||
if len(mc.cfg.User) > 0 {
|
||||
pos += copy(data[pos:], mc.cfg.User)
|
||||
data = append(data, mc.cfg.User...)
|
||||
}
|
||||
data[pos] = 0x00
|
||||
pos++
|
||||
data = append(data, 0)
|
||||
|
||||
// Auth Data [length encoded integer]
|
||||
pos += copy(data[pos:], authRespLEI)
|
||||
pos += copy(data[pos:], authResp)
|
||||
data = appendLengthEncodedInteger(data, uint64(len(authResp)))
|
||||
data = append(data, authResp...)
|
||||
|
||||
// Databasename [null terminated string]
|
||||
if len(mc.cfg.DBName) > 0 {
|
||||
pos += copy(data[pos:], mc.cfg.DBName)
|
||||
data[pos] = 0x00
|
||||
pos++
|
||||
// Database name [null terminated string]
|
||||
if mc.capabilities&clientConnectWithDB != 0 {
|
||||
data = append(data, mc.cfg.DBName...)
|
||||
data = append(data, 0)
|
||||
}
|
||||
|
||||
pos += copy(data[pos:], plugin)
|
||||
data[pos] = 0x00
|
||||
pos++
|
||||
data = append(data, plugin...)
|
||||
data = append(data, 0)
|
||||
|
||||
// Connection Attributes
|
||||
if sendConnectAttrs {
|
||||
pos += copy(data[pos:], connAttrsLEI)
|
||||
pos += copy(data[pos:], []byte(mc.connector.encodedAttributes))
|
||||
if mc.capabilities&clientConnectAttrs != 0 {
|
||||
connAttrsLen := len(mc.connector.encodedAttributes)
|
||||
data = appendLengthEncodedInteger(data, uint64(connAttrsLen))
|
||||
data = append(data, mc.connector.encodedAttributes...)
|
||||
}
|
||||
|
||||
// Send Auth packet
|
||||
return mc.writePacket(data[:pos])
|
||||
return mc.writePacket(data)
|
||||
}
|
||||
|
||||
// http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_auth_switch_response.html
|
||||
func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error {
|
||||
pktLen := 4 + len(authData)
|
||||
data, err := mc.buf.takeBuffer(pktLen)
|
||||
@@ -511,7 +516,7 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) {
|
||||
|
||||
case iEOF:
|
||||
if len(data) == 1 {
|
||||
// https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::OldAuthSwitchRequest
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_connection_phase_packets_protocol_old_auth_switch_request.html
|
||||
return nil, "mysql_old_password", nil
|
||||
}
|
||||
pluginEndIndex := bytes.IndexByte(data, 0x00)
|
||||
@@ -545,36 +550,41 @@ func (mc *okHandler) readResultOK() error {
|
||||
|
||||
// Result Set Header Packet
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response.html
|
||||
func (mc *okHandler) readResultSetHeaderPacket() (int, error) {
|
||||
func (mc *okHandler) readResultSetHeaderPacket() (int, bool, error) {
|
||||
// handleOkPacket replaces both values; other cases leave the values unchanged.
|
||||
mc.result.affectedRows = append(mc.result.affectedRows, 0)
|
||||
mc.result.insertIds = append(mc.result.insertIds, 0)
|
||||
|
||||
data, err := mc.conn().readPacket()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
return 0, false, err
|
||||
}
|
||||
|
||||
switch data[0] {
|
||||
case iOK:
|
||||
return 0, mc.handleOkPacket(data)
|
||||
return 0, false, mc.handleOkPacket(data)
|
||||
|
||||
case iERR:
|
||||
return 0, mc.conn().handleErrorPacket(data)
|
||||
return 0, false, mc.conn().handleErrorPacket(data)
|
||||
|
||||
case iLocalInFile:
|
||||
return 0, mc.handleInFileRequest(string(data[1:]))
|
||||
return 0, false, mc.handleInFileRequest(string(data[1:]))
|
||||
}
|
||||
|
||||
// column count
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html
|
||||
num, _, _ := readLengthEncodedInteger(data)
|
||||
// https://mariadb.com/kb/en/result-set-packets/#column-count-packet
|
||||
num, _, len := readLengthEncodedInteger(data)
|
||||
|
||||
if mc.extCapabilities&clientCacheMetadata != 0 {
|
||||
return int(num), data[len] == 0x01, nil
|
||||
}
|
||||
// ignore remaining data in the packet. see #1478.
|
||||
return int(num), nil
|
||||
return int(num), true, nil
|
||||
}
|
||||
|
||||
// Error Packet
|
||||
// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-ERR_Packet
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_err_packet.html
|
||||
func (mc *mysqlConn) handleErrorPacket(data []byte) error {
|
||||
if data[0] != iERR {
|
||||
return ErrMalformPkt
|
||||
@@ -656,7 +666,7 @@ func (mc *mysqlConn) clearResult() *okHandler {
|
||||
}
|
||||
|
||||
// Ok Packet
|
||||
// http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_ok_packet.html
|
||||
func (mc *okHandler) handleOkPacket(data []byte) error {
|
||||
var n, m int
|
||||
var affectedRows, insertId uint64
|
||||
@@ -690,24 +700,19 @@ func (mc *okHandler) handleOkPacket(data []byte) error {
|
||||
}
|
||||
|
||||
// Read Packets as Field Packets until EOF-Packet or an Error appears
|
||||
// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-Protocol::ColumnDefinition41
|
||||
func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_column_definition.html#sect_protocol_com_query_response_text_resultset_column_definition_41
|
||||
func (mc *mysqlConn) readColumns(count int, old []mysqlField) ([]mysqlField, error) {
|
||||
columns := make([]mysqlField, count)
|
||||
if len(old) != count {
|
||||
old = nil
|
||||
}
|
||||
|
||||
for i := 0; ; i++ {
|
||||
for i := range count {
|
||||
data, err := mc.readPacket()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// EOF Packet
|
||||
if data[0] == iEOF && (len(data) == 5 || len(data) == 1) {
|
||||
if i == count {
|
||||
return columns, nil
|
||||
}
|
||||
return nil, fmt.Errorf("column count mismatch n:%d len:%d", count, len(columns))
|
||||
}
|
||||
|
||||
// Catalog
|
||||
pos, err := skipLengthEncodedString(data)
|
||||
if err != nil {
|
||||
@@ -728,7 +733,12 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
|
||||
return nil, err
|
||||
}
|
||||
pos += n
|
||||
columns[i].tableName = string(tableName)
|
||||
if old != nil && old[i].tableName == string(tableName) {
|
||||
// avoid allocating new string
|
||||
columns[i].tableName = old[i].tableName
|
||||
} else {
|
||||
columns[i].tableName = string(tableName)
|
||||
}
|
||||
} else {
|
||||
n, err = skipLengthEncodedString(data[pos:])
|
||||
if err != nil {
|
||||
@@ -749,7 +759,12 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
columns[i].name = string(name)
|
||||
if old != nil && old[i].name == string(name) {
|
||||
// avoid allocating new string
|
||||
columns[i].name = old[i].name
|
||||
} else {
|
||||
columns[i].name = string(name)
|
||||
}
|
||||
pos += n
|
||||
|
||||
// Original name [len coded string]
|
||||
@@ -780,17 +795,17 @@ func (mc *mysqlConn) readColumns(count int) ([]mysqlField, error) {
|
||||
|
||||
// Decimals [uint8]
|
||||
columns[i].decimals = data[pos]
|
||||
//pos++
|
||||
|
||||
// Default value [len coded binary]
|
||||
//if pos < len(data) {
|
||||
// defaultVal, _, err = bytesToLengthCodedBinary(data[pos:])
|
||||
//}
|
||||
}
|
||||
|
||||
// skip EOF packet if client does not support deprecateEOF
|
||||
if err := mc.skipEof(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return columns, nil
|
||||
}
|
||||
|
||||
// Read Packets as Field Packets until EOF-Packet or an Error appears
|
||||
// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::ResultsetRow
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset_row.html
|
||||
func (rows *textRows) readRow(dest []driver.Value) error {
|
||||
mc := rows.mc
|
||||
|
||||
@@ -804,9 +819,20 @@ func (rows *textRows) readRow(dest []driver.Value) error {
|
||||
}
|
||||
|
||||
// EOF Packet
|
||||
if data[0] == iEOF && len(data) == 5 {
|
||||
// server_status [2 bytes]
|
||||
rows.mc.status = readStatus(data[3:])
|
||||
// text row packets may starts with LengthEncodedString.
|
||||
// In such case, 0xFE can mean string larger than 0xffffff.
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_dt_integers.html#sect_protocol_basic_dt_int_le
|
||||
if data[0] == iEOF && len(data) <= 0xffffff {
|
||||
if mc.capabilities&clientDeprecateEOF == 0 {
|
||||
// Deprecated EOF packet
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_basic_eof_packet.html
|
||||
mc.status = readStatus(data[3:])
|
||||
} else {
|
||||
// Ok Packet with an 0xFE header
|
||||
_, _, n := readLengthEncodedInteger(data[1:]) // affected_rows
|
||||
_, _, m := readLengthEncodedInteger(data[1+n:]) // last_insert_id
|
||||
mc.status = readStatus(data[1+n+m:])
|
||||
}
|
||||
rows.rs.done = true
|
||||
if !rows.HasNextResultSet() {
|
||||
rows.mc = nil
|
||||
@@ -880,8 +906,34 @@ func (rows *textRows) readRow(dest []driver.Value) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// Reads Packets until EOF-Packet or an Error appears. Returns count of Packets read
|
||||
func (mc *mysqlConn) readUntilEOF() error {
|
||||
func (mc *mysqlConn) skipPackets(n int) error {
|
||||
for range n {
|
||||
if _, err := mc.readPacket(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// skips EOF packet after n * ColumnDefinition packets when clientDeprecateEOF is not set
|
||||
func (mc *mysqlConn) skipEof() error {
|
||||
if mc.capabilities&clientDeprecateEOF == 0 {
|
||||
if _, err := mc.readPacket(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (mc *mysqlConn) skipColumns(n int) error {
|
||||
if err := mc.skipPackets(n); err != nil {
|
||||
return err
|
||||
}
|
||||
return mc.skipEof()
|
||||
}
|
||||
|
||||
// Reads Packets until EOF-Packet or an Error appears.
|
||||
func (mc *mysqlConn) skipRows() error {
|
||||
for {
|
||||
data, err := mc.readPacket()
|
||||
if err != nil {
|
||||
@@ -892,10 +944,20 @@ func (mc *mysqlConn) readUntilEOF() error {
|
||||
case iERR:
|
||||
return mc.handleErrorPacket(data)
|
||||
case iEOF:
|
||||
if len(data) == 5 {
|
||||
mc.status = readStatus(data[3:])
|
||||
// text row packets may starts with LengthEncodedString.
|
||||
// In such case, 0xFE can mean string larger than 0xffffff.
|
||||
if len(data) <= 0xffffff {
|
||||
if mc.capabilities&clientDeprecateEOF == 0 {
|
||||
// EOF packet
|
||||
mc.status = readStatus(data[3:])
|
||||
} else {
|
||||
// OK packet with an 0xFE header
|
||||
_, _, n := readLengthEncodedInteger(data[1:]) // affected_rows
|
||||
_, _, m := readLengthEncodedInteger(data[1+n:]) // last_insert_id
|
||||
mc.status = readStatus(data[1+n+m:])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -905,7 +967,7 @@ func (mc *mysqlConn) readUntilEOF() error {
|
||||
******************************************************************************/
|
||||
|
||||
// Prepare Result Packets
|
||||
// http://dev.mysql.com/doc/internals/en/com-stmt-prepare-response.html
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_prepare.html#sect_protocol_com_stmt_prepare_response
|
||||
func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) {
|
||||
data, err := stmt.mc.readPacket()
|
||||
if err == nil {
|
||||
@@ -932,7 +994,7 @@ func (stmt *mysqlStmt) readPrepareResultPacket() (uint16, error) {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
// http://dev.mysql.com/doc/internals/en/com-stmt-send-long-data.html
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_send_long_data.html
|
||||
func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error {
|
||||
maxLen := stmt.mc.maxAllowedPacket - 1
|
||||
pktLen := maxLen
|
||||
@@ -979,7 +1041,7 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error {
|
||||
}
|
||||
|
||||
// Execute Prepared Statement
|
||||
// http://dev.mysql.com/doc/internals/en/com-stmt-execute.html
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_stmt_execute.html
|
||||
func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
|
||||
if len(args) != stmt.paramCount {
|
||||
return fmt.Errorf(
|
||||
@@ -993,10 +1055,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
|
||||
mc := stmt.mc
|
||||
|
||||
// Determine threshold dynamically to avoid packet size shortage.
|
||||
longDataSize := mc.maxAllowedPacket / (stmt.paramCount + 1)
|
||||
if longDataSize < 64 {
|
||||
longDataSize = 64
|
||||
}
|
||||
longDataSize := max(mc.maxAllowedPacket/(stmt.paramCount+1), 64)
|
||||
|
||||
// Reset packet-sequence
|
||||
mc.resetSequence()
|
||||
@@ -1185,17 +1244,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error {
|
||||
// mc.affectedRows and mc.insertIds.
|
||||
func (mc *okHandler) discardResults() error {
|
||||
for mc.status&statusMoreResultsExists != 0 {
|
||||
resLen, err := mc.readResultSetHeaderPacket()
|
||||
resLen, _, err := mc.readResultSetHeaderPacket()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resLen > 0 {
|
||||
// columns
|
||||
if err := mc.conn().readUntilEOF(); err != nil {
|
||||
if err := mc.conn().skipColumns(resLen); err != nil {
|
||||
return err
|
||||
}
|
||||
// rows
|
||||
if err := mc.conn().readUntilEOF(); err != nil {
|
||||
if err := mc.conn().skipRows(); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -1203,7 +1262,7 @@ func (mc *okHandler) discardResults() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// http://dev.mysql.com/doc/internals/en/binary-protocol-resultset-row.html
|
||||
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row
|
||||
func (rows *binaryRows) readRow(dest []driver.Value) error {
|
||||
data, err := rows.mc.readPacket()
|
||||
if err != nil {
|
||||
@@ -1212,9 +1271,17 @@ func (rows *binaryRows) readRow(dest []driver.Value) error {
|
||||
|
||||
// packet indicator [1 byte]
|
||||
if data[0] != iOK {
|
||||
// EOF Packet
|
||||
if data[0] == iEOF && len(data) == 5 {
|
||||
rows.mc.status = readStatus(data[3:])
|
||||
// EOF/OK Packet
|
||||
if data[0] == iEOF {
|
||||
if rows.mc.capabilities&clientDeprecateEOF == 0 {
|
||||
// EOF packet
|
||||
rows.mc.status = readStatus(data[3:])
|
||||
} else {
|
||||
// OK Packet with an 0xFE header
|
||||
_, _, n := readLengthEncodedInteger(data[1:])
|
||||
_, _, m := readLengthEncodedInteger(data[1+n:])
|
||||
rows.mc.status = readStatus(data[1+n+m:])
|
||||
}
|
||||
rows.rs.done = true
|
||||
if !rows.HasNextResultSet() {
|
||||
rows.mc = nil
|
||||
|
||||
+4
-2
@@ -8,6 +8,8 @@
|
||||
|
||||
package mysql
|
||||
|
||||
import "slices"
|
||||
|
||||
import "database/sql/driver"
|
||||
|
||||
// Result exposes data not available through *connection.Result.
|
||||
@@ -42,9 +44,9 @@ func (res *mysqlResult) RowsAffected() (int64, error) {
|
||||
}
|
||||
|
||||
func (res *mysqlResult) AllLastInsertIds() []int64 {
|
||||
return append([]int64{}, res.insertIds...) // defensive copy
|
||||
return slices.Clone(res.insertIds) // defensive copy
|
||||
}
|
||||
|
||||
func (res *mysqlResult) AllRowsAffected() []int64 {
|
||||
return append([]int64{}, res.affectedRows...) // defensive copy
|
||||
return slices.Clone(res.affectedRows) // defensive copy
|
||||
}
|
||||
|
||||
+5
-5
@@ -113,7 +113,7 @@ func (rows *mysqlRows) Close() (err error) {
|
||||
|
||||
// Remove unread packets from stream
|
||||
if !rows.rs.done {
|
||||
err = mc.readUntilEOF()
|
||||
err = mc.skipRows()
|
||||
}
|
||||
if err == nil {
|
||||
handleOk := mc.clearResult()
|
||||
@@ -143,7 +143,7 @@ func (rows *mysqlRows) nextResultSet() (int, error) {
|
||||
|
||||
// Remove unread packets from stream
|
||||
if !rows.rs.done {
|
||||
if err := rows.mc.readUntilEOF(); err != nil {
|
||||
if err := rows.mc.skipRows(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
rows.rs.done = true
|
||||
@@ -156,7 +156,7 @@ func (rows *mysqlRows) nextResultSet() (int, error) {
|
||||
rows.rs = resultSet{}
|
||||
// rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to
|
||||
// nextResultSet.
|
||||
resLen, err := rows.mc.resultUnchanged().readResultSetHeaderPacket()
|
||||
resLen, _, err := rows.mc.resultUnchanged().readResultSetHeaderPacket()
|
||||
if err != nil {
|
||||
// Clean up about multi-results flag
|
||||
rows.rs.done = true
|
||||
@@ -186,7 +186,7 @@ func (rows *binaryRows) NextResultSet() error {
|
||||
return err
|
||||
}
|
||||
|
||||
rows.rs.columns, err = rows.mc.readColumns(resLen)
|
||||
rows.rs.columns, err = rows.mc.readColumns(resLen, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -208,7 +208,7 @@ func (rows *textRows) NextResultSet() (err error) {
|
||||
return err
|
||||
}
|
||||
|
||||
rows.rs.columns, err = rows.mc.readColumns(resLen)
|
||||
rows.rs.columns, err = rows.mc.readColumns(resLen, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
+26
-8
@@ -20,6 +20,7 @@ type mysqlStmt struct {
|
||||
mc *mysqlConn
|
||||
id uint32
|
||||
paramCount int
|
||||
columns []mysqlField
|
||||
}
|
||||
|
||||
func (stmt *mysqlStmt) Close() error {
|
||||
@@ -64,19 +65,26 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) {
|
||||
handleOk := stmt.mc.clearResult()
|
||||
|
||||
// Read Result
|
||||
resLen, err := handleOk.readResultSetHeaderPacket()
|
||||
resLen, metadataFollows, err := handleOk.readResultSetHeaderPacket()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resLen > 0 {
|
||||
// Columns
|
||||
if err = mc.readUntilEOF(); err != nil {
|
||||
return nil, err
|
||||
if metadataFollows && stmt.mc.extCapabilities&clientCacheMetadata != 0 {
|
||||
// we can not skip column metadata because next stmt.Query() may use it.
|
||||
if stmt.columns, err = mc.readColumns(resLen, stmt.columns); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
} else {
|
||||
if err = mc.skipColumns(resLen); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
// Rows
|
||||
if err := mc.readUntilEOF(); err != nil {
|
||||
if err = mc.skipRows(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
@@ -107,7 +115,7 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
|
||||
|
||||
// Read Result
|
||||
handleOk := stmt.mc.clearResult()
|
||||
resLen, err := handleOk.readResultSetHeaderPacket()
|
||||
resLen, metadataFollows, err := handleOk.readResultSetHeaderPacket()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -116,7 +124,17 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
|
||||
|
||||
if resLen > 0 {
|
||||
rows.mc = mc
|
||||
rows.rs.columns, err = mc.readColumns(resLen)
|
||||
if metadataFollows {
|
||||
if rows.rs.columns, err = mc.readColumns(resLen, stmt.columns); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stmt.columns = rows.rs.columns
|
||||
} else {
|
||||
if err = mc.skipEof(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows.rs.columns = stmt.columns
|
||||
}
|
||||
} else {
|
||||
rows.rs.done = true
|
||||
|
||||
@@ -131,7 +149,7 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) {
|
||||
return rows, err
|
||||
}
|
||||
|
||||
var jsonType = reflect.TypeOf(json.RawMessage{})
|
||||
var jsonType = reflect.TypeFor[json.RawMessage]()
|
||||
|
||||
type converter struct{}
|
||||
|
||||
@@ -193,7 +211,7 @@ func (c converter) ConvertValue(v any) (driver.Value, error) {
|
||||
return nil, fmt.Errorf("unsupported type %T, a %s", v, rv.Kind())
|
||||
}
|
||||
|
||||
var valuerReflectType = reflect.TypeOf((*driver.Valuer)(nil)).Elem()
|
||||
var valuerReflectType = reflect.TypeFor[driver.Valuer]()
|
||||
|
||||
// callValuerValue returns vr.Value(), with one exception:
|
||||
// If vr.Value is an auto-generated method on a pointer type and the
|
||||
|
||||
+65
-90
@@ -182,7 +182,7 @@ func parseDateTime(b []byte, loc *time.Location) (time.Time, error) {
|
||||
|
||||
func parseByteYear(b []byte) (int, error) {
|
||||
year, n := 0, 1000
|
||||
for i := 0; i < 4; i++ {
|
||||
for i := range 4 {
|
||||
v, err := bToi(b[i])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -207,7 +207,7 @@ func parseByte2Digits(b1, b2 byte) (int, error) {
|
||||
|
||||
func parseByteNanoSec(b []byte) (int, error) {
|
||||
ns, digit := 0, 100000 // max is 6-digits
|
||||
for i := 0; i < len(b); i++ {
|
||||
for i := range b {
|
||||
v, err := bToi(b[i])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
@@ -625,108 +625,80 @@ func reserveBuffer(buf []byte, appendSize int) []byte {
|
||||
return buf[:newSize]
|
||||
}
|
||||
|
||||
// escapeBytesBackslash escapes []byte with backslashes (\)
|
||||
// This escapes the contents of a string (provided as []byte) by adding backslashes before special
|
||||
// characters, and turning others into specific escape sequences, such as
|
||||
// turning newlines into \n and null bytes into \0.
|
||||
// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L823-L932
|
||||
func escapeBytesBackslash(buf, v []byte) []byte {
|
||||
pos := len(buf)
|
||||
buf = reserveBuffer(buf, len(v)*2)
|
||||
// Lookup table for backslash escapes (used for both string and bytes)
|
||||
var backslashEscapeTable [256]byte
|
||||
|
||||
for _, c := range v {
|
||||
switch c {
|
||||
case '\x00':
|
||||
buf[pos+1] = '0'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\n':
|
||||
buf[pos+1] = 'n'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\r':
|
||||
buf[pos+1] = 'r'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\x1a':
|
||||
buf[pos+1] = 'Z'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\'':
|
||||
buf[pos+1] = '\''
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '"':
|
||||
buf[pos+1] = '"'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\\':
|
||||
buf[pos+1] = '\\'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
default:
|
||||
buf[pos] = c
|
||||
pos++
|
||||
}
|
||||
}
|
||||
|
||||
return buf[:pos]
|
||||
func init() {
|
||||
backslashEscapeTable['\x00'] = '0'
|
||||
backslashEscapeTable['\n'] = 'n'
|
||||
backslashEscapeTable['\r'] = 'r'
|
||||
backslashEscapeTable['\x1a'] = 'Z'
|
||||
backslashEscapeTable['\''] = '\''
|
||||
backslashEscapeTable['"'] = '"'
|
||||
backslashEscapeTable['\\'] = '\\'
|
||||
}
|
||||
|
||||
// escapeStringBackslash is similar to escapeBytesBackslash but for string.
|
||||
func escapeStringBackslash(buf []byte, v string) []byte {
|
||||
pos := len(buf)
|
||||
buf = reserveBuffer(buf, len(v)*2)
|
||||
|
||||
buf = reserveBuffer(buf, len(v)*2+2)
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
for i := 0; i < len(v); i++ {
|
||||
c := v[i]
|
||||
switch c {
|
||||
case '\x00':
|
||||
buf[pos+1] = '0'
|
||||
if esc := backslashEscapeTable[c]; esc != 0 {
|
||||
buf[pos+1] = esc
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\n':
|
||||
buf[pos+1] = 'n'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\r':
|
||||
buf[pos+1] = 'r'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\x1a':
|
||||
buf[pos+1] = 'Z'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\'':
|
||||
buf[pos+1] = '\''
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '"':
|
||||
buf[pos+1] = '"'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
case '\\':
|
||||
buf[pos+1] = '\\'
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
default:
|
||||
} else {
|
||||
buf[pos] = c
|
||||
pos++
|
||||
}
|
||||
}
|
||||
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
return buf[:pos]
|
||||
}
|
||||
|
||||
// escapeBytesQuotes escapes apostrophes in []byte by doubling them up.
|
||||
// This escapes the contents of a string by doubling up any apostrophes that
|
||||
// it contains. This is used when the NO_BACKSLASH_ESCAPES SQL_MODE is in
|
||||
// effect on the server.
|
||||
// https://github.com/mysql/mysql-server/blob/mysql-5.7.5/mysys/charset.c#L963-L1038
|
||||
func escapeBytesQuotes(buf, v []byte) []byte {
|
||||
// escapeBytesBackslash appends _binary'...' or '...' with backslash escaping for bytes.
|
||||
func escapeBytesBackslash(buf, v []byte, binary bool) []byte {
|
||||
pos := len(buf)
|
||||
buf = reserveBuffer(buf, len(v)*2)
|
||||
if binary {
|
||||
buf = reserveBuffer(buf, len(v)*2+9)
|
||||
copy(buf[pos:], []byte("_binary'"))
|
||||
pos += 8
|
||||
} else {
|
||||
buf = reserveBuffer(buf, len(v)*2+2)
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
}
|
||||
for _, c := range v {
|
||||
if esc := backslashEscapeTable[c]; esc != 0 {
|
||||
buf[pos+1] = esc
|
||||
buf[pos] = '\\'
|
||||
pos += 2
|
||||
} else {
|
||||
buf[pos] = c
|
||||
pos++
|
||||
}
|
||||
}
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
return buf[:pos]
|
||||
}
|
||||
|
||||
// escapeBytesQuotes appends _binary'...' or '...' with single-quote escaping for bytes.
|
||||
func escapeBytesQuotes(buf, v []byte, binary bool) []byte {
|
||||
pos := len(buf)
|
||||
if binary {
|
||||
buf = reserveBuffer(buf, len(v)*2+9)
|
||||
copy(buf[pos:], []byte("_binary'"))
|
||||
pos += 8
|
||||
} else {
|
||||
buf = reserveBuffer(buf, len(v)*2+2)
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
}
|
||||
for _, c := range v {
|
||||
if c == '\'' {
|
||||
buf[pos+1] = '\''
|
||||
@@ -737,16 +709,18 @@ func escapeBytesQuotes(buf, v []byte) []byte {
|
||||
pos++
|
||||
}
|
||||
}
|
||||
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
return buf[:pos]
|
||||
}
|
||||
|
||||
// escapeStringQuotes is similar to escapeBytesQuotes but for string.
|
||||
func escapeStringQuotes(buf []byte, v string) []byte {
|
||||
pos := len(buf)
|
||||
buf = reserveBuffer(buf, len(v)*2)
|
||||
|
||||
for i := 0; i < len(v); i++ {
|
||||
buf = reserveBuffer(buf, len(v)*2+2)
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
for i := range len(v) {
|
||||
c := v[i]
|
||||
if c == '\'' {
|
||||
buf[pos+1] = '\''
|
||||
@@ -757,7 +731,8 @@ func escapeStringQuotes(buf []byte, v string) []byte {
|
||||
pos++
|
||||
}
|
||||
}
|
||||
|
||||
buf[pos] = '\''
|
||||
pos++
|
||||
return buf[:pos]
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
Copyright (c) 2016 HashiCorp, Inc.
|
||||
Copyright IBM Corp. 2016, 2025
|
||||
|
||||
Mozilla Public License, version 2.0
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) HashiCorp, Inc.
|
||||
# Copyright IBM Corp. 2016, 2025
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
version: v1
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
# Copyright (c) HashiCorp, Inc.
|
||||
# Copyright IBM Corp. 2016, 2025
|
||||
# SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
version: v1
|
||||
|
||||
+6
-4
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
@@ -669,7 +669,7 @@ func (c *Client) Start() (addr net.Addr, err error) {
|
||||
// Setup a temporary certificate for client/server mtls, and send the public
|
||||
// certificate to the plugin.
|
||||
if c.config.AutoMTLS {
|
||||
c.logger.Info("configuring client automatic mTLS")
|
||||
c.logger.Debug("configuring client automatic mTLS")
|
||||
certPEM, keyPEM, err := generateCert()
|
||||
if err != nil {
|
||||
c.logger.Error("failed to generate client certificate", "error", err)
|
||||
@@ -753,9 +753,12 @@ func (c *Client) Start() (addr net.Addr, err error) {
|
||||
// Create a context for when we kill
|
||||
c.doneCtx, c.ctxCancel = context.WithCancel(context.Background())
|
||||
|
||||
// Add two to pipesWaitGroup: one for logStderr, one for the goroutine
|
||||
// below that consumes Stdout. We mustn't continue to Add once we might Wait.
|
||||
c.pipesWaitGroup.Add(2)
|
||||
|
||||
// Start goroutine that logs the stderr
|
||||
c.clientWaitGroup.Add(1)
|
||||
c.pipesWaitGroup.Add(1)
|
||||
// logStderr calls c.pipesWaitGroup.Done()
|
||||
go c.logStderr(runner.Name(), runner.Stderr())
|
||||
|
||||
@@ -791,7 +794,6 @@ func (c *Client) Start() (addr net.Addr, err error) {
|
||||
// out of stdout
|
||||
linesCh := make(chan string)
|
||||
c.clientWaitGroup.Add(1)
|
||||
c.pipesWaitGroup.Add(1)
|
||||
go func() {
|
||||
defer c.clientWaitGroup.Done()
|
||||
defer c.pipesWaitGroup.Done()
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package cmdrunner
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package cmdrunner
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package cmdrunner
|
||||
|
||||
+1
-2
@@ -1,8 +1,7 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package cmdrunner
|
||||
|
||||
|
||||
+1
-2
@@ -1,8 +1,7 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
//go:build windows
|
||||
// +build windows
|
||||
|
||||
package cmdrunner
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package cmdrunner
|
||||
|
||||
+6
-2
@@ -1,8 +1,7 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
//go:build !windows
|
||||
// +build !windows
|
||||
|
||||
package cmdrunner
|
||||
|
||||
@@ -16,6 +15,11 @@ import (
|
||||
func _pidAlive(pid int) bool {
|
||||
proc, err := os.FindProcess(pid)
|
||||
if err == nil {
|
||||
// On Linux with Go 1.23+, FindProcess opens a pidfd which must be
|
||||
// released or it leaks an FD on every call. Release errors are
|
||||
// intentionally ignored; the handle is short-lived and there's
|
||||
// nothing actionable to recover from a release failure.
|
||||
defer func() { _ = proc.Release() }()
|
||||
err = proc.Signal(syscall.Signal(0))
|
||||
}
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package cmdrunner
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package grpcmux
|
||||
|
||||
Generated
Vendored
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package grpcmux
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package grpcmux
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package grpcmux
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package grpcmux
|
||||
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
syntax = "proto3";
|
||||
package plugin;
|
||||
option go_package = "./plugin";
|
||||
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
|
||||
|
||||
message ConnInfo {
|
||||
uint32 service_id = 1;
|
||||
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
syntax = "proto3";
|
||||
package plugin;
|
||||
option go_package = "./plugin";
|
||||
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
|
||||
|
||||
message Empty {
|
||||
}
|
||||
|
||||
+2
-2
@@ -1,9 +1,9 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
syntax = "proto3";
|
||||
package plugin;
|
||||
option go_package = "./plugin";
|
||||
option go_package = "github.com/hashicorp/go-plugin/internal/plugin";
|
||||
|
||||
import "google/protobuf/empty.proto";
|
||||
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
// The plugin package exposes functions and helpers for communicating to
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package runner
|
||||
|
||||
+2
-2
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
@@ -307,7 +307,7 @@ func Serve(opts *ServeConfig) {
|
||||
// If the client is configured using AutoMTLS, the certificate will be here,
|
||||
// and we need to generate our own in response.
|
||||
if tlsConfig == nil && clientCert != "" {
|
||||
logger.Info("configuring server automatic mTLS")
|
||||
logger.Debug("configuring server automatic mTLS")
|
||||
clientCertPool := x509.NewCertPool()
|
||||
if !clientCertPool.AppendCertsFromPEM([]byte(clientCert)) {
|
||||
logger.Error("client cert provided but failed to parse", "cert", clientCert)
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+1
-1
@@ -1,4 +1,4 @@
|
||||
// Copyright (c) HashiCorp, Inc.
|
||||
// Copyright IBM Corp. 2016, 2025
|
||||
// SPDX-License-Identifier: MPL-2.0
|
||||
|
||||
package plugin
|
||||
|
||||
+5
@@ -175,6 +175,11 @@ func (c *Client) appendObjectDo(ctx context.Context, bucketName, objectName stri
|
||||
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
|
||||
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
|
||||
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
|
||||
ChecksumMD5: h.Get(ChecksumMD5.Key()),
|
||||
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
|
||||
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
|
||||
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
|
||||
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
|
||||
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
+17
-5
@@ -150,6 +150,11 @@ type UploadInfo struct {
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC64NVME string
|
||||
ChecksumMD5 string
|
||||
ChecksumSHA512 string
|
||||
ChecksumXXHash64 string
|
||||
ChecksumXXHash3 string
|
||||
ChecksumXXHash128 string
|
||||
ChecksumMode string
|
||||
}
|
||||
|
||||
@@ -168,11 +173,12 @@ type ObjectInfo struct {
|
||||
// each parts concatenated into one string.
|
||||
ETag string `json:"etag"`
|
||||
|
||||
Key string `json:"name"` // Name of the object
|
||||
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
|
||||
Size int64 `json:"size"` // Size in bytes of the object.
|
||||
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
|
||||
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
|
||||
Key string `json:"name"` // Name of the object
|
||||
LastModified time.Time `json:"lastModified"` // Date and time the object was last modified.
|
||||
Size int64 `json:"size"` // Size in bytes of the object.
|
||||
ContentType string `json:"contentType"` // A standard MIME type describing the format of the object data.
|
||||
ContentEncoding string `json:"contentEncoding"` // A standard MIME type describing encoding of the object data.
|
||||
Expires time.Time `json:"expires"` // The date and time at which the object is no longer able to be cached.
|
||||
|
||||
// Collection of additional metadata on the object.
|
||||
// eg: x-amz-meta-*, content-encoding etc.
|
||||
@@ -226,6 +232,12 @@ type ObjectInfo struct {
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC64NVME string
|
||||
ChecksumMD5 string
|
||||
ChecksumSHA512 string
|
||||
ChecksumXXHash64 string
|
||||
ChecksumXXHash3 string
|
||||
ChecksumXXHash128 string
|
||||
ChecksumAlgorithm string
|
||||
ChecksumMode string `xml:"ChecksumType"`
|
||||
|
||||
Internal *struct {
|
||||
|
||||
+96
-10
@@ -23,6 +23,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/minio/minio-go/v7/pkg/encrypt"
|
||||
@@ -88,10 +89,17 @@ type ObjectAttributesResponse struct {
|
||||
StorageClass string
|
||||
ObjectSize int
|
||||
Checksum struct {
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumMD5 string `xml:",omitempty"`
|
||||
ChecksumSHA512 string `xml:",omitempty"`
|
||||
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
|
||||
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
|
||||
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
|
||||
ChecksumType string `xml:",omitempty"`
|
||||
}
|
||||
ObjectParts struct {
|
||||
PartsCount int
|
||||
@@ -105,12 +113,90 @@ type ObjectAttributesResponse struct {
|
||||
|
||||
// ObjectAttributePart is used by ObjectAttributesResponse to describe an object part
|
||||
type ObjectAttributePart struct {
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
PartNumber int
|
||||
Size int
|
||||
ChecksumCRC32 string `xml:",omitempty"`
|
||||
ChecksumCRC32C string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumMD5 string `xml:",omitempty"`
|
||||
ChecksumSHA512 string `xml:",omitempty"`
|
||||
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
|
||||
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
|
||||
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
|
||||
PartNumber int
|
||||
Size int
|
||||
}
|
||||
|
||||
// ChecksumMap returns a map of checksums for the object.
|
||||
func (o *ObjectAttributesResponse) ChecksumMap() map[string]string {
|
||||
res := make(map[string]string)
|
||||
setif := func(typ ChecksumType, value string) {
|
||||
if value != "" {
|
||||
res[typ.Key()] = value
|
||||
}
|
||||
}
|
||||
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
|
||||
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
|
||||
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
|
||||
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
|
||||
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
|
||||
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
|
||||
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
|
||||
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
|
||||
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
|
||||
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
|
||||
return res
|
||||
}
|
||||
|
||||
// ChecksumMode returns the checksum mode of the object.
|
||||
// If unable to determine, returns ChecksumUnknownMode.
|
||||
func (o *ObjectAttributesResponse) ChecksumMode() ChecksumMode {
|
||||
t := o.ChecksumType()
|
||||
if !t.IsSet() {
|
||||
return ChecksumUnknownMode
|
||||
}
|
||||
switch o.Checksum.ChecksumType {
|
||||
case amzChecksumModeComposite:
|
||||
return ChecksumCompositeMode
|
||||
case amzChecksumModeFullObject:
|
||||
return ChecksumFullObjectMode
|
||||
case "":
|
||||
// Likely not supported by the server.
|
||||
if o.Checksum.ChecksumCRC64NVME != "" || !strings.ContainsRune(o.ETag, '-') {
|
||||
// Always full object.
|
||||
return ChecksumFullObjectMode
|
||||
}
|
||||
if !t.CanMergeCRC() {
|
||||
// Only composite possible.
|
||||
return ChecksumCompositeMode
|
||||
}
|
||||
}
|
||||
return ChecksumUnknownMode
|
||||
}
|
||||
|
||||
// ChecksumType returns the checksum type of the object.
|
||||
// If none is set, returns ChecksumNone.
|
||||
func (o *ObjectAttributesResponse) ChecksumType() ChecksumType {
|
||||
t := ChecksumNone
|
||||
setif := func(typ ChecksumType, value string) {
|
||||
if value != "" {
|
||||
t = typ
|
||||
}
|
||||
}
|
||||
setif(ChecksumCRC32C, o.Checksum.ChecksumCRC32C)
|
||||
setif(ChecksumCRC32, o.Checksum.ChecksumCRC32)
|
||||
setif(ChecksumCRC64NVME, o.Checksum.ChecksumCRC64NVME)
|
||||
setif(ChecksumSHA1, o.Checksum.ChecksumSHA1)
|
||||
setif(ChecksumSHA256, o.Checksum.ChecksumSHA256)
|
||||
setif(ChecksumMD5, o.Checksum.ChecksumMD5)
|
||||
setif(ChecksumSHA512, o.Checksum.ChecksumSHA512)
|
||||
setif(ChecksumXXHash64, o.Checksum.ChecksumXXHash64)
|
||||
setif(ChecksumXXHash3, o.Checksum.ChecksumXXHash3)
|
||||
setif(ChecksumXXHash128, o.Checksum.ChecksumXXHash128)
|
||||
if t.IsSet() && o.Checksum.ChecksumType == amzChecksumModeFullObject {
|
||||
t |= ChecksumFullObject
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (o *ObjectAttributes) parseResponse(resp *http.Response) (err error) {
|
||||
|
||||
+7
-1
@@ -404,7 +404,7 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
|
||||
}
|
||||
|
||||
var (
|
||||
keyMarker = ""
|
||||
keyMarker = opts.StartAfter
|
||||
versionIDMarker = ""
|
||||
preName = ""
|
||||
preKey = ""
|
||||
@@ -432,12 +432,18 @@ func (c *Client) listObjectVersions(ctx context.Context, bucketName string, opts
|
||||
UserMetadata: version.UserMetadata,
|
||||
Internal: version.Internal,
|
||||
NumVersions: numVersions,
|
||||
ChecksumAlgorithm: version.ChecksumAlgorithm,
|
||||
ChecksumMode: version.ChecksumType,
|
||||
ChecksumCRC32: version.ChecksumCRC32,
|
||||
ChecksumCRC32C: version.ChecksumCRC32C,
|
||||
ChecksumSHA1: version.ChecksumSHA1,
|
||||
ChecksumSHA256: version.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: version.ChecksumCRC64NVME,
|
||||
ChecksumMD5: version.ChecksumMD5,
|
||||
ChecksumSHA512: version.ChecksumSHA512,
|
||||
ChecksumXXHash64: version.ChecksumXXHash64,
|
||||
ChecksumXXHash3: version.ChecksumXXHash3,
|
||||
ChecksumXXHash128: version.ChecksumXXHash128,
|
||||
}
|
||||
if !yield(info) {
|
||||
return false
|
||||
|
||||
+15
@@ -192,6 +192,11 @@ func (c *Client) putObjectMultipartNoStream(ctx context.Context, bucketName, obj
|
||||
ChecksumSHA1: part.ChecksumSHA1,
|
||||
ChecksumSHA256: part.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
|
||||
ChecksumMD5: part.ChecksumMD5,
|
||||
ChecksumSHA512: part.ChecksumSHA512,
|
||||
ChecksumXXHash64: part.ChecksumXXHash64,
|
||||
ChecksumXXHash3: part.ChecksumXXHash3,
|
||||
ChecksumXXHash128: part.ChecksumXXHash128,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -353,6 +358,11 @@ func (c *Client) uploadPart(ctx context.Context, p uploadPartParams) (ObjectPart
|
||||
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
|
||||
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
|
||||
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
|
||||
ChecksumMD5: h.Get(ChecksumMD5.Key()),
|
||||
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
|
||||
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
|
||||
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
|
||||
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
|
||||
}
|
||||
objPart.Size = p.size
|
||||
objPart.PartNumber = p.partNumber
|
||||
@@ -458,6 +468,11 @@ func (c *Client) completeMultipartUpload(ctx context.Context, bucketName, object
|
||||
ChecksumCRC32: completeMultipartUploadResult.ChecksumCRC32,
|
||||
ChecksumCRC32C: completeMultipartUploadResult.ChecksumCRC32C,
|
||||
ChecksumCRC64NVME: completeMultipartUploadResult.ChecksumCRC64NVME,
|
||||
ChecksumMD5: completeMultipartUploadResult.ChecksumMD5,
|
||||
ChecksumSHA512: completeMultipartUploadResult.ChecksumSHA512,
|
||||
ChecksumXXHash64: completeMultipartUploadResult.ChecksumXXHash64,
|
||||
ChecksumXXHash3: completeMultipartUploadResult.ChecksumXXHash3,
|
||||
ChecksumXXHash128: completeMultipartUploadResult.ChecksumXXHash128,
|
||||
ChecksumMode: completeMultipartUploadResult.ChecksumType,
|
||||
}, nil
|
||||
}
|
||||
|
||||
+20
@@ -259,6 +259,11 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN
|
||||
ChecksumSHA1: uploadRes.Part.ChecksumSHA1,
|
||||
ChecksumSHA256: uploadRes.Part.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: uploadRes.Part.ChecksumCRC64NVME,
|
||||
ChecksumMD5: uploadRes.Part.ChecksumMD5,
|
||||
ChecksumSHA512: uploadRes.Part.ChecksumSHA512,
|
||||
ChecksumXXHash64: uploadRes.Part.ChecksumXXHash64,
|
||||
ChecksumXXHash3: uploadRes.Part.ChecksumXXHash3,
|
||||
ChecksumXXHash128: uploadRes.Part.ChecksumXXHash128,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -418,6 +423,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b
|
||||
ChecksumSHA1: part.ChecksumSHA1,
|
||||
ChecksumSHA256: part.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
|
||||
ChecksumMD5: part.ChecksumMD5,
|
||||
ChecksumSHA512: part.ChecksumSHA512,
|
||||
ChecksumXXHash64: part.ChecksumXXHash64,
|
||||
ChecksumXXHash3: part.ChecksumXXHash3,
|
||||
ChecksumXXHash128: part.ChecksumXXHash128,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -617,6 +627,11 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam
|
||||
ChecksumSHA1: part.ChecksumSHA1,
|
||||
ChecksumSHA256: part.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
|
||||
ChecksumMD5: part.ChecksumMD5,
|
||||
ChecksumSHA512: part.ChecksumSHA512,
|
||||
ChecksumXXHash64: part.ChecksumXXHash64,
|
||||
ChecksumXXHash3: part.ChecksumXXHash3,
|
||||
ChecksumXXHash128: part.ChecksumXXHash128,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -796,6 +811,11 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string,
|
||||
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
|
||||
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
|
||||
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
|
||||
ChecksumMD5: h.Get(ChecksumMD5.Key()),
|
||||
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
|
||||
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
|
||||
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
|
||||
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
|
||||
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
+5
@@ -502,6 +502,11 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam
|
||||
ChecksumSHA1: part.ChecksumSHA1,
|
||||
ChecksumSHA256: part.ChecksumSHA256,
|
||||
ChecksumCRC64NVME: part.ChecksumCRC64NVME,
|
||||
ChecksumMD5: part.ChecksumMD5,
|
||||
ChecksumSHA512: part.ChecksumSHA512,
|
||||
ChecksumXXHash64: part.ChecksumXXHash64,
|
||||
ChecksumXXHash3: part.ChecksumXXHash3,
|
||||
ChecksumXXHash128: part.ChecksumXXHash128,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
+41
@@ -113,6 +113,12 @@ type Version struct {
|
||||
ChecksumSHA1 string `xml:",omitempty"`
|
||||
ChecksumSHA256 string `xml:",omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumMD5 string `xml:",omitempty"`
|
||||
ChecksumSHA512 string `xml:",omitempty"`
|
||||
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
|
||||
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
|
||||
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
|
||||
ChecksumAlgorithm string `xml:",omitempty"`
|
||||
ChecksumType string `xml:",omitempty"`
|
||||
|
||||
isDeleteMarker bool
|
||||
@@ -297,6 +303,11 @@ type ObjectPart struct {
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC64NVME string
|
||||
ChecksumMD5 string
|
||||
ChecksumSHA512 string
|
||||
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
|
||||
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
|
||||
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
|
||||
}
|
||||
|
||||
// Checksum will return the checksum for the given type.
|
||||
@@ -313,6 +324,16 @@ func (c ObjectPart) Checksum(t ChecksumType) string {
|
||||
return c.ChecksumSHA256
|
||||
case t.Is(ChecksumCRC64NVME):
|
||||
return c.ChecksumCRC64NVME
|
||||
case t.Is(ChecksumMD5):
|
||||
return c.ChecksumMD5
|
||||
case t.Is(ChecksumSHA512):
|
||||
return c.ChecksumSHA512
|
||||
case t.Is(ChecksumXXHash64):
|
||||
return c.ChecksumXXHash64
|
||||
case t.Is(ChecksumXXHash3):
|
||||
return c.ChecksumXXHash3
|
||||
case t.Is(ChecksumXXHash128):
|
||||
return c.ChecksumXXHash128
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -382,6 +403,11 @@ type completeMultipartUploadResult struct {
|
||||
ChecksumSHA1 string
|
||||
ChecksumSHA256 string
|
||||
ChecksumCRC64NVME string
|
||||
ChecksumMD5 string
|
||||
ChecksumSHA512 string
|
||||
ChecksumXXHash64 string `xml:"ChecksumXXHASH64"`
|
||||
ChecksumXXHash3 string `xml:"ChecksumXXHASH3"`
|
||||
ChecksumXXHash128 string `xml:"ChecksumXXHASH128"`
|
||||
ChecksumType string
|
||||
}
|
||||
|
||||
@@ -398,6 +424,11 @@ type CompletePart struct {
|
||||
ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"`
|
||||
ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"`
|
||||
ChecksumCRC64NVME string `xml:",omitempty"`
|
||||
ChecksumMD5 string `xml:",omitempty"`
|
||||
ChecksumSHA512 string `xml:",omitempty"`
|
||||
ChecksumXXHash64 string `xml:"ChecksumXXHASH64,omitempty"`
|
||||
ChecksumXXHash3 string `xml:"ChecksumXXHASH3,omitempty"`
|
||||
ChecksumXXHash128 string `xml:"ChecksumXXHASH128,omitempty"`
|
||||
}
|
||||
|
||||
// Checksum will return the checksum for the given type.
|
||||
@@ -414,6 +445,16 @@ func (c CompletePart) Checksum(t ChecksumType) string {
|
||||
return c.ChecksumSHA256
|
||||
case t.Is(ChecksumCRC64NVME):
|
||||
return c.ChecksumCRC64NVME
|
||||
case t.Is(ChecksumMD5):
|
||||
return c.ChecksumMD5
|
||||
case t.Is(ChecksumSHA512):
|
||||
return c.ChecksumSHA512
|
||||
case t.Is(ChecksumXXHash64):
|
||||
return c.ChecksumXXHash64
|
||||
case t.Is(ChecksumXXHash3):
|
||||
return c.ChecksumXXHash3
|
||||
case t.Is(ChecksumXXHash128):
|
||||
return c.ChecksumXXHash128
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
+17
-9
@@ -52,6 +52,12 @@ type UpdateObjectEncryptionOptions struct {
|
||||
VersionID string
|
||||
}
|
||||
|
||||
// UpdateObjectEncryptionResult holds the result of an UpdateObjectEncryption call.
|
||||
type UpdateObjectEncryptionResult struct {
|
||||
// VersionID is the version ID of the object that was updated, if versioning is enabled.
|
||||
VersionID string
|
||||
}
|
||||
|
||||
// UpdateObjectEncryption changes the encryption configuration of an existing object in-place.
|
||||
// The object must already be encrypted with SSE-S3 or SSE-KMS. SSE-C objects are not supported.
|
||||
// This operation rotates the data encryption key envelope without re-reading/re-writing object data.
|
||||
@@ -62,19 +68,19 @@ type UpdateObjectEncryptionOptions struct {
|
||||
// - objectName: Name of the object
|
||||
// - opts: Options including KMSKeyArn (required), optional BucketKeyEnabled, and optional VersionID
|
||||
//
|
||||
// Returns an error if the operation fails.
|
||||
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) error {
|
||||
// Returns the version ID of the updated object (if versioning is enabled) and an error if the operation fails.
|
||||
func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectName string, opts UpdateObjectEncryptionOptions) (UpdateObjectEncryptionResult, error) {
|
||||
// Input validation.
|
||||
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
|
||||
return err
|
||||
return UpdateObjectEncryptionResult{}, err
|
||||
}
|
||||
|
||||
if err := s3utils.CheckValidObjectName(objectName); err != nil {
|
||||
return err
|
||||
return UpdateObjectEncryptionResult{}, err
|
||||
}
|
||||
|
||||
if opts.KMSKeyArn == "" {
|
||||
return errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
|
||||
return UpdateObjectEncryptionResult{}, errInvalidArgument("KMSKeyArn is required for UpdateObjectEncryption.")
|
||||
}
|
||||
|
||||
// Get resources properly escaped and lined up before
|
||||
@@ -96,7 +102,7 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
|
||||
|
||||
bodyData, err := xml.Marshal(reqBody)
|
||||
if err != nil {
|
||||
return err
|
||||
return UpdateObjectEncryptionResult{}, err
|
||||
}
|
||||
|
||||
reqMetadata := requestMetadata{
|
||||
@@ -113,10 +119,12 @@ func (c *Client) UpdateObjectEncryption(ctx context.Context, bucketName, objectN
|
||||
resp, err := c.executeMethod(ctx, http.MethodPut, reqMetadata)
|
||||
defer closeResponse(resp)
|
||||
if err != nil {
|
||||
return err
|
||||
return UpdateObjectEncryptionResult{}, err
|
||||
}
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return httpRespToErrorResponse(resp, bucketName, objectName)
|
||||
return UpdateObjectEncryptionResult{}, httpRespToErrorResponse(resp, bucketName, objectName)
|
||||
}
|
||||
return nil
|
||||
return UpdateObjectEncryptionResult{
|
||||
VersionID: resp.Header.Get(amzVersionID),
|
||||
}, nil
|
||||
}
|
||||
|
||||
+13
-1
@@ -198,6 +198,9 @@ func New(endpoint string, opts *Options) (*Client, error) {
|
||||
// Amazon S3 endpoints are resolved into dual-stack endpoints by default
|
||||
// for backwards compatibility.
|
||||
clnt.s3DualstackEnabled = true
|
||||
} else if s3utils.IsAmazonOutpostsEndpoint(*clnt.endpointURL) {
|
||||
// S3 on Outposts uses signature v4 with service name s3-outposts.
|
||||
clnt.overrideSignerType = credentials.SignatureV4
|
||||
}
|
||||
|
||||
return clnt, nil
|
||||
@@ -912,7 +915,11 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
|
||||
req = signer.PreSignV2(*req, accessKeyID, secretAccessKey, metadata.expires, isVirtualHost)
|
||||
} else if signerType.IsV4() {
|
||||
// Presign URL with signature v4.
|
||||
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
|
||||
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
|
||||
req = signer.PreSignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
|
||||
} else {
|
||||
req = signer.PreSignV4(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.expires)
|
||||
}
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
@@ -971,6 +978,9 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
|
||||
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
|
||||
req = signer.StreamingSignV4Express(req, accessKeyID,
|
||||
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
|
||||
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
|
||||
req = signer.StreamingSignV4Outposts(req, accessKeyID,
|
||||
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
|
||||
} else {
|
||||
req = signer.StreamingSignV4(req, accessKeyID,
|
||||
secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher())
|
||||
@@ -991,6 +1001,8 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request
|
||||
|
||||
if s3utils.IsAmazonExpressRegionalEndpoint(*c.endpointURL) {
|
||||
req = signer.SignV4TrailerExpress(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
|
||||
} else if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
|
||||
req = signer.SignV4TrailerOutposts(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
|
||||
} else {
|
||||
// Add signature version '4' authorization header.
|
||||
req = signer.SignV4Trailer(*req, accessKeyID, secretAccessKey, sessionToken, location, metadata.trailer)
|
||||
|
||||
+6
-1
@@ -209,6 +209,11 @@ func (c *Client) getBucketLocationRequest(ctx context.Context, bucketName string
|
||||
}
|
||||
|
||||
req.Header.Set("X-Amz-Content-Sha256", contentSha256)
|
||||
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
|
||||
if s3utils.IsAmazonOutpostsEndpoint(*c.endpointURL) {
|
||||
region := getDefaultLocation(*c.endpointURL, c.region)
|
||||
req = signer.SignV4Outposts(*req, accessKeyID, secretAccessKey, sessionToken, region)
|
||||
} else {
|
||||
req = signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
|
||||
+67
-4
@@ -18,8 +18,10 @@
|
||||
package minio
|
||||
|
||||
import (
|
||||
"crypto/md5"
|
||||
"crypto/sha1"
|
||||
"crypto/sha256"
|
||||
"crypto/sha512"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
@@ -30,8 +32,10 @@ import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"github.com/cespare/xxhash/v2"
|
||||
"github.com/klauspost/crc32"
|
||||
"github.com/minio/crc64nvme"
|
||||
"github.com/zeebo/xxh3"
|
||||
)
|
||||
|
||||
// ChecksumMode contains information about the checksum mode on the object
|
||||
@@ -49,6 +53,9 @@ const (
|
||||
|
||||
// checksumModeMask is a mask for valid checksum mode types.
|
||||
checksumModeMask = checksumLastMode - 1
|
||||
|
||||
// ChecksumUnknownMode indicates no or unknown checksum mode.
|
||||
ChecksumUnknownMode ChecksumMode = 0
|
||||
)
|
||||
|
||||
// Is returns if c is all of t.
|
||||
@@ -64,9 +71,9 @@ func (c ChecksumMode) Key() string {
|
||||
func (c ChecksumMode) String() string {
|
||||
switch c & checksumModeMask {
|
||||
case ChecksumFullObjectMode:
|
||||
return "FULL_OBJECT"
|
||||
return amzChecksumModeFullObject
|
||||
case ChecksumCompositeMode:
|
||||
return "COMPOSITE"
|
||||
return amzChecksumModeComposite
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -86,6 +93,16 @@ const (
|
||||
ChecksumCRC32C
|
||||
// ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial.
|
||||
ChecksumCRC64NVME
|
||||
// ChecksumMD5 indicates an MD5 checksum.
|
||||
ChecksumMD5
|
||||
// ChecksumSHA512 indicates a SHA-512 checksum.
|
||||
ChecksumSHA512
|
||||
// ChecksumXXHash64 indicates an XXHash64 checksum.
|
||||
ChecksumXXHash64
|
||||
// ChecksumXXHash3 indicates an XXH3-64 checksum.
|
||||
ChecksumXXHash3
|
||||
// ChecksumXXHash128 indicates an XXH3-128 checksum.
|
||||
ChecksumXXHash128
|
||||
|
||||
// Keep after all valid checksums
|
||||
checksumLast
|
||||
@@ -112,7 +129,15 @@ const (
|
||||
amzChecksumSHA1 = "x-amz-checksum-sha1"
|
||||
amzChecksumSHA256 = "x-amz-checksum-sha256"
|
||||
amzChecksumCRC64NVME = "x-amz-checksum-crc64nvme"
|
||||
amzChecksumMD5 = "x-amz-checksum-md5"
|
||||
amzChecksumSHA512 = "x-amz-checksum-sha512"
|
||||
amzChecksumXXHash64 = "x-amz-checksum-xxhash64"
|
||||
amzChecksumXXHash3 = "x-amz-checksum-xxhash3"
|
||||
amzChecksumXXHash128 = "x-amz-checksum-xxhash128"
|
||||
amzChecksumMode = "x-amz-checksum-type"
|
||||
|
||||
amzChecksumModeComposite = "COMPOSITE"
|
||||
amzChecksumModeFullObject = "FULL_OBJECT"
|
||||
)
|
||||
|
||||
// Base returns the base type, without modifiers.
|
||||
@@ -139,6 +164,16 @@ func (c ChecksumType) Key() string {
|
||||
return amzChecksumSHA256
|
||||
case ChecksumCRC64NVME:
|
||||
return amzChecksumCRC64NVME
|
||||
case ChecksumMD5:
|
||||
return amzChecksumMD5
|
||||
case ChecksumSHA512:
|
||||
return amzChecksumSHA512
|
||||
case ChecksumXXHash64:
|
||||
return amzChecksumXXHash64
|
||||
case ChecksumXXHash3:
|
||||
return amzChecksumXXHash3
|
||||
case ChecksumXXHash128:
|
||||
return amzChecksumXXHash128
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -146,7 +181,8 @@ func (c ChecksumType) Key() string {
|
||||
// CanComposite will return if the checksum type can be used for composite multipart upload on AWS.
|
||||
func (c ChecksumType) CanComposite() bool {
|
||||
switch c & checksumMask {
|
||||
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C:
|
||||
case ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC32C,
|
||||
ChecksumMD5, ChecksumSHA512, ChecksumXXHash64, ChecksumXXHash3, ChecksumXXHash128:
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -186,6 +222,14 @@ func (c ChecksumType) RawByteLen() int {
|
||||
return sha256.Size
|
||||
case ChecksumCRC64NVME:
|
||||
return crc64nvme.Size
|
||||
case ChecksumXXHash64, ChecksumXXHash3:
|
||||
return 8
|
||||
case ChecksumMD5:
|
||||
return md5.Size
|
||||
case ChecksumSHA512:
|
||||
return sha512.Size
|
||||
case ChecksumXXHash128:
|
||||
return 16
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -206,6 +250,16 @@ func (c ChecksumType) Hasher() hash.Hash {
|
||||
return sha256.New()
|
||||
case ChecksumCRC64NVME:
|
||||
return crc64nvme.New()
|
||||
case ChecksumMD5:
|
||||
return md5.New()
|
||||
case ChecksumSHA512:
|
||||
return sha512.New()
|
||||
case ChecksumXXHash64:
|
||||
return xxhash.New()
|
||||
case ChecksumXXHash3:
|
||||
return xxh3.New()
|
||||
case ChecksumXXHash128:
|
||||
return xxh3.New128()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -233,7 +287,6 @@ func (c ChecksumType) EncodeToString(b []byte) string {
|
||||
}
|
||||
|
||||
// String returns the type as a string.
|
||||
// CRC32, CRC32C, SHA1, and SHA256 for valid values.
|
||||
// Empty string for unset and "<invalid>" if not valid.
|
||||
func (c ChecksumType) String() string {
|
||||
switch c & checksumMask {
|
||||
@@ -249,6 +302,16 @@ func (c ChecksumType) String() string {
|
||||
return ""
|
||||
case ChecksumCRC64NVME:
|
||||
return "CRC64NVME"
|
||||
case ChecksumMD5:
|
||||
return "MD5"
|
||||
case ChecksumSHA512:
|
||||
return "SHA512"
|
||||
case ChecksumXXHash64:
|
||||
return "XXHASH64"
|
||||
case ChecksumXXHash3:
|
||||
return "XXHASH3"
|
||||
case ChecksumXXHash128:
|
||||
return "XXHASH128"
|
||||
}
|
||||
return "<invalid>"
|
||||
}
|
||||
|
||||
+4
-4
@@ -94,15 +94,15 @@ var awsS3EndpointMap = map[string]awsS3Endpoint{
|
||||
},
|
||||
"us-iso-east-1": {
|
||||
"s3.us-iso-east-1.c2s.ic.gov",
|
||||
"s3.dualstack.us-iso-east-1.c2s.ic.gov",
|
||||
"", // dualstack endpoint doesn't exist
|
||||
},
|
||||
"us-isob-east-1": {
|
||||
"s3.us-isob-east-1.sc2s.sgov.gov",
|
||||
"s3.dualstack.us-isob-east-1.sc2s.sgov.gov",
|
||||
"", // dualstack endpoint doesn't exist
|
||||
},
|
||||
"us-iso-west-1": {
|
||||
"s3.us-iso-west-1.c2s.ic.gov",
|
||||
"s3.dualstack.us-iso-west-1.c2s.ic.gov",
|
||||
"", // dualstack endpoint doesn't exist
|
||||
},
|
||||
"us-west-2": {
|
||||
"s3.us-west-2.amazonaws.com",
|
||||
@@ -269,7 +269,7 @@ func getS3Endpoint(bucketLocation string, useDualstack bool) (endpoint string) {
|
||||
}
|
||||
return "s3.us-east-1.amazonaws.com"
|
||||
}
|
||||
if useDualstack {
|
||||
if useDualstack && s3Endpoint.dualstackEndpoint != "" {
|
||||
return s3Endpoint.dualstackEndpoint
|
||||
}
|
||||
return s3Endpoint.endpoint
|
||||
|
||||
+160
-4
@@ -73,8 +73,25 @@ const (
|
||||
enableKMS = "ENABLE_KMS"
|
||||
appVersion = "0.1.0"
|
||||
skipCERTValidation = "SKIP_CERT_VALIDATION"
|
||||
|
||||
// TODO: remove when server supports the 2026 checksum types.
|
||||
ignore2026Checksums = true
|
||||
)
|
||||
|
||||
func ignore2026ChecksumError(cs minio.ChecksumType, err error) bool {
|
||||
if !ignore2026Checksums {
|
||||
return false
|
||||
}
|
||||
switch cs.Base() {
|
||||
case minio.ChecksumMD5, minio.ChecksumSHA512,
|
||||
minio.ChecksumXXHash64, minio.ChecksumXXHash3, minio.ChecksumXXHash128:
|
||||
default:
|
||||
return false
|
||||
}
|
||||
var er minio.ErrorResponse
|
||||
return errors.As(err, &er) && er.Code == "InvalidArgument"
|
||||
}
|
||||
|
||||
func createHTTPTransport() (transport *http.Transport) {
|
||||
var err error
|
||||
transport, err = minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS)))
|
||||
@@ -2096,6 +2113,11 @@ func testPutObjectWithChecksums() {
|
||||
tests := []struct {
|
||||
cs minio.ChecksumType
|
||||
}{
|
||||
{cs: minio.ChecksumMD5},
|
||||
{cs: minio.ChecksumSHA512},
|
||||
{cs: minio.ChecksumXXHash64},
|
||||
{cs: minio.ChecksumXXHash3},
|
||||
{cs: minio.ChecksumXXHash128},
|
||||
{cs: minio.ChecksumCRC32C},
|
||||
{cs: minio.ChecksumCRC32},
|
||||
{cs: minio.ChecksumSHA1},
|
||||
@@ -2160,6 +2182,10 @@ func testPutObjectWithChecksums() {
|
||||
UserMetadata: meta,
|
||||
})
|
||||
if err != nil {
|
||||
if ignore2026ChecksumError(test.cs, err) {
|
||||
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
|
||||
continue
|
||||
}
|
||||
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
||||
return
|
||||
}
|
||||
@@ -2168,6 +2194,11 @@ func testPutObjectWithChecksums() {
|
||||
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
|
||||
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
|
||||
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
|
||||
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
|
||||
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
|
||||
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
|
||||
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
|
||||
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
|
||||
if resp.ChecksumMode != minio.ChecksumFullObjectMode.String() {
|
||||
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", resp.ChecksumMode, minio.ChecksumFullObjectMode.String()))
|
||||
}
|
||||
@@ -2191,6 +2222,11 @@ func testPutObjectWithChecksums() {
|
||||
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
|
||||
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
|
||||
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
|
||||
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
|
||||
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
|
||||
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
|
||||
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
|
||||
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
|
||||
if st.ChecksumMode != minio.ChecksumFullObjectMode.String() {
|
||||
logError(testName, function, args, startTime, "", "Checksum mode is not full object", fmt.Errorf("got %s, want %s", st.ChecksumMode, minio.ChecksumFullObjectMode.String()))
|
||||
}
|
||||
@@ -2238,6 +2274,11 @@ func testPutObjectWithChecksums() {
|
||||
cmpChecksum(st.ChecksumCRC32, "")
|
||||
cmpChecksum(st.ChecksumCRC32C, "")
|
||||
cmpChecksum(st.ChecksumCRC64NVME, "")
|
||||
cmpChecksum(st.ChecksumMD5, "")
|
||||
cmpChecksum(st.ChecksumSHA512, "")
|
||||
cmpChecksum(st.ChecksumXXHash64, "")
|
||||
cmpChecksum(st.ChecksumXXHash3, "")
|
||||
cmpChecksum(st.ChecksumXXHash128, "")
|
||||
|
||||
delete(args, "range")
|
||||
delete(args, "metadata")
|
||||
@@ -2283,6 +2324,11 @@ func testPutObjectWithTrailingChecksums() {
|
||||
tests := []struct {
|
||||
cs minio.ChecksumType
|
||||
}{
|
||||
{cs: minio.ChecksumMD5},
|
||||
{cs: minio.ChecksumSHA512},
|
||||
{cs: minio.ChecksumXXHash64},
|
||||
{cs: minio.ChecksumXXHash3},
|
||||
{cs: minio.ChecksumXXHash128},
|
||||
{cs: minio.ChecksumCRC64NVME},
|
||||
{cs: minio.ChecksumCRC32C},
|
||||
{cs: minio.ChecksumCRC32},
|
||||
@@ -2329,6 +2375,10 @@ func testPutObjectWithTrailingChecksums() {
|
||||
Checksum: test.cs,
|
||||
})
|
||||
if err != nil {
|
||||
if ignore2026ChecksumError(test.cs, err) {
|
||||
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
|
||||
continue
|
||||
}
|
||||
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
||||
return
|
||||
}
|
||||
@@ -2341,6 +2391,11 @@ func testPutObjectWithTrailingChecksums() {
|
||||
cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"])
|
||||
cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
|
||||
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
|
||||
cmpChecksum(resp.ChecksumMD5, meta["x-amz-checksum-md5"])
|
||||
cmpChecksum(resp.ChecksumSHA512, meta["x-amz-checksum-sha512"])
|
||||
cmpChecksum(resp.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
|
||||
cmpChecksum(resp.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
|
||||
cmpChecksum(resp.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
|
||||
|
||||
// Read the data back
|
||||
gopts := minio.GetObjectOptions{Checksum: true}
|
||||
@@ -2361,7 +2416,12 @@ func testPutObjectWithTrailingChecksums() {
|
||||
cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"])
|
||||
cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"])
|
||||
cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
|
||||
cmpChecksum(resp.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
|
||||
cmpChecksum(st.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
|
||||
cmpChecksum(st.ChecksumMD5, meta["x-amz-checksum-md5"])
|
||||
cmpChecksum(st.ChecksumSHA512, meta["x-amz-checksum-sha512"])
|
||||
cmpChecksum(st.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
|
||||
cmpChecksum(st.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
|
||||
cmpChecksum(st.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
|
||||
|
||||
if st.Size != int64(bufSize) {
|
||||
logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err)
|
||||
@@ -2407,6 +2467,11 @@ func testPutObjectWithTrailingChecksums() {
|
||||
cmpChecksum(st.ChecksumCRC32, "")
|
||||
cmpChecksum(st.ChecksumCRC32C, "")
|
||||
cmpChecksum(st.ChecksumCRC64NVME, "")
|
||||
cmpChecksum(st.ChecksumMD5, "")
|
||||
cmpChecksum(st.ChecksumSHA512, "")
|
||||
cmpChecksum(st.ChecksumXXHash64, "")
|
||||
cmpChecksum(st.ChecksumXXHash3, "")
|
||||
cmpChecksum(st.ChecksumXXHash128, "")
|
||||
|
||||
function = "GetObjectAttributes(...)"
|
||||
s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{})
|
||||
@@ -2418,6 +2483,16 @@ func testPutObjectWithTrailingChecksums() {
|
||||
cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"])
|
||||
cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"])
|
||||
cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"])
|
||||
cmpChecksum(s.Checksum.ChecksumCRC64NVME, meta["x-amz-checksum-crc64nvme"])
|
||||
cmpChecksum(s.Checksum.ChecksumMD5, meta["x-amz-checksum-md5"])
|
||||
cmpChecksum(s.Checksum.ChecksumSHA512, meta["x-amz-checksum-sha512"])
|
||||
cmpChecksum(s.Checksum.ChecksumXXHash64, meta["x-amz-checksum-xxhash64"])
|
||||
cmpChecksum(s.Checksum.ChecksumXXHash3, meta["x-amz-checksum-xxhash3"])
|
||||
cmpChecksum(s.Checksum.ChecksumXXHash128, meta["x-amz-checksum-xxhash128"])
|
||||
if s.Checksum.ChecksumType != "" && s.Checksum.ChecksumType != minio.ChecksumFullObjectMode.String() {
|
||||
logError(testName, function, args, startTime, "", "ChecksumType mismatch in GetObjectAttributes", fmt.Errorf("want %s, got %s", minio.ChecksumFullObjectMode.String(), s.Checksum.ChecksumType))
|
||||
return
|
||||
}
|
||||
|
||||
delete(args, "range")
|
||||
delete(args, "metadata")
|
||||
@@ -2495,6 +2570,11 @@ func testPutMultipartObjectWithChecksums() {
|
||||
tests := []struct {
|
||||
cs minio.ChecksumType
|
||||
}{
|
||||
{cs: minio.ChecksumMD5},
|
||||
{cs: minio.ChecksumSHA512},
|
||||
{cs: minio.ChecksumXXHash64},
|
||||
{cs: minio.ChecksumXXHash3},
|
||||
{cs: minio.ChecksumXXHash128},
|
||||
{cs: minio.ChecksumFullObjectCRC32},
|
||||
{cs: minio.ChecksumFullObjectCRC32C},
|
||||
{cs: minio.ChecksumCRC64NVME},
|
||||
@@ -2552,6 +2632,10 @@ func testPutMultipartObjectWithChecksums() {
|
||||
Checksum: cs,
|
||||
})
|
||||
if err != nil {
|
||||
if ignore2026ChecksumError(test.cs, err) {
|
||||
logIgnored(testName, function, args, startTime, "server does not support "+test.cs.String())
|
||||
continue
|
||||
}
|
||||
logError(testName, function, args, startTime, "", "PutObject failed", err)
|
||||
return
|
||||
}
|
||||
@@ -2567,6 +2651,16 @@ func testPutMultipartObjectWithChecksums() {
|
||||
cmpChecksum(resp.ChecksumSHA256, wantChksm)
|
||||
case minio.ChecksumCRC64NVME:
|
||||
cmpChecksum(resp.ChecksumCRC64NVME, wantChksm)
|
||||
case minio.ChecksumMD5:
|
||||
cmpChecksum(resp.ChecksumMD5, wantChksm)
|
||||
case minio.ChecksumSHA512:
|
||||
cmpChecksum(resp.ChecksumSHA512, wantChksm)
|
||||
case minio.ChecksumXXHash64:
|
||||
cmpChecksum(resp.ChecksumXXHash64, wantChksm)
|
||||
case minio.ChecksumXXHash3:
|
||||
cmpChecksum(resp.ChecksumXXHash3, wantChksm)
|
||||
case minio.ChecksumXXHash128:
|
||||
cmpChecksum(resp.ChecksumXXHash128, wantChksm)
|
||||
}
|
||||
|
||||
args["section"] = "HeadObject"
|
||||
@@ -2586,6 +2680,16 @@ func testPutMultipartObjectWithChecksums() {
|
||||
cmpChecksum(st.ChecksumSHA256, wantChksm)
|
||||
case minio.ChecksumCRC64NVME:
|
||||
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
|
||||
case minio.ChecksumMD5:
|
||||
cmpChecksum(st.ChecksumMD5, wantChksm)
|
||||
case minio.ChecksumSHA512:
|
||||
cmpChecksum(st.ChecksumSHA512, wantChksm)
|
||||
case minio.ChecksumXXHash64:
|
||||
cmpChecksum(st.ChecksumXXHash64, wantChksm)
|
||||
case minio.ChecksumXXHash3:
|
||||
cmpChecksum(st.ChecksumXXHash3, wantChksm)
|
||||
case minio.ChecksumXXHash128:
|
||||
cmpChecksum(st.ChecksumXXHash128, wantChksm)
|
||||
}
|
||||
|
||||
// Use the CopyObject API to make a copy, in the case it was a composite checksum,
|
||||
@@ -2622,6 +2726,16 @@ func testPutMultipartObjectWithChecksums() {
|
||||
cmpChecksum(st.ChecksumSHA256, wantFullObjectChksm)
|
||||
case minio.ChecksumCRC64NVME:
|
||||
cmpChecksum(st.ChecksumCRC64NVME, wantFullObjectChksm)
|
||||
case minio.ChecksumMD5:
|
||||
cmpChecksum(st.ChecksumMD5, wantFullObjectChksm)
|
||||
case minio.ChecksumSHA512:
|
||||
cmpChecksum(st.ChecksumSHA512, wantFullObjectChksm)
|
||||
case minio.ChecksumXXHash64:
|
||||
cmpChecksum(st.ChecksumXXHash64, wantFullObjectChksm)
|
||||
case minio.ChecksumXXHash3:
|
||||
cmpChecksum(st.ChecksumXXHash3, wantFullObjectChksm)
|
||||
case minio.ChecksumXXHash128:
|
||||
cmpChecksum(st.ChecksumXXHash128, wantFullObjectChksm)
|
||||
}
|
||||
|
||||
args["section"] = "GetObjectAttributes"
|
||||
@@ -2644,6 +2758,35 @@ func testPutMultipartObjectWithChecksums() {
|
||||
cmpChecksum(s.Checksum.ChecksumSHA1, wantChksm)
|
||||
case minio.ChecksumSHA256:
|
||||
cmpChecksum(s.Checksum.ChecksumSHA256, wantChksm)
|
||||
case minio.ChecksumCRC64NVME:
|
||||
cmpChecksum(s.Checksum.ChecksumCRC64NVME, wantChksm)
|
||||
case minio.ChecksumMD5:
|
||||
cmpChecksum(s.Checksum.ChecksumMD5, wantChksm)
|
||||
case minio.ChecksumSHA512:
|
||||
cmpChecksum(s.Checksum.ChecksumSHA512, wantChksm)
|
||||
case minio.ChecksumXXHash64:
|
||||
cmpChecksum(s.Checksum.ChecksumXXHash64, wantChksm)
|
||||
case minio.ChecksumXXHash3:
|
||||
cmpChecksum(s.Checksum.ChecksumXXHash3, wantChksm)
|
||||
case minio.ChecksumXXHash128:
|
||||
cmpChecksum(s.Checksum.ChecksumXXHash128, wantChksm)
|
||||
}
|
||||
|
||||
if s.Checksum.ChecksumType != "" {
|
||||
var wantType string
|
||||
if test.cs.FullObjectRequested() {
|
||||
wantType = minio.ChecksumFullObjectMode.String()
|
||||
} else {
|
||||
wantType = minio.ChecksumCompositeMode.String()
|
||||
}
|
||||
cmpChecksum(s.Checksum.ChecksumType, wantType)
|
||||
}
|
||||
|
||||
for _, part := range s.ObjectParts.Parts {
|
||||
if test.cs == minio.ChecksumCRC64NVME && part.ChecksumCRC64NVME == "" {
|
||||
logError(testName, function, args, startTime, "", "Part missing CRC64NVME checksum in GetObjectAttributes", fmt.Errorf("part %d", part.PartNumber))
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// Read the data back
|
||||
@@ -2684,6 +2827,16 @@ func testPutMultipartObjectWithChecksums() {
|
||||
if st.ChecksumCRC64NVME != "" {
|
||||
cmpChecksum(st.ChecksumCRC64NVME, wantChksm)
|
||||
}
|
||||
case minio.ChecksumMD5:
|
||||
cmpChecksum(st.ChecksumMD5, wantChksm)
|
||||
case minio.ChecksumSHA512:
|
||||
cmpChecksum(st.ChecksumSHA512, wantChksm)
|
||||
case minio.ChecksumXXHash64:
|
||||
cmpChecksum(st.ChecksumXXHash64, wantChksm)
|
||||
case minio.ChecksumXXHash3:
|
||||
cmpChecksum(st.ChecksumXXHash3, wantChksm)
|
||||
case minio.ChecksumXXHash128:
|
||||
cmpChecksum(st.ChecksumXXHash128, wantChksm)
|
||||
}
|
||||
|
||||
delete(args, "metadata")
|
||||
@@ -3475,6 +3628,8 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
|
||||
checksumFound = true
|
||||
} else if v.ChecksumCRC32C != "" {
|
||||
checksumFound = true
|
||||
} else if v.ChecksumCRC64NVME != "" {
|
||||
checksumFound = true
|
||||
}
|
||||
if !checksumFound {
|
||||
partsMissingChecksum = true
|
||||
@@ -3497,6 +3652,7 @@ func validateObjectAttributeRequest(OA *minio.ObjectAttributes, opts *minio.Obje
|
||||
|
||||
hasFullObjectChecksum := (OA.Checksum.ChecksumCRC32 != "" ||
|
||||
OA.Checksum.ChecksumCRC32C != "" ||
|
||||
OA.Checksum.ChecksumCRC64NVME != "" ||
|
||||
OA.Checksum.ChecksumSHA1 != "" ||
|
||||
OA.Checksum.ChecksumSHA256 != "")
|
||||
|
||||
@@ -14789,6 +14945,9 @@ func main() {
|
||||
|
||||
// execute tests
|
||||
if isFullMode() {
|
||||
testPutObjectWithChecksums()
|
||||
testPutObjectWithTrailingChecksums()
|
||||
testPutMultipartObjectWithChecksums()
|
||||
testCopyObjectWithChecksums()
|
||||
testReplaceObjectWithChecksums()
|
||||
testCorsSetGetDelete()
|
||||
@@ -14808,9 +14967,6 @@ func main() {
|
||||
testComposeObjectErrorCasesV2()
|
||||
testCompose10KSourcesV2()
|
||||
testUserMetadataCopyingV2()
|
||||
testPutObjectWithChecksums()
|
||||
testPutObjectWithTrailingChecksums()
|
||||
testPutMultipartObjectWithChecksums()
|
||||
testPutObject0ByteV2()
|
||||
testPutObjectMetadataNonUSASCIIV2()
|
||||
testPutObjectNoLengthV2()
|
||||
|
||||
+21
@@ -119,6 +119,10 @@ var elbAmazonCnRegex = regexp.MustCompile(`elb(.*?).amazonaws.com.cn$`)
|
||||
// amazonS3HostPrivateLink - regular expression used to determine if an arg is s3 host in AWS PrivateLink interface endpoints style
|
||||
var amazonS3HostPrivateLink = regexp.MustCompile(`^(?:bucket|accesspoint).vpce-.*?.s3.(.*?).vpce.amazonaws.com$`)
|
||||
|
||||
// amazonS3HostOutposts - regular expression used to determine if an arg is S3 on Outposts endpoint.
|
||||
// Pattern: <something>.s3-outposts.<region>.amazonaws.com
|
||||
var amazonS3HostOutposts = regexp.MustCompile(`^(.+)\.s3-outposts\.([a-z0-9-]+)\.amazonaws\.com$`)
|
||||
|
||||
// GetRegionFromURL - returns a region from url host.
|
||||
func GetRegionFromURL(endpointURL url.URL) string {
|
||||
if endpointURL == sentinelURL {
|
||||
@@ -181,6 +185,11 @@ func GetRegionFromURL(endpointURL url.URL) string {
|
||||
return parts[1]
|
||||
}
|
||||
|
||||
parts = amazonS3HostOutposts.FindStringSubmatch(endpointURL.Hostname())
|
||||
if len(parts) > 2 {
|
||||
return parts[2]
|
||||
}
|
||||
|
||||
parts = amazonS3HostDot.FindStringSubmatch(endpointURL.Hostname())
|
||||
if len(parts) > 1 {
|
||||
if strings.HasPrefix(parts[1], "xpress-") {
|
||||
@@ -210,8 +219,20 @@ func IsAmazonExpressZonalEndpoint(endpointURL url.URL) bool {
|
||||
return amazonS3HostExpress.MatchString(endpointURL.Hostname())
|
||||
}
|
||||
|
||||
// IsAmazonOutpostsEndpoint - Match if the endpoint is S3 on Outposts endpoint.
|
||||
func IsAmazonOutpostsEndpoint(endpointURL url.URL) bool {
|
||||
if endpointURL == sentinelURL {
|
||||
return false
|
||||
}
|
||||
return amazonS3HostOutposts.MatchString(endpointURL.Hostname())
|
||||
}
|
||||
|
||||
// IsAmazonEndpoint - Match if it is exactly Amazon S3 endpoint.
|
||||
// S3 on Outposts is not treated as Amazon S3 here so that the client keeps path-style and does not replace the host.
|
||||
func IsAmazonEndpoint(endpointURL url.URL) bool {
|
||||
if IsAmazonOutpostsEndpoint(endpointURL) {
|
||||
return false
|
||||
}
|
||||
if endpointURL.Hostname() == "s3-external-1.amazonaws.com" || endpointURL.Hostname() == "s3.amazonaws.com" {
|
||||
return true
|
||||
}
|
||||
|
||||
+80
-30
@@ -90,32 +90,34 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 {
|
||||
return streamLen
|
||||
}
|
||||
|
||||
// buildChunkStringToSign - returns the string to sign given chunk data
|
||||
// and previous signature.
|
||||
func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
|
||||
// buildChunkStringToSignWithService - like buildChunkStringToSign but with configurable service type.
|
||||
func buildChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
stringToSignParts := []string{
|
||||
streamingPayloadHdr,
|
||||
t.Format(iso8601DateFormat),
|
||||
getScope(region, t, ServiceTypeS3),
|
||||
getScope(region, t, serviceType),
|
||||
previousSig,
|
||||
emptySHA256,
|
||||
chunkChecksum,
|
||||
}
|
||||
|
||||
return strings.Join(stringToSignParts, "\n")
|
||||
}
|
||||
|
||||
// buildTrailerChunkStringToSign - returns the string to sign given chunk data
|
||||
// and previous signature.
|
||||
func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string {
|
||||
// buildTrailerChunkStringToSignWithService - like buildTrailerChunkStringToSign but with configurable service type.
|
||||
func buildTrailerChunkStringToSignWithService(t time.Time, region, previousSig, chunkChecksum, serviceType string) string {
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
stringToSignParts := []string{
|
||||
streamingTrailerHdr,
|
||||
t.Format(iso8601DateFormat),
|
||||
getScope(region, t, ServiceTypeS3),
|
||||
getScope(region, t, serviceType),
|
||||
previousSig,
|
||||
chunkChecksum,
|
||||
}
|
||||
|
||||
return strings.Join(stringToSignParts, "\n")
|
||||
}
|
||||
|
||||
@@ -150,36 +152,42 @@ func buildChunkHeader(chunkLen int64, signature string) []byte {
|
||||
}
|
||||
|
||||
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
|
||||
// serviceType defaults to ServiceTypeS3 when empty.
|
||||
func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region,
|
||||
previousSignature, secretAccessKey string,
|
||||
previousSignature, secretAccessKey, serviceType string,
|
||||
) string {
|
||||
chunkStringToSign := buildChunkStringToSign(reqTime, region,
|
||||
previousSignature, chunkCheckSum)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
chunkStringToSign := buildChunkStringToSignWithService(reqTime, region,
|
||||
previousSignature, chunkCheckSum, serviceType)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
|
||||
return getSignature(signingKey, chunkStringToSign)
|
||||
}
|
||||
|
||||
// buildChunkSignature - returns chunk signature for a given chunk and previous signature.
|
||||
// buildTrailerChunkSignature - returns chunk signature for trailer chunk.
|
||||
// serviceType defaults to ServiceTypeS3 when empty.
|
||||
func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region,
|
||||
previousSignature, secretAccessKey string,
|
||||
previousSignature, secretAccessKey, serviceType string,
|
||||
) string {
|
||||
chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region,
|
||||
previousSignature, chunkChecksum)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3)
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
chunkStringToSign := buildTrailerChunkStringToSignWithService(reqTime, region,
|
||||
previousSignature, chunkChecksum, serviceType)
|
||||
signingKey := getSigningKey(secretAccessKey, region, reqTime, serviceType)
|
||||
return getSignature(signingKey, chunkStringToSign)
|
||||
}
|
||||
|
||||
// getSeedSignature - returns the seed signature for a given request.
|
||||
func (s *StreamingReader) setSeedSignature(req *http.Request) {
|
||||
// Get canonical request
|
||||
serviceType := s.serviceType
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
canonicalRequest := getCanonicalRequest(*req, ignoredStreamingHeaders, getHashedPayload(*req))
|
||||
|
||||
// Get string to sign from canonical request.
|
||||
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, ServiceTypeS3)
|
||||
|
||||
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, ServiceTypeS3)
|
||||
|
||||
// Calculate signature.
|
||||
stringToSign := getStringToSignV4(s.reqTime, s.region, canonicalRequest, serviceType)
|
||||
signingKey := getSigningKey(s.secretAccessKey, s.region, s.reqTime, serviceType)
|
||||
s.seedSignature = getSignature(signingKey, stringToSign)
|
||||
}
|
||||
|
||||
@@ -190,6 +198,7 @@ type StreamingReader struct {
|
||||
secretAccessKey string
|
||||
sessionToken string
|
||||
region string
|
||||
serviceType string // e.g. ServiceTypeS3, ServiceTypeS3Outposts; empty means S3
|
||||
prevSignature string
|
||||
seedSignature string
|
||||
contentLen int64 // Content-Length from req header
|
||||
@@ -214,8 +223,12 @@ func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) {
|
||||
s.sh256.Write(s.chunkBuf[:chunkLen])
|
||||
chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil))
|
||||
|
||||
serviceType := s.serviceType
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
signature := buildChunkSignature(chunckChecksum, s.reqTime,
|
||||
s.region, s.prevSignature, s.secretAccessKey)
|
||||
s.region, s.prevSignature, s.secretAccessKey, serviceType)
|
||||
|
||||
// For next chunk signature computation
|
||||
s.prevSignature = signature
|
||||
@@ -249,9 +262,12 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) {
|
||||
s.sh256.Reset()
|
||||
s.sh256.Write(s.chunkBuf)
|
||||
chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil))
|
||||
// Compute chunk signature
|
||||
serviceType := s.serviceType
|
||||
if serviceType == "" {
|
||||
serviceType = ServiceTypeS3
|
||||
}
|
||||
signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime,
|
||||
s.region, s.prevSignature, s.secretAccessKey)
|
||||
s.region, s.prevSignature, s.secretAccessKey, serviceType)
|
||||
|
||||
// For next chunk signature computation
|
||||
s.prevSignature = signature
|
||||
@@ -376,6 +392,40 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok
|
||||
return req
|
||||
}
|
||||
|
||||
// StreamingSignV4Outposts - provides chunked upload signatureV4 support for S3 on Outposts (service name s3-outposts).
|
||||
func StreamingSignV4Outposts(req *http.Request, accessKeyID, secretAccessKey, sessionToken,
|
||||
region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher,
|
||||
) *http.Request {
|
||||
prepareStreamingRequest(req, sessionToken, dataLen, reqTime)
|
||||
if req.Body == nil {
|
||||
req.Body = io.NopCloser(bytes.NewReader([]byte("")))
|
||||
}
|
||||
stReader := &StreamingReader{
|
||||
baseReadCloser: req.Body,
|
||||
accessKeyID: accessKeyID,
|
||||
secretAccessKey: secretAccessKey,
|
||||
sessionToken: sessionToken,
|
||||
region: region,
|
||||
serviceType: ServiceTypeS3Outposts,
|
||||
reqTime: reqTime,
|
||||
chunkBuf: make([]byte, payloadChunkSize),
|
||||
contentLen: dataLen,
|
||||
chunkNum: 1,
|
||||
totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1,
|
||||
lastChunkSize: int(dataLen % payloadChunkSize),
|
||||
sh256: sh256,
|
||||
}
|
||||
if len(req.Trailer) > 0 {
|
||||
stReader.trailer = req.Trailer
|
||||
req.Trailer = nil
|
||||
}
|
||||
stReader.setSeedSignature(req)
|
||||
stReader.setStreamingAuthHeader(req, ServiceTypeS3Outposts)
|
||||
stReader.prevSignature = stReader.seedSignature
|
||||
req.Body = stReader
|
||||
return req
|
||||
}
|
||||
|
||||
// Read - this method performs chunk upload signature providing a
|
||||
// io.Reader interface.
|
||||
func (s *StreamingReader) Read(buf []byte) (int, error) {
|
||||
|
||||
+51
-3
@@ -38,9 +38,10 @@ const (
|
||||
|
||||
// Different service types
|
||||
const (
|
||||
ServiceTypeS3 = "s3"
|
||||
ServiceTypeSTS = "sts"
|
||||
ServiceTypeS3Express = "s3express"
|
||||
ServiceTypeS3 = "s3"
|
||||
ServiceTypeSTS = "sts"
|
||||
ServiceTypeS3Express = "s3express"
|
||||
ServiceTypeS3Outposts = "s3-outposts"
|
||||
)
|
||||
|
||||
// Excerpts from @lsegal -
|
||||
@@ -256,6 +257,38 @@ func PreSignV4(req http.Request, accessKeyID, secretAccessKey, sessionToken, loc
|
||||
return &req
|
||||
}
|
||||
|
||||
// PreSignV4Outposts presign the request for S3 on Outposts (service name s3-outposts).
|
||||
func PreSignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, expires int64) *http.Request {
|
||||
// Presign is not needed for anonymous credentials.
|
||||
if accessKeyID == "" || secretAccessKey == "" {
|
||||
return &req
|
||||
}
|
||||
|
||||
t := time.Now().UTC()
|
||||
credential := GetCredential(accessKeyID, location, t, ServiceTypeS3Outposts)
|
||||
signedHeaders := getSignedHeaders(req, v4IgnoredHeaders)
|
||||
query := req.URL.Query()
|
||||
query.Set("X-Amz-Algorithm", signV4Algorithm)
|
||||
query.Set("X-Amz-Date", t.Format(iso8601DateFormat))
|
||||
query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10))
|
||||
query.Set("X-Amz-SignedHeaders", signedHeaders)
|
||||
query.Set("X-Amz-Credential", credential)
|
||||
if sessionToken != "" {
|
||||
if v := req.Header.Get("x-amz-s3session-token"); v != "" {
|
||||
query.Set("X-Amz-S3session-Token", sessionToken)
|
||||
} else {
|
||||
query.Set("X-Amz-Security-Token", sessionToken)
|
||||
}
|
||||
}
|
||||
req.URL.RawQuery = query.Encode()
|
||||
canonicalRequest := getCanonicalRequest(req, v4IgnoredHeaders, getHashedPayload(req))
|
||||
stringToSign := getStringToSignV4(t, location, canonicalRequest, ServiceTypeS3Outposts)
|
||||
signingKey := getSigningKey(secretAccessKey, location, t, ServiceTypeS3Outposts)
|
||||
signature := getSignature(signingKey, stringToSign)
|
||||
req.URL.RawQuery += "&X-Amz-Signature=" + signature
|
||||
return &req
|
||||
}
|
||||
|
||||
// PostPresignSignatureV4 - presigned signature for PostPolicy
|
||||
// requests.
|
||||
func PostPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string {
|
||||
@@ -393,3 +426,18 @@ func SignV4TrailerExpress(req http.Request, accessKeyID, secretAccessKey, sessio
|
||||
func SignV4Trailer(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
|
||||
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3, trailer)
|
||||
}
|
||||
|
||||
// SignV4Outposts sign the request for S3 on Outposts (service name s3-outposts).
|
||||
func SignV4Outposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string) *http.Request {
|
||||
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, nil)
|
||||
}
|
||||
|
||||
// SignV4WithServiceType signs a request with AWS Signature Version 4 using a custom service type.
|
||||
func SignV4WithServiceType(req http.Request, accessKeyID, secretAccessKey, sessionToken, location, serviceType string) *http.Request {
|
||||
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, serviceType, nil)
|
||||
}
|
||||
|
||||
// SignV4TrailerOutposts sign the request with trailer for S3 on Outposts (service name s3-outposts).
|
||||
func SignV4TrailerOutposts(req http.Request, accessKeyID, secretAccessKey, sessionToken, location string, trailer http.Header) *http.Request {
|
||||
return signV4(req, accessKeyID, secretAccessKey, sessionToken, location, ServiceTypeS3Outposts, trailer)
|
||||
}
|
||||
|
||||
+21
@@ -18,6 +18,7 @@
|
||||
package tags
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"encoding/xml"
|
||||
"io"
|
||||
"net/url"
|
||||
@@ -293,6 +294,26 @@ func (tags Tags) ToMap() map[string]string {
|
||||
return tags.TagSet.toMap()
|
||||
}
|
||||
|
||||
// MarshalJSON encodes Tags as a flat JSON object {"key":"value",...}.
|
||||
func (tags Tags) MarshalJSON() ([]byte, error) {
|
||||
return json.Marshal(tags.ToMap())
|
||||
}
|
||||
|
||||
// UnmarshalJSON decodes a flat JSON object {"key":"value",...} into Tags.
|
||||
func (tags *Tags) UnmarshalJSON(data []byte) error {
|
||||
var m map[string]string
|
||||
if err := json.Unmarshal(data, &m); err != nil {
|
||||
return err
|
||||
}
|
||||
if tags.TagSet == nil {
|
||||
tags.TagSet = &tagSet{
|
||||
tagMap: make(map[string]string),
|
||||
}
|
||||
}
|
||||
tags.TagSet.tagMap = m
|
||||
return nil
|
||||
}
|
||||
|
||||
// MapToObjectTags converts an input map of key and value into
|
||||
// *Tags data structure with validation.
|
||||
func MapToObjectTags(tagMap map[string]string) (*Tags, error) {
|
||||
|
||||
+27
@@ -179,6 +179,11 @@ func isValidEndpointURL(endpointURL url.URL) error {
|
||||
return errInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
|
||||
}
|
||||
}
|
||||
if strings.Contains(host, "s3-outposts") {
|
||||
if !s3utils.IsAmazonOutpostsEndpoint(endpointURL) {
|
||||
return errInvalidArgument("S3 Outposts endpoint must match <prefix>.s3-outposts.<region>.amazonaws.com")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -304,6 +309,21 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
|
||||
Region: h.Get("x-amz-bucket-region"),
|
||||
}
|
||||
}
|
||||
mtimeStr := h.Get("X-Minio-Source-Mtime")
|
||||
if mtimeStr != "" {
|
||||
mtime, err = time.Parse(time.RFC3339Nano, mtimeStr)
|
||||
if err != nil {
|
||||
return ObjectInfo{}, ErrorResponse{
|
||||
Code: InternalError,
|
||||
Message: fmt.Sprintf("X-Minio-Source-Mtime is not in supported format: %v", err),
|
||||
BucketName: bucketName,
|
||||
Key: objectName,
|
||||
RequestID: h.Get("x-amz-request-id"),
|
||||
HostID: h.Get("x-amz-id-2"),
|
||||
Region: h.Get("x-amz-bucket-region"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Fetch content type if any present.
|
||||
contentType := strings.TrimSpace(h.Get("Content-Type"))
|
||||
@@ -381,6 +401,7 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
|
||||
Size: size,
|
||||
LastModified: mtime,
|
||||
ContentType: contentType,
|
||||
ContentEncoding: strings.TrimSpace(h.Get("Content-Encoding")),
|
||||
Expires: expiry,
|
||||
VersionID: h.Get(amzVersionID),
|
||||
IsDeleteMarker: deleteMarker,
|
||||
@@ -402,6 +423,12 @@ func ToObjectInfo(bucketName, objectName string, h http.Header) (ObjectInfo, err
|
||||
ChecksumSHA1: h.Get(ChecksumSHA1.Key()),
|
||||
ChecksumSHA256: h.Get(ChecksumSHA256.Key()),
|
||||
ChecksumCRC64NVME: h.Get(ChecksumCRC64NVME.Key()),
|
||||
ChecksumMD5: h.Get(ChecksumMD5.Key()),
|
||||
ChecksumSHA512: h.Get(ChecksumSHA512.Key()),
|
||||
ChecksumXXHash64: h.Get(ChecksumXXHash64.Key()),
|
||||
ChecksumXXHash3: h.Get(ChecksumXXHash3.Key()),
|
||||
ChecksumXXHash128: h.Get(ChecksumXXHash128.Key()),
|
||||
ChecksumAlgorithm: h.Get(amzChecksumAlgo),
|
||||
ChecksumMode: h.Get(ChecksumFullObjectMode.Key()),
|
||||
}, nil
|
||||
}
|
||||
|
||||
+33
-15
@@ -696,7 +696,7 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
|
||||
// If we are here we have an auth callout defined and we have failed auth so far
|
||||
// so we will callout to our auth backend for processing.
|
||||
if !skip {
|
||||
authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy)
|
||||
authorized, reason = s.processClientOrLeafCallout(c, opts, proxyRequired, trustedProxy, ujwt)
|
||||
}
|
||||
// Check if we are authorized and in the auth callout account, and if so add in deny publish permissions for the auth subject.
|
||||
if authorized {
|
||||
@@ -797,26 +797,42 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
|
||||
token = opts.Authorization
|
||||
}
|
||||
|
||||
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
|
||||
if s.trustedKeys != nil {
|
||||
ujwt = c.opts.JWT
|
||||
if ujwt == _EMPTY_ && c.isMqtt() {
|
||||
// For MQTT, we pass the password as the JWT too, but do so here so it's not
|
||||
// publicly exposed in the client options if it isn't a JWT.
|
||||
// MQTT can carry JWTs in the password field. Reconstruct it here for auth
|
||||
// processing and auth callout, but do not populate c.opts.JWT yet or it would
|
||||
// be exposed through monitoring and advisory paths even when the password is
|
||||
// not actually a JWT.
|
||||
if ujwt == _EMPTY_ && c.isMqtt() && c.opts.JWT == _EMPTY_ {
|
||||
// Don't set juc here, leave that to the next s.trustedKeys != nil block,
|
||||
// so that we don't try to trust a JWT when we aren't in operator mode. We
|
||||
// will allow it to be passed through auth callout though.
|
||||
if _, err := jwt.DecodeUserClaims(c.opts.Password); err == nil {
|
||||
ujwt = c.opts.Password
|
||||
}
|
||||
if ujwt == _EMPTY_ && opts.DefaultSentinel != _EMPTY_ {
|
||||
c.opts.JWT = opts.DefaultSentinel
|
||||
ujwt = c.opts.JWT
|
||||
}
|
||||
|
||||
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
|
||||
if s.trustedKeys != nil {
|
||||
if ujwt == _EMPTY_ {
|
||||
// Need to be sure that it's a NATS JWT, otherwise we will not correctly
|
||||
// attempt the default sentinel below.
|
||||
if _, err = jwt.DecodeUserClaims(c.opts.JWT); err == nil {
|
||||
ujwt = c.opts.JWT
|
||||
}
|
||||
}
|
||||
if ujwt == _EMPTY_ {
|
||||
// Didn't fall through with a valid NATS JWT, so try the default sentinel
|
||||
// if configured.
|
||||
if opts.DefaultSentinel != _EMPTY_ {
|
||||
c.opts.JWT = opts.DefaultSentinel
|
||||
ujwt = c.opts.JWT
|
||||
}
|
||||
}
|
||||
if ujwt == _EMPTY_ {
|
||||
s.mu.Unlock()
|
||||
c.Debugf("Authentication requires a user JWT")
|
||||
return false
|
||||
}
|
||||
// So we have a valid user jwt here.
|
||||
juc, err = jwt.DecodeUserClaims(ujwt)
|
||||
if err != nil {
|
||||
if juc, err = jwt.DecodeUserClaims(ujwt); err != nil {
|
||||
s.mu.Unlock()
|
||||
c.Debugf("User JWT not valid: %v", err)
|
||||
return false
|
||||
@@ -1015,8 +1031,10 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
|
||||
c.Debugf("Connection type not allowed")
|
||||
return false
|
||||
}
|
||||
// skip validation of nonce when presented with a bearer token
|
||||
// FIXME: if BearerToken is only for WSS, need check for server with that port enabled
|
||||
// Skip validation of nonce when presented with a bearer token.
|
||||
// While support for bearer tokens was added for WebSockets, there is no
|
||||
// security benefit in restricting their use to that client protocol: the
|
||||
// client can just go use the other protocol.
|
||||
if !juc.BearerToken {
|
||||
// Verify the signature against the nonce.
|
||||
if c.opts.Sig == _EMPTY_ {
|
||||
|
||||
+10
-4
@@ -41,7 +41,7 @@ func titleCase(m string) string {
|
||||
}
|
||||
|
||||
// Process a callout on this client's behalf.
|
||||
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool) (authorized bool, errStr string) {
|
||||
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool, ujwt string) (authorized bool, errStr string) {
|
||||
isOperatorMode := len(opts.TrustedKeys) > 0
|
||||
|
||||
// this is the account the user connected in, or the one running the callout
|
||||
@@ -374,7 +374,7 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequi
|
||||
// Grab client info for the request.
|
||||
c.mu.Lock()
|
||||
c.fillClientInfo(&claim.ClientInformation)
|
||||
c.fillConnectOpts(&claim.ConnectOptions)
|
||||
c.fillConnectOpts(&claim.ConnectOptions, ujwt)
|
||||
// If we have a sig in the client opts, fill in nonce.
|
||||
if claim.ConnectOptions.SignedNonce != _EMPTY_ {
|
||||
claim.ClientInformation.Nonce = string(c.nonce)
|
||||
@@ -474,16 +474,22 @@ func (c *client) fillClientInfo(ci *jwt.ClientInformation) {
|
||||
|
||||
// Fill in client options.
|
||||
// Lock should be held.
|
||||
func (c *client) fillConnectOpts(opts *jwt.ConnectOptions) {
|
||||
func (c *client) fillConnectOpts(opts *jwt.ConnectOptions, ujwt string) {
|
||||
if c == nil || (c.kind != CLIENT && c.kind != LEAF && c.kind != JETSTREAM && c.kind != ACCOUNT) {
|
||||
return
|
||||
}
|
||||
|
||||
o := c.opts
|
||||
if ujwt == _EMPTY_ {
|
||||
// The caller may supply a reconstructed JWT that should be sent to auth
|
||||
// callout without storing it in c.opts.JWT. If not, fall back to the client
|
||||
// option as before.
|
||||
ujwt = o.JWT
|
||||
}
|
||||
|
||||
// Do it this way to fail to compile if fields are added to jwt.ClientInformation.
|
||||
*opts = jwt.ConnectOptions{
|
||||
JWT: o.JWT,
|
||||
JWT: ujwt,
|
||||
Nkey: o.Nkey,
|
||||
SignedNonce: o.Sig,
|
||||
Token: o.Token,
|
||||
|
||||
+2
-2
@@ -239,7 +239,7 @@ func (ss SequenceSet) EncodeLen() int {
|
||||
return minLen + (ss.Nodes() * ((numBuckets+1)*8 + 2))
|
||||
}
|
||||
|
||||
func (ss SequenceSet) Encode(buf []byte) ([]byte, error) {
|
||||
func (ss SequenceSet) Encode(buf []byte) []byte {
|
||||
nn, encLen := ss.Nodes(), ss.EncodeLen()
|
||||
|
||||
if cap(buf) < encLen {
|
||||
@@ -268,7 +268,7 @@ func (ss SequenceSet) Encode(buf []byte) ([]byte, error) {
|
||||
le.PutUint16(buf[i:], uint16(n.h))
|
||||
i += 2
|
||||
})
|
||||
return buf[:i], nil
|
||||
return buf[:i]
|
||||
}
|
||||
|
||||
// ErrBadEncoding is returned when we can not decode properly.
|
||||
|
||||
+191
-122
@@ -1061,18 +1061,19 @@ func (c *client) setPermissions(perms *Permissions) {
|
||||
return
|
||||
}
|
||||
c.perms = &permissions{}
|
||||
slcache := c.srv != nil && !c.srv.getOpts().NoSublistCache
|
||||
|
||||
// Loop over publish permissions
|
||||
if perms.Publish != nil {
|
||||
if perms.Publish.Allow != nil {
|
||||
c.perms.pub.allow = NewSublistWithCache()
|
||||
c.perms.pub.allow = NewSublist(slcache)
|
||||
}
|
||||
for _, pubSubject := range perms.Publish.Allow {
|
||||
sub := &subscription{subject: []byte(pubSubject)}
|
||||
c.perms.pub.allow.Insert(sub)
|
||||
}
|
||||
if len(perms.Publish.Deny) > 0 {
|
||||
c.perms.pub.deny = NewSublistWithCache()
|
||||
c.perms.pub.deny = NewSublist(slcache)
|
||||
}
|
||||
for _, pubSubject := range perms.Publish.Deny {
|
||||
sub := &subscription{subject: []byte(pubSubject)}
|
||||
@@ -1091,7 +1092,7 @@ func (c *client) setPermissions(perms *Permissions) {
|
||||
if perms.Subscribe != nil {
|
||||
var err error
|
||||
if len(perms.Subscribe.Allow) > 0 {
|
||||
c.perms.sub.allow = NewSublistWithCache()
|
||||
c.perms.sub.allow = NewSublist(slcache)
|
||||
}
|
||||
for _, subSubject := range perms.Subscribe.Allow {
|
||||
sub := &subscription{}
|
||||
@@ -1103,7 +1104,7 @@ func (c *client) setPermissions(perms *Permissions) {
|
||||
c.perms.sub.allow.Insert(sub)
|
||||
}
|
||||
if len(perms.Subscribe.Deny) > 0 {
|
||||
c.perms.sub.deny = NewSublistWithCache()
|
||||
c.perms.sub.deny = NewSublist(slcache)
|
||||
// Also hold onto this array for later.
|
||||
c.darray = perms.Subscribe.Deny
|
||||
}
|
||||
@@ -1200,6 +1201,7 @@ func (c *client) mergeDenyPermissions(what denyType, denyPubs []string) {
|
||||
if c.perms == nil {
|
||||
c.perms = &permissions{}
|
||||
}
|
||||
slcache := c.srv != nil && !c.srv.getOpts().NoSublistCache
|
||||
var perms []*perm
|
||||
switch what {
|
||||
case pub:
|
||||
@@ -1211,7 +1213,7 @@ func (c *client) mergeDenyPermissions(what denyType, denyPubs []string) {
|
||||
}
|
||||
for _, p := range perms {
|
||||
if p.deny == nil {
|
||||
p.deny = NewSublistWithCache()
|
||||
p.deny = NewSublist(slcache)
|
||||
}
|
||||
FOR_DENY:
|
||||
for _, subj := range denyPubs {
|
||||
@@ -2254,12 +2256,20 @@ func (c *client) processConnect(arg []byte) error {
|
||||
// least ClientProtoInfo, we need to increment the following counter.
|
||||
// This is decremented when client is removed from the server's
|
||||
// clients map.
|
||||
if kind == CLIENT && proto >= ClientProtoInfo {
|
||||
if kind == CLIENT && proto >= ClientProtoInfo && firstConnect {
|
||||
srv.mu.Lock()
|
||||
srv.cproto++
|
||||
srv.mu.Unlock()
|
||||
}
|
||||
|
||||
// A second CONNECT may move the client into a different account via
|
||||
// checkAuthentication. Drop any previously-registered subscriptions
|
||||
// from the current account first so they don't leak in that account's
|
||||
// sublist after the client switches.
|
||||
if !firstConnect {
|
||||
c.clearAccountSubs(false)
|
||||
}
|
||||
|
||||
// Check for Auth
|
||||
if ok := srv.checkAuthentication(c); !ok {
|
||||
// We may fail here because we reached max limits on an account.
|
||||
@@ -3273,19 +3283,20 @@ func (c *client) canSubscribe(subject string, optQueue ...string) bool {
|
||||
r := c.perms.sub.deny.Match(subject)
|
||||
allowed = len(r.psubs) == 0
|
||||
|
||||
if queue != _EMPTY_ && len(r.qsubs) > 0 {
|
||||
if allowed && queue != _EMPTY_ && len(r.qsubs) > 0 {
|
||||
// If the queue appears in the deny list, then DO NOT allow.
|
||||
allowed = !queueMatches(queue, r.qsubs)
|
||||
}
|
||||
|
||||
// We use the actual subscription to signal us to spin up the deny mperms
|
||||
// and cache. We check if the subject is a wildcard that contains any of
|
||||
// and cache. We check if the subject is a wildcard that intersects any of
|
||||
// the deny clauses.
|
||||
// FIXME(dlc) - We could be smarter and track when these go away and remove.
|
||||
if allowed && c.mperms == nil && subjectHasWildcard(subject) {
|
||||
// Whip through the deny array and check if this wildcard subject is within scope.
|
||||
// Whip through the deny array and check if this wildcard subject can
|
||||
// overlap with any denied deliveries.
|
||||
for _, sub := range c.darray {
|
||||
if subjectIsSubsetMatch(sub, subject) {
|
||||
if SubjectsCollide(sub, subject) {
|
||||
c.loadMsgDenyFilter()
|
||||
break
|
||||
}
|
||||
@@ -3658,14 +3669,7 @@ func (c *client) deliverMsg(prodIsMQTT bool, sub *subscription, acc *Account, su
|
||||
|
||||
// Check if we are a leafnode and have perms to check.
|
||||
if client.kind == LEAF && client.perms != nil {
|
||||
var subjectToCheck []byte
|
||||
if subject[0] == '_' && bytes.HasPrefix(subject, []byte(gwReplyPrefix)) {
|
||||
subjectToCheck = subject[gwSubjectOffset:]
|
||||
} else if subject[0] == '$' && bytes.HasPrefix(subject, []byte(oldGWReplyPrefix)) {
|
||||
subjectToCheck = subject[oldGWReplyStart:]
|
||||
} else {
|
||||
subjectToCheck = subject
|
||||
}
|
||||
subjectToCheck, _ := getGWRoutedSubjectOrSelf(subject)
|
||||
if !client.pubAllowedFullCheck(string(subjectToCheck), true, true) {
|
||||
mt.addEgressEvent(client, sub, errMsgTracePubViolation)
|
||||
client.mu.Unlock()
|
||||
@@ -4068,7 +4072,7 @@ func (c *client) allowedMsgTraceDest(hdr []byte, hasLock bool) (string, bool) {
|
||||
return _EMPTY_, true
|
||||
}
|
||||
td := sliceHeader(MsgTraceDest, hdr)
|
||||
if len(td) == 0 {
|
||||
if len(td) == 0 || bytes.Equal(td, traceDestDisabledAsBytes) {
|
||||
return _EMPTY_, true
|
||||
}
|
||||
dest := bytesToString(td)
|
||||
@@ -4131,17 +4135,7 @@ func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bo
|
||||
if !hasLock {
|
||||
c.mu.Lock()
|
||||
}
|
||||
if resp := c.replies[subject]; resp != nil {
|
||||
resp.n++
|
||||
// Check if we have sent too many responses.
|
||||
if c.perms.resp.MaxMsgs > 0 && resp.n > c.perms.resp.MaxMsgs {
|
||||
delete(c.replies, subject)
|
||||
} else if c.perms.resp.Expires > 0 && time.Since(resp.t) > c.perms.resp.Expires {
|
||||
delete(c.replies, subject)
|
||||
} else {
|
||||
allowed = true
|
||||
}
|
||||
}
|
||||
allowed = c.responseAllowed(subject)
|
||||
if !hasLock {
|
||||
c.mu.Unlock()
|
||||
}
|
||||
@@ -4155,6 +4149,25 @@ func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bo
|
||||
return allowed
|
||||
}
|
||||
|
||||
// Returns true if this subject matches a tracked dynamic reply permission.
|
||||
// Lock must be held.
|
||||
func (c *client) responseAllowed(subject string) bool {
|
||||
if c.perms == nil || c.perms.resp == nil {
|
||||
return false
|
||||
}
|
||||
if resp := c.replies[subject]; resp != nil {
|
||||
resp.n++
|
||||
if c.perms.resp.MaxMsgs > 0 && resp.n > c.perms.resp.MaxMsgs {
|
||||
delete(c.replies, subject)
|
||||
} else if c.perms.resp.Expires > 0 && time.Since(resp.t) > c.perms.resp.Expires {
|
||||
delete(c.replies, subject)
|
||||
} else {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Test whether a reply subject is a service import reply.
|
||||
func isServiceReply(reply []byte) bool {
|
||||
// This function is inlined and checking this way is actually faster
|
||||
@@ -4162,16 +4175,51 @@ func isServiceReply(reply []byte) bool {
|
||||
return len(reply) > 3 && bytesToString(reply[:4]) == replyPrefix
|
||||
}
|
||||
|
||||
// Test whether a subject is a JetStream ACK.
|
||||
func isJSAckSubject(subject []byte) bool {
|
||||
return len(subject) > jsAckPreLen && bytesToString(subject[:jsAckPreLen]) == jsAckPre
|
||||
}
|
||||
|
||||
// jsAckDeliverIdx returns the byte offset of the `@` separator in an encoded
|
||||
// `$JS.ACK....@<deliver>` reply, or -1 if reply is not in that form. Stream,
|
||||
// consumer, and subject tokens may legally contain `@`, so we accept only the
|
||||
// first `@` that follows the eight dots of the JS ACK token:
|
||||
//
|
||||
// $JS.ACK.<stream>.<consumer>.<delivered>.<sseq>.<cseq>.<tm>.<pending>@<deliver>
|
||||
func jsAckDeliverIdx(reply []byte) int {
|
||||
if !isJSAckSubject(reply) {
|
||||
return -1
|
||||
}
|
||||
dots := 0
|
||||
for i, b := range reply {
|
||||
switch b {
|
||||
case '.':
|
||||
dots++
|
||||
case '@':
|
||||
if dots >= 8 {
|
||||
return i
|
||||
}
|
||||
}
|
||||
}
|
||||
return -1
|
||||
}
|
||||
|
||||
// replyHasJSAckSuffix reports whether reply is already in `$JS.ACK....@<deliver>`
|
||||
// form, so callers don't double-append the suffix on a re-entrant pass
|
||||
// (service-import or chained JS push).
|
||||
func replyHasJSAckSuffix(reply []byte) bool {
|
||||
return jsAckDeliverIdx(reply) != -1
|
||||
}
|
||||
|
||||
// Test whether a reply subject is a service import or a gateway routed reply.
|
||||
func isReservedReply(reply []byte) bool {
|
||||
if isServiceReply(reply) {
|
||||
return true
|
||||
}
|
||||
rLen := len(reply)
|
||||
// Faster to check with string([:]) than byte-by-byte
|
||||
if rLen > jsAckPreLen && bytesToString(reply[:jsAckPreLen]) == jsAckPre {
|
||||
if isJSAckSubject(reply) {
|
||||
return true
|
||||
} else if rLen > gwReplyPrefixLen && bytesToString(reply[:gwReplyPrefixLen]) == gwReplyPrefix {
|
||||
} else if len(reply) > gwReplyPrefixLen && bytesToString(reply[:gwReplyPrefixLen]) == gwReplyPrefix {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
@@ -4370,7 +4418,7 @@ func (c *client) processInboundClientMsg(msg []byte) (bool, bool) {
|
||||
// Now deal with gateways
|
||||
if c.srv.gateway.enabled {
|
||||
reply := c.pa.reply
|
||||
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 {
|
||||
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(reply) > 0 && !replyHasJSAckSuffix(reply) {
|
||||
reply = append(reply, '@')
|
||||
reply = append(reply, c.pa.deliver...)
|
||||
}
|
||||
@@ -4418,7 +4466,7 @@ func (c *client) handleGWReplyMap(msg []byte) bool {
|
||||
}
|
||||
if c.srv.gateway.enabled {
|
||||
reply := c.pa.reply
|
||||
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(c.pa.reply) > 0 {
|
||||
if len(c.pa.deliver) > 0 && c.kind == JETSTREAM && len(reply) > 0 && !replyHasJSAckSuffix(reply) {
|
||||
reply = append(reply, '@')
|
||||
reply = append(reply, c.pa.deliver...)
|
||||
}
|
||||
@@ -4531,7 +4579,8 @@ func (c *client) setHeader(key, value string, msg []byte) []byte {
|
||||
// Write original header if present.
|
||||
if c.pa.hdr > LEN_CR_LF {
|
||||
omi = c.pa.hdr
|
||||
hdr := removeHeaderIfPresent(msg[:c.pa.hdr-LEN_CR_LF], key)
|
||||
// Need to copy since we're removing the header in place.
|
||||
hdr := removeHeaderIfPresent(copyBytes(msg[:c.pa.hdr-LEN_CR_LF]), key)
|
||||
if len(hdr) == 0 {
|
||||
bb.WriteString(hdrLine)
|
||||
} else {
|
||||
@@ -4825,6 +4874,12 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
|
||||
// but the local server must replace it with the identity of the
|
||||
// authenticated leaf connection instead of trusting forwarded values.
|
||||
ci = c.getClientInfo(share)
|
||||
if hadPrevSi && cis != nil && cis.Reply != _EMPTY_ {
|
||||
ci.Reply = cis.Reply
|
||||
} else if bytes.HasSuffix(c.pa.reply, []byte(FastBatchSuffix)) {
|
||||
// Fast batch requires knowledge of the original reply subject.
|
||||
ci.Reply = bytesToString(c.pa.reply)
|
||||
}
|
||||
if hadPrevSi {
|
||||
ci.Service = acc.Name
|
||||
if !share && (si.share || isSysImport) {
|
||||
@@ -4843,6 +4898,10 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
|
||||
}
|
||||
} else if c.kind != LEAF || c.pa.hdr < 0 || len(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])) == 0 {
|
||||
ci = c.getClientInfo(share)
|
||||
// Fast batch requires knowledge of the original reply subject.
|
||||
if bytes.HasSuffix(c.pa.reply, []byte(FastBatchSuffix)) {
|
||||
ci.Reply = bytesToString(c.pa.reply)
|
||||
}
|
||||
// If we did not share but the imports destination is the system account add in the server and cluster info.
|
||||
if !share && isSysImport {
|
||||
c.addServerAndClusterInfo(ci)
|
||||
@@ -4902,8 +4961,7 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
|
||||
// We also need to disable the message trace headers so that
|
||||
// if the message is routed, it does not initialize tracing in the
|
||||
// remote.
|
||||
positions := disableTraceHeaders(c, msg)
|
||||
defer enableTraceHeaders(msg, positions)
|
||||
msg = c.setHeader(MsgTraceDest, MsgTraceDestDisabled, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -5037,21 +5095,9 @@ func (c *client) processMsgResults(acc *Account, r *SublistResult, msg, deliver,
|
||||
// Check for JetStream encoded reply subjects.
|
||||
// For now these will only be on $JS.ACK prefixed reply subjects.
|
||||
var remapped bool
|
||||
if len(creply) > 0 && c.kind != CLIENT && !isInternalClient(c.kind) && bytes.HasPrefix(creply, []byte(jsAckPre)) {
|
||||
if len(creply) > 0 && c.kind != CLIENT && !isInternalClient(c.kind) {
|
||||
// We need to rewrite the subject and the reply.
|
||||
// But, we must be careful that the stream name, consumer name, and subject can contain '@' characters.
|
||||
// JS ACK contains at least 8 dots, find the first @ after this prefix.
|
||||
// - $JS.ACK.<stream>.<consumer>.<delivered>.<sseq>.<cseq>.<tm>.<pending>
|
||||
counter := 0
|
||||
li := bytes.IndexFunc(creply, func(rn rune) bool {
|
||||
if rn == '.' {
|
||||
counter++
|
||||
} else if rn == '@' {
|
||||
return counter >= 8
|
||||
}
|
||||
return false
|
||||
})
|
||||
if li != -1 && li < len(creply)-1 {
|
||||
if li := jsAckDeliverIdx(creply); li != -1 && li < len(creply)-1 {
|
||||
remapped = true
|
||||
subj, creply = creply[li+1:], creply[:li]
|
||||
}
|
||||
@@ -5471,7 +5517,7 @@ sendToRoutesOrLeafs:
|
||||
// at the end of the reply subject if it exists. But only if this wasn't
|
||||
// already performed, otherwise we'd end up with a duplicate '@' suffix
|
||||
// resulting in a protocol error.
|
||||
if len(deliver) > 0 && len(reply) > 0 && !remapped {
|
||||
if len(deliver) > 0 && len(reply) > 0 && !remapped && !replyHasJSAckSuffix(reply) {
|
||||
reply = append(reply, '@')
|
||||
reply = append(reply, deliver...)
|
||||
}
|
||||
@@ -5754,11 +5800,12 @@ func (c *client) clearAuthTimer() bool {
|
||||
return stopped
|
||||
}
|
||||
|
||||
// We may reuse atmr for expiring user jwts,
|
||||
// so check connectReceived.
|
||||
// Track whether the parser should still enforce pre-CONNECT rules.
|
||||
// This is handshake state, not timer state, since some handshakes
|
||||
// use a different timer while still expecting CONNECT.
|
||||
// Lock assume held on entry.
|
||||
func (c *client) awaitingAuth() bool {
|
||||
return !c.flags.isSet(connectReceived) && c.atmr != nil
|
||||
return c.flags.isSet(expectConnect) && !c.flags.isSet(connectReceived)
|
||||
}
|
||||
|
||||
// This will set the atmr for the JWT expiration time.
|
||||
@@ -5987,37 +6034,12 @@ func (c *client) closeConnection(reason ClosedState) {
|
||||
srv = c.srv
|
||||
noReconnect = c.flags.isSet(noReconnect)
|
||||
acc = c.acc
|
||||
spoke bool
|
||||
)
|
||||
|
||||
// Snapshot for use if we are a client connection.
|
||||
// FIXME(dlc) - we can just stub in a new one for client
|
||||
// and reference existing one.
|
||||
var subs []*subscription
|
||||
if kind == CLIENT || kind == LEAF || kind == JETSTREAM {
|
||||
var _subs [32]*subscription
|
||||
subs = _subs[:0]
|
||||
// Do not set c.subs to nil or delete the sub from c.subs here because
|
||||
// it will be needed in saveClosedClient (which has been started as a
|
||||
// go routine in markConnAsClosed). Cleanup will be done there.
|
||||
for _, sub := range c.subs {
|
||||
// Auto-unsubscribe subscriptions must be unsubscribed forcibly.
|
||||
sub.max = 0
|
||||
sub.close()
|
||||
subs = append(subs, sub)
|
||||
}
|
||||
spoke = c.isSpokeLeafNode()
|
||||
}
|
||||
|
||||
c.mu.Unlock()
|
||||
|
||||
// Remove client's or leaf node or jetstream subscriptions.
|
||||
if acc != nil && (kind == CLIENT || kind == LEAF || kind == JETSTREAM) {
|
||||
acc.sl.RemoveBatch(subs)
|
||||
} else if kind == ROUTER {
|
||||
if kind == ROUTER {
|
||||
c.removeRemoteSubs()
|
||||
}
|
||||
|
||||
if srv != nil {
|
||||
// Unregister
|
||||
srv.removeClient(c)
|
||||
@@ -6025,45 +6047,11 @@ func (c *client) closeConnection(reason ClosedState) {
|
||||
if acc != nil {
|
||||
// Update remote subscriptions.
|
||||
if kind == CLIENT || kind == LEAF || kind == JETSTREAM {
|
||||
qsubs := map[string]*qsub{}
|
||||
for _, sub := range subs {
|
||||
// Call unsubscribe here to cleanup shadow subscriptions and such.
|
||||
c.unsubscribe(acc, sub, true, false)
|
||||
// Update route as normal for a normal subscriber.
|
||||
if sub.queue == nil {
|
||||
if !spoke {
|
||||
srv.updateRouteSubscriptionMap(acc, sub, -1)
|
||||
if srv.gateway.enabled {
|
||||
srv.gatewayUpdateSubInterest(acc.Name, sub, -1)
|
||||
}
|
||||
}
|
||||
acc.updateLeafNodes(sub, -1)
|
||||
} else {
|
||||
// We handle queue subscribers special in case we
|
||||
// have a bunch we can just send one update to the
|
||||
// connected routes.
|
||||
num := int32(1)
|
||||
if kind == LEAF {
|
||||
num = sub.qw
|
||||
}
|
||||
key := keyFromSub(sub)
|
||||
if esub, ok := qsubs[key]; ok {
|
||||
esub.n += num
|
||||
} else {
|
||||
qsubs[key] = &qsub{sub, num}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process any qsubs here.
|
||||
for _, esub := range qsubs {
|
||||
if !spoke {
|
||||
srv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n))
|
||||
if srv.gateway.enabled {
|
||||
srv.gatewayUpdateSubInterest(acc.Name, esub.sub, -(esub.n))
|
||||
}
|
||||
}
|
||||
acc.updateLeafNodes(esub.sub, -(esub.n))
|
||||
}
|
||||
// Remove client's subscriptions from the account and unregister
|
||||
// client from that account. Keep c.subs populated because
|
||||
// saveClosedClient (started as a goroutine in markConnAsClosed)
|
||||
// still needs to read it.
|
||||
c.clearAccountSubs(true)
|
||||
}
|
||||
// Always remove from the account, otherwise we can leak clients.
|
||||
// Note that SYSTEM and ACCOUNT types from above cleanup their own subs.
|
||||
@@ -6090,6 +6078,87 @@ func (c *client) closeConnection(reason ClosedState) {
|
||||
c.reconnect()
|
||||
}
|
||||
|
||||
// clearAccountSubs removes the client's subscriptions from its current account
|
||||
// and unregisters it from that account. If close is true, c.subs is left
|
||||
// populated for saveClosedClient; otherwise c.subs is cleared and c.acc
|
||||
// registered back to the global account.
|
||||
// Client lock MUST NOT be held on entry.
|
||||
func (c *client) clearAccountSubs(close bool) {
|
||||
c.mu.Lock()
|
||||
kind := c.kind
|
||||
srv := c.srv
|
||||
acc := c.acc
|
||||
if acc == nil || (kind != CLIENT && kind != LEAF && kind != JETSTREAM) {
|
||||
c.mu.Unlock()
|
||||
return
|
||||
}
|
||||
var _subs [32]*subscription
|
||||
subs := _subs[:0]
|
||||
// Do not set c.subs to nil or delete the sub from c.subs here because
|
||||
// it will be needed in saveClosedClient (which has been started as a
|
||||
// go routine in markConnAsClosed). Cleanup will be done there.
|
||||
for _, sub := range c.subs {
|
||||
// Auto-unsubscribe subscriptions must be unsubscribed forcibly.
|
||||
sub.max = 0
|
||||
sub.close()
|
||||
subs = append(subs, sub)
|
||||
if !close {
|
||||
delete(c.subs, string(sub.sid))
|
||||
}
|
||||
}
|
||||
spoke := c.isSpokeLeafNode()
|
||||
c.mu.Unlock()
|
||||
|
||||
acc.sl.RemoveBatch(subs)
|
||||
|
||||
if srv != nil {
|
||||
qsubs := map[string]*qsub{}
|
||||
for _, sub := range subs {
|
||||
// Call unsubscribe here to cleanup shadow subscriptions and such.
|
||||
c.unsubscribe(acc, sub, true, false)
|
||||
// Update route as normal for a normal subscriber.
|
||||
if sub.queue == nil {
|
||||
if !spoke {
|
||||
srv.updateRouteSubscriptionMap(acc, sub, -1)
|
||||
if srv.gateway.enabled {
|
||||
srv.gatewayUpdateSubInterest(acc.Name, sub, -1)
|
||||
}
|
||||
}
|
||||
acc.updateLeafNodes(sub, -1)
|
||||
} else {
|
||||
// We handle queue subscribers special in case we
|
||||
// have a bunch we can just send one update to the
|
||||
// connected routes.
|
||||
num := int32(1)
|
||||
if kind == LEAF {
|
||||
num = sub.qw
|
||||
}
|
||||
key := keyFromSub(sub)
|
||||
if esub, ok := qsubs[key]; ok {
|
||||
esub.n += num
|
||||
} else {
|
||||
qsubs[key] = &qsub{sub, num}
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process any qsubs here.
|
||||
for _, esub := range qsubs {
|
||||
if !spoke {
|
||||
srv.updateRouteSubscriptionMap(acc, esub.sub, -(esub.n))
|
||||
if srv.gateway.enabled {
|
||||
srv.gatewayUpdateSubInterest(acc.Name, esub.sub, -(esub.n))
|
||||
}
|
||||
}
|
||||
acc.updateLeafNodes(esub.sub, -(esub.n))
|
||||
}
|
||||
}
|
||||
|
||||
if !close {
|
||||
// Register back to global account, mimicking the state after client initialization.
|
||||
c.registerWithAccount(srv.globalAccount())
|
||||
}
|
||||
}
|
||||
|
||||
// Depending on the kind of connections, this may attempt to recreate a connection.
|
||||
// The actual reconnect attempt will be started in a go routine.
|
||||
func (c *client) reconnect() {
|
||||
@@ -6180,7 +6249,7 @@ func (c *client) reconnect() {
|
||||
srv.Debugf("Gateway %q not in configuration, not attempting reconnect", gwName)
|
||||
}
|
||||
} else if leafCfg != nil {
|
||||
// Check if this is a solicited leaf node. Start up a reconnect.
|
||||
// This is a solicited leaf node. Start up a reconnect.
|
||||
srv.startGoRoutine(func() { srv.reConnectToRemoteLeafNode(leafCfg) })
|
||||
}
|
||||
}
|
||||
|
||||
+1
-1
@@ -66,7 +66,7 @@ func init() {
|
||||
|
||||
const (
|
||||
// VERSION is the current version for the server.
|
||||
VERSION = "2.12.6"
|
||||
VERSION = "2.14.0"
|
||||
|
||||
// PROTO is the currently supported protocol.
|
||||
// 0 was the original
|
||||
|
||||
+563
-106
File diff suppressed because it is too large
Load Diff
+327
@@ -0,0 +1,327 @@
|
||||
// Copyright 2025 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//
|
||||
// Based on code from https://github.com/robfig/cron
|
||||
// Copyright (C) 2012 Rob Figueiredo
|
||||
// All Rights Reserved.
|
||||
//
|
||||
// MIT LICENSE
|
||||
//
|
||||
// Permission is hereby granted, free of charge, to any person obtaining a copy of
|
||||
// this software and associated documentation files (the "Software"), to deal in
|
||||
// the Software without restriction, including without limitation the rights to
|
||||
// use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
|
||||
// the Software, and to permit persons to whom the Software is furnished to do so,
|
||||
// subject to the following conditions:
|
||||
//
|
||||
// The above copyright notice and this permission notice shall be included in all
|
||||
// copies or substantial portions of the Software.
|
||||
//
|
||||
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
|
||||
// FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
|
||||
// COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
|
||||
// IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
|
||||
// CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// parseCron parses the given cron pattern and returns the next time it will fire based on the provided ts.
|
||||
func parseCron(pattern string, loc *time.Location, ts int64) (time.Time, error) {
|
||||
fields := strings.Fields(pattern)
|
||||
if len(fields) != 6 {
|
||||
return time.Time{}, fmt.Errorf("pattern requires 6 fields, got %d", len(fields))
|
||||
}
|
||||
|
||||
// If no time zone is passed, default to UTC.
|
||||
if loc == nil {
|
||||
loc = time.UTC
|
||||
}
|
||||
|
||||
// Parse each field.
|
||||
var err error
|
||||
var second, minute, hour, dayOfMonth, month, dayOfWeek uint64
|
||||
if second, err = getField(fields[0], seconds); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if minute, err = getField(fields[1], minutes); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if hour, err = getField(fields[2], hours); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if dayOfMonth, err = getField(fields[3], dom); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if month, err = getField(fields[4], months); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
if dayOfWeek, err = getField(fields[5], dow); err != nil {
|
||||
return time.Time{}, err
|
||||
}
|
||||
|
||||
// General approach
|
||||
//
|
||||
// For Month, Day, Hour, Minute, Second:
|
||||
// Check if the time value matches. If yes, continue to the next field.
|
||||
// If the field doesn't match the schedule, then increment the field until it matches.
|
||||
// While incrementing the field, a wrap-around brings it back to the beginning
|
||||
// of the field list (since it is necessary to re-verify previous field values)
|
||||
next := time.Unix(0, ts).In(loc)
|
||||
|
||||
// Start at the earliest possible time (the upcoming second).
|
||||
next = next.Truncate(time.Second).Add(time.Second)
|
||||
|
||||
// This flag indicates whether a field has been truncated at one point.
|
||||
truncated := false
|
||||
|
||||
// If no time is found within five years, return error.
|
||||
yearLimit := next.Year() + 5
|
||||
|
||||
WRAP:
|
||||
if next.Year() > yearLimit {
|
||||
return time.Time{}, errors.New("pattern exceeds maximum range")
|
||||
}
|
||||
for 1<<uint(next.Month())&month == 0 {
|
||||
if !truncated {
|
||||
truncated = true
|
||||
next = time.Date(next.Year(), next.Month(), 1, 0, 0, 0, 0, loc)
|
||||
}
|
||||
if next = next.AddDate(0, 1, 0); next.Month() == time.January {
|
||||
goto WRAP
|
||||
}
|
||||
}
|
||||
for !dayMatches(dayOfMonth, dayOfWeek, next) {
|
||||
if !truncated {
|
||||
truncated = true
|
||||
next = time.Date(next.Year(), next.Month(), next.Day(), 0, 0, 0, 0, loc)
|
||||
}
|
||||
if next = next.AddDate(0, 0, 1); next.Day() == 1 {
|
||||
goto WRAP
|
||||
}
|
||||
}
|
||||
for 1<<uint(next.Hour())&hour == 0 {
|
||||
if !truncated {
|
||||
truncated = true
|
||||
next = time.Date(next.Year(), next.Month(), next.Day(), next.Hour(), 0, 0, 0, loc)
|
||||
}
|
||||
if next = next.Add(time.Hour); next.Hour() == 0 {
|
||||
goto WRAP
|
||||
}
|
||||
}
|
||||
for 1<<uint(next.Minute())&minute == 0 {
|
||||
if !truncated {
|
||||
truncated = true
|
||||
next = next.Truncate(time.Minute)
|
||||
}
|
||||
if next = next.Add(time.Minute); next.Minute() == 0 {
|
||||
goto WRAP
|
||||
}
|
||||
}
|
||||
for 1<<uint(next.Second())&second == 0 {
|
||||
if !truncated {
|
||||
truncated = true
|
||||
next = next.Truncate(time.Second)
|
||||
}
|
||||
if next = next.Add(time.Second); next.Second() == 0 {
|
||||
goto WRAP
|
||||
}
|
||||
}
|
||||
return next, nil
|
||||
}
|
||||
|
||||
// getField returns an Int with the bits set representing all of the times that
|
||||
// the field represents or error parsing field value. A "field" is a comma-separated
|
||||
// list of "ranges".
|
||||
func getField(field string, r bounds) (uint64, error) {
|
||||
var bits uint64
|
||||
ranges := strings.FieldsFuncSeq(field, func(r rune) bool { return r == ',' })
|
||||
for expr := range ranges {
|
||||
bit, err := getRange(expr, r)
|
||||
if err != nil {
|
||||
return bits, err
|
||||
}
|
||||
bits |= bit
|
||||
}
|
||||
return bits, nil
|
||||
}
|
||||
|
||||
// getRange returns the bits indicated by the given expression: number | number [ "-" number ] [ "/" number ]
|
||||
// or error parsing range.
|
||||
func getRange(expr string, r bounds) (uint64, error) {
|
||||
var (
|
||||
start, end, step uint
|
||||
rangeAndStep = strings.Split(expr, "/")
|
||||
lowAndHigh = strings.Split(rangeAndStep[0], "-")
|
||||
singleDigit = len(lowAndHigh) == 1
|
||||
err error
|
||||
)
|
||||
|
||||
var extra uint64
|
||||
if lowAndHigh[0] == "*" || lowAndHigh[0] == "?" {
|
||||
start = r.min
|
||||
end = r.max
|
||||
extra = starBit
|
||||
} else {
|
||||
start, err = parseIntOrName(lowAndHigh[0], r.names)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
switch len(lowAndHigh) {
|
||||
case 1:
|
||||
end = start
|
||||
case 2:
|
||||
end, err = parseIntOrName(lowAndHigh[1], r.names)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
default:
|
||||
return 0, fmt.Errorf("too many hyphens: %s", expr)
|
||||
}
|
||||
}
|
||||
|
||||
switch len(rangeAndStep) {
|
||||
case 1:
|
||||
step = 1
|
||||
case 2:
|
||||
step, err = mustParseInt(rangeAndStep[1])
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// Special handling: "N/step" means "N-max/step".
|
||||
if singleDigit {
|
||||
end = r.max
|
||||
}
|
||||
if step > 1 {
|
||||
extra = 0
|
||||
}
|
||||
default:
|
||||
return 0, fmt.Errorf("too many slashes: %s", expr)
|
||||
}
|
||||
|
||||
if start < r.min {
|
||||
return 0, fmt.Errorf("beginning of range (%d) below minimum (%d): %s", start, r.min, expr)
|
||||
}
|
||||
if end > r.max {
|
||||
return 0, fmt.Errorf("end of range (%d) above maximum (%d): %s", end, r.max, expr)
|
||||
}
|
||||
if start > end {
|
||||
return 0, fmt.Errorf("beginning of range (%d) beyond end of range (%d): %s", start, end, expr)
|
||||
}
|
||||
if step == 0 {
|
||||
return 0, fmt.Errorf("step of range should be a positive number: %s", expr)
|
||||
}
|
||||
return getBits(start, end, step) | extra, nil
|
||||
}
|
||||
|
||||
// parseIntOrName returns the (possibly-named) integer contained in expr.
|
||||
func parseIntOrName(expr string, names map[string]uint) (uint, error) {
|
||||
if names != nil {
|
||||
if namedInt, ok := names[strings.ToLower(expr)]; ok {
|
||||
return namedInt, nil
|
||||
}
|
||||
}
|
||||
return mustParseInt(expr)
|
||||
}
|
||||
|
||||
// mustParseInt parses the given expression as an int or returns an error.
|
||||
func mustParseInt(expr string) (uint, error) {
|
||||
num, err := strconv.Atoi(expr)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse int from %s: %s", expr, err)
|
||||
}
|
||||
if num < 0 {
|
||||
return 0, fmt.Errorf("negative number (%d) not allowed: %s", num, expr)
|
||||
}
|
||||
return uint(num), nil
|
||||
}
|
||||
|
||||
// getBits sets all bits in the range [min, max], modulo the given step size.
|
||||
func getBits(min, max, step uint) uint64 {
|
||||
var bits uint64
|
||||
|
||||
// If step is 1, use shifts.
|
||||
if step == 1 {
|
||||
return ^(math.MaxUint64 << (max + 1)) & (math.MaxUint64 << min)
|
||||
}
|
||||
|
||||
// Else, use a simple loop.
|
||||
for i := min; i <= max; i += step {
|
||||
bits |= 1 << i
|
||||
}
|
||||
return bits
|
||||
}
|
||||
|
||||
// bounds provides a range of acceptable values (plus a map of name to value).
|
||||
type bounds struct {
|
||||
min, max uint
|
||||
names map[string]uint
|
||||
}
|
||||
|
||||
// The bounds for each field.
|
||||
var (
|
||||
seconds = bounds{0, 59, nil}
|
||||
minutes = bounds{0, 59, nil}
|
||||
hours = bounds{0, 23, nil}
|
||||
dom = bounds{1, 31, nil}
|
||||
months = bounds{1, 12, map[string]uint{
|
||||
"jan": 1,
|
||||
"feb": 2,
|
||||
"mar": 3,
|
||||
"apr": 4,
|
||||
"may": 5,
|
||||
"jun": 6,
|
||||
"jul": 7,
|
||||
"aug": 8,
|
||||
"sep": 9,
|
||||
"oct": 10,
|
||||
"nov": 11,
|
||||
"dec": 12,
|
||||
}}
|
||||
dow = bounds{0, 6, map[string]uint{
|
||||
"sun": 0,
|
||||
"mon": 1,
|
||||
"tue": 2,
|
||||
"wed": 3,
|
||||
"thu": 4,
|
||||
"fri": 5,
|
||||
"sat": 6,
|
||||
}}
|
||||
)
|
||||
|
||||
const (
|
||||
// Set the top bit if a star was included in the expression.
|
||||
starBit = 1 << 63
|
||||
)
|
||||
|
||||
// dayMatches returns true if the schedule's day-of-week and day-of-month
|
||||
// restrictions are satisfied by the given time.
|
||||
func dayMatches(dayOfMonth, dayOfWeek uint64, t time.Time) bool {
|
||||
var (
|
||||
domMatch = 1<<uint(t.Day())&dayOfMonth > 0
|
||||
dowMatch = 1<<uint(t.Weekday())&dayOfWeek > 0
|
||||
)
|
||||
if dayOfMonth&starBit > 0 || dayOfWeek&starBit > 0 {
|
||||
return domMatch && dowMatch
|
||||
}
|
||||
return domMatch || dowMatch
|
||||
}
|
||||
+211
-1
@@ -2008,5 +2008,215 @@
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSMessageSchedulesSourceInvalidErr",
|
||||
"code": 400,
|
||||
"error_code": 10203,
|
||||
"description": "message schedules source is invalid",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSConsumerInvalidResetErr",
|
||||
"code": 400,
|
||||
"error_code": 10204,
|
||||
"description": "invalid reset: {err}",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSBatchPublishDisabledErr",
|
||||
"code": 400,
|
||||
"error_code": 10205,
|
||||
"description": "batch publish is disabled",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSBatchPublishInvalidPatternErr",
|
||||
"code": 400,
|
||||
"error_code": 10206,
|
||||
"description": "batch publish pattern is invalid",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSBatchPublishInvalidBatchIDErr",
|
||||
"code": 400,
|
||||
"error_code": 10207,
|
||||
"description": "batch publish ID is invalid",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSBatchPublishUnknownBatchIDErr",
|
||||
"code": 400,
|
||||
"error_code": 10208,
|
||||
"description": "batch publish ID unknown",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSMirrorWithBatchPublishErr",
|
||||
"code": 400,
|
||||
"error_code": 10209,
|
||||
"description": "stream mirrors can not also use batch publishing",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSAtomicPublishTooManyInflight",
|
||||
"code": 429,
|
||||
"error_code": 10210,
|
||||
"description": "atomic publish too many inflight",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSBatchPublishTooManyInflight",
|
||||
"code": 429,
|
||||
"error_code": 10211,
|
||||
"description": "batch publish too many inflight",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSMessageSchedulesSchedulerInvalidErr",
|
||||
"code": 400,
|
||||
"error_code": 10212,
|
||||
"description": "message schedules invalid scheduler",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSMirrorDurableConsumerCfgInvalid",
|
||||
"code": 400,
|
||||
"error_code": 10213,
|
||||
"description": "stream mirror consumer config is invalid",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSMirrorConsumerRequiresAckFCErr",
|
||||
"code": 400,
|
||||
"error_code": 10214,
|
||||
"description": "stream mirror consumer requires flow control ack policy",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSSourceDurableConsumerCfgInvalid",
|
||||
"code": 400,
|
||||
"error_code": 10215,
|
||||
"description": "stream source consumer config is invalid",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSSourceDurableConsumerDuplicateDetected",
|
||||
"code": 400,
|
||||
"error_code": 10216,
|
||||
"description": "duplicate stream source consumer detected",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSSourceConsumerRequiresAckFCErr",
|
||||
"code": 400,
|
||||
"error_code": 10217,
|
||||
"description": "stream source consumer requires flow control ack policy",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSConsumerAckFCRequiresPushErr",
|
||||
"code": 400,
|
||||
"error_code": 10218,
|
||||
"description": "flow control ack policy requires a push based consumer",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSConsumerAckFCRequiresFCErr",
|
||||
"code": 400,
|
||||
"error_code": 10219,
|
||||
"description": "flow control ack policy requires flow control",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSConsumerAckFCRequiresMaxAckPendingErr",
|
||||
"code": 400,
|
||||
"error_code": 10220,
|
||||
"description": "flow control ack policy requires max ack pending",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSConsumerAckFCRequiresNoAckWaitErr",
|
||||
"code": 400,
|
||||
"error_code": 10221,
|
||||
"description": "flow control ack policy requires unset ack wait",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSConsumerAckFCRequiresNoMaxDeliverErr",
|
||||
"code": 400,
|
||||
"error_code": 10222,
|
||||
"description": "flow control ack policy requires unset max deliver",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
},
|
||||
{
|
||||
"constant": "JSMessageSchedulesTimeZoneInvalidErr",
|
||||
"code": 400,
|
||||
"error_code": 10223,
|
||||
"description": "message schedules time zone is invalid",
|
||||
"comment": "",
|
||||
"help": "",
|
||||
"url": "",
|
||||
"deprecates": ""
|
||||
}
|
||||
]
|
||||
]
|
||||
+21
-11
@@ -247,14 +247,15 @@ type ServerCapability uint64
|
||||
|
||||
// ServerInfo identifies remote servers.
|
||||
type ServerInfo struct {
|
||||
Name string `json:"name"`
|
||||
Host string `json:"host"`
|
||||
ID string `json:"id"`
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Version string `json:"ver"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
Name string `json:"name"`
|
||||
Host string `json:"host"`
|
||||
ID string `json:"id"`
|
||||
Cluster string `json:"cluster,omitempty"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Version string `json:"ver"`
|
||||
Tags []string `json:"tags,omitempty"`
|
||||
Metadata map[string]string `json:"metadata,omitempty"`
|
||||
FeatureFlags map[string]bool `json:"feature_flags,omitempty"`
|
||||
// Whether JetStream is enabled (deprecated in favor of the `ServerCapability`).
|
||||
JetStream bool `json:"jetstream"`
|
||||
// Generic capability flags
|
||||
@@ -328,6 +329,7 @@ type ClientInfo struct {
|
||||
ClientType string `json:"client_type,omitempty"`
|
||||
MQTTClient string `json:"client_id,omitempty"` // This is the MQTT client ID
|
||||
Nonce string `json:"nonce,omitempty"`
|
||||
Reply string `json:"reply,omitempty"` // Original reply subject after a service import (only when needed).
|
||||
}
|
||||
|
||||
// forAssignmentSnap returns the minimum amount of ClientInfo we need for assignment snapshots.
|
||||
@@ -518,7 +520,7 @@ RESET:
|
||||
|
||||
// Grab tags and metadata.
|
||||
opts := s.getOpts()
|
||||
tags, metadata := opts.Tags, opts.Metadata
|
||||
tags, metadata, featureFlags := opts.Tags, opts.Metadata, opts.getMergedFeatureFlags()
|
||||
|
||||
for s.eventsRunning() {
|
||||
select {
|
||||
@@ -536,6 +538,7 @@ RESET:
|
||||
si.Time = time.Now().UTC()
|
||||
si.Tags = tags
|
||||
si.Metadata = metadata
|
||||
si.FeatureFlags = featureFlags
|
||||
si.Flags = 0
|
||||
if js {
|
||||
// New capability based flags.
|
||||
@@ -1052,8 +1055,15 @@ func (s *Server) sendStatsz(subj string) {
|
||||
Size: mg.ClusterSize(),
|
||||
}
|
||||
}
|
||||
if ipq := s.jsAPIRoutedReqs; ipq != nil && jStat.Meta != nil {
|
||||
jStat.Meta.Pending = ipq.len()
|
||||
if jStat.Meta != nil {
|
||||
if ipq := s.jsAPIRoutedReqs; ipq != nil {
|
||||
jStat.Meta.PendingRequests = ipq.len()
|
||||
}
|
||||
if ipq := s.jsAPIRoutedInfoReqs; ipq != nil {
|
||||
jStat.Meta.PendingInfos = ipq.len()
|
||||
}
|
||||
jStat.Meta.Pending = jStat.Meta.PendingRequests + jStat.Meta.PendingInfos
|
||||
jStat.Meta.Snapshot = s.metaClusterSnapshotStats(js, mg)
|
||||
}
|
||||
}
|
||||
jStat.Limits = &s.getOpts().JetStreamLimits
|
||||
|
||||
+130
@@ -0,0 +1,130 @@
|
||||
// Copyright 2026 The NATS Authors
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
package server
|
||||
|
||||
import (
|
||||
"maps"
|
||||
"slices"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
FeatureFlagJsAckFormatV2 = "js_ack_fc_v2"
|
||||
FeatureFlagJsRaftDeleteRange = "js_raft_delete_range"
|
||||
)
|
||||
|
||||
var featureFlags = map[string]bool{
|
||||
// Use v2 format for `$JS.ACK.>` and `$JS.FC.>`.
|
||||
// - Introduced: 2.14.0, both v1 and v2 supported, only using v1.
|
||||
// - Enabled: TBD, both supported, v2 becomes the default.
|
||||
//
|
||||
// - v1: $JS.ACK.<stream name>.<consumer name>.<num delivered>.<stream sequence>.<consumer sequence>.<timestamp>.<num pending>
|
||||
// - v2: $JS.ACK.<domain>.<account hash>.<stream name>.<consumer name>.<num delivered>.<stream sequence>.<consumer sequence>.<timestamp>.<num pending>
|
||||
// See also: https://github.com/nats-io/nats-architecture-and-design/blob/main/adr/ADR-15.md#jsack
|
||||
FeatureFlagJsAckFormatV2: false,
|
||||
|
||||
// Propose delete range gaps as a single `deleteRangeOp` Raft append entry
|
||||
// instead of one entry per deleted sequence. Dramatically reduces Raft cost
|
||||
// on mirrors whose origin has a large number of interior deletes.
|
||||
// - Introduced: 2.14.0, apply-side always supports receiving `deleteRangeOp`.
|
||||
// - Enabled: TBD, once all supported versions carry the apply-side.
|
||||
//
|
||||
// WARNING: Only enable once every peer in the cluster is on a version that
|
||||
// supports receiving `deleteRangeOp`. Older peers panic on apply of an
|
||||
// unknown stream entry operation.
|
||||
FeatureFlagJsRaftDeleteRange: false,
|
||||
}
|
||||
|
||||
// getFeatureFlag is used to retrieve either the default or overwritten value for a feature flag.
|
||||
// The user's value takes precedence over the system's default. However, if the flag doesn't exist, it's disabled.
|
||||
// The *Options returned by Server.getOpts() is treated as immutable, mutations go through setOpts,
|
||||
// so no lock is required on the map read here.
|
||||
func (o *Options) getFeatureFlag(k string) bool {
|
||||
defaultValue, ok := featureFlags[k]
|
||||
if !ok {
|
||||
return false // Not supported.
|
||||
}
|
||||
if userValue, ok := o.FeatureFlags[k]; ok {
|
||||
return userValue
|
||||
}
|
||||
return defaultValue
|
||||
}
|
||||
|
||||
// getMergedFeatureFlags returns a merged map of feature flags, with the user's values taking precedence.
|
||||
func (o *Options) getMergedFeatureFlags() map[string]bool {
|
||||
merged := make(map[string]bool)
|
||||
for k, v := range featureFlags {
|
||||
merged[k] = v
|
||||
}
|
||||
for k, v := range o.FeatureFlags {
|
||||
if _, ok := featureFlags[k]; !ok {
|
||||
continue
|
||||
}
|
||||
merged[k] = v
|
||||
}
|
||||
return merged
|
||||
}
|
||||
|
||||
// printFeatureFlags logs the currently used feature flags on server startup.
|
||||
func (s *Server) printFeatureFlags(o *Options) {
|
||||
if len(o.FeatureFlags) == 0 {
|
||||
return
|
||||
}
|
||||
keys := slices.Sorted(maps.Keys(o.FeatureFlags))
|
||||
|
||||
var (
|
||||
configured strings.Builder
|
||||
unsupported strings.Builder
|
||||
)
|
||||
|
||||
for _, k := range keys {
|
||||
// Unsupported
|
||||
defaultValue, ok := featureFlags[k]
|
||||
if !ok {
|
||||
if unsupported.Len() > 0 {
|
||||
unsupported.WriteString(", ")
|
||||
}
|
||||
unsupported.WriteString(k)
|
||||
continue
|
||||
}
|
||||
|
||||
v := o.FeatureFlags[k]
|
||||
if configured.Len() > 0 {
|
||||
configured.WriteString(", ")
|
||||
}
|
||||
configured.WriteString(k)
|
||||
configured.WriteString(" (")
|
||||
if defaultValue {
|
||||
if v {
|
||||
configured.WriteString("enabled")
|
||||
} else {
|
||||
configured.WriteString("opt-out")
|
||||
}
|
||||
} else if v {
|
||||
configured.WriteString("opt-in")
|
||||
} else {
|
||||
configured.WriteString("disabled")
|
||||
}
|
||||
configured.WriteString(")")
|
||||
}
|
||||
if configured.Len() == 0 {
|
||||
configured.WriteString("none")
|
||||
}
|
||||
|
||||
s.Noticef(" Feature flags:")
|
||||
s.Noticef(" Configured: %s", configured.String())
|
||||
if unsupported.Len() > 0 {
|
||||
s.Noticef(" Unsupported: %s", unsupported.String())
|
||||
}
|
||||
}
|
||||
+1909
-651
File diff suppressed because it is too large
Load Diff
+14
-6
@@ -1156,9 +1156,7 @@ func (c *client) processGatewayInfo(info *Info) {
|
||||
// defensive code above that if we did not register this connection
|
||||
// because we already have an outbound for this name, then
|
||||
// close this connection (and make sure it does not try to reconnect)
|
||||
c.mu.Lock()
|
||||
c.flags.set(noReconnect)
|
||||
c.mu.Unlock()
|
||||
c.setNoReconnect()
|
||||
c.closeConnection(WrongGateway)
|
||||
return
|
||||
}
|
||||
@@ -1981,7 +1979,7 @@ func (c *client) processGatewayRUnsub(arg []byte) error {
|
||||
return nil
|
||||
} else {
|
||||
// Plain sub, assume optimistic sends, create entry.
|
||||
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()}
|
||||
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistForServer(c.srv)}
|
||||
newe = true
|
||||
}
|
||||
// This is when a sub or queue sub is supposed to be in
|
||||
@@ -2090,7 +2088,7 @@ func (c *client) processGatewayRSub(arg []byte) error {
|
||||
} else if queue == nil {
|
||||
return nil
|
||||
} else {
|
||||
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistWithCache()}
|
||||
e = &outsie{ni: make(map[string]struct{}), sl: NewSublistForServer(c.srv)}
|
||||
newe = true
|
||||
useSl = true
|
||||
}
|
||||
@@ -2952,6 +2950,16 @@ func getSubjectFromGWRoutedReply(reply []byte, isOldPrefix bool) []byte {
|
||||
return reply[gwSubjectOffset:]
|
||||
}
|
||||
|
||||
// Returns the subject embedded in the given routed
|
||||
// reply subject and whether the prefix was stripped.
|
||||
// If the subject is not routed, returns it unchanged.
|
||||
func getGWRoutedSubjectOrSelf(subject []byte) ([]byte, bool) {
|
||||
if isGWPrefix, oldPrefix := isGWRoutedSubjectAndIsOldPrefix(subject); isGWPrefix {
|
||||
return getSubjectFromGWRoutedReply(subject, oldPrefix), true
|
||||
}
|
||||
return subject, false
|
||||
}
|
||||
|
||||
// This should be invoked only from processInboundGatewayMsg() or
|
||||
// processInboundRoutedMsg() and is checking if the subject
|
||||
// (c.pa.subject) has the _GR_ prefix. If so, this is processed
|
||||
@@ -3201,7 +3209,7 @@ func (c *client) gatewayAllSubsReceiveStart(info *Info) {
|
||||
e.mode = Transitioning
|
||||
e.Unlock()
|
||||
} else {
|
||||
e := &outsie{sl: NewSublistWithCache()}
|
||||
e := &outsie{sl: NewSublistForServer(c.srv)}
|
||||
e.mode = Transitioning
|
||||
c.mu.Lock()
|
||||
c.gw.outsim.Store(account, e)
|
||||
|
||||
+60
@@ -170,6 +170,66 @@ func (s *GenericSublist[T]) NumInterest(subject string) (np int) {
|
||||
return
|
||||
}
|
||||
|
||||
// MatchesFullWildcard returns true if there is top-level ">" interest.
|
||||
func (s *GenericSublist[T]) MatchesFullWildcard() bool {
|
||||
if s == nil {
|
||||
return false
|
||||
}
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
return s.root.fwc != nil
|
||||
}
|
||||
|
||||
// MatchesSingleFilter returns the filter when the sublist contains exactly one unique subject.
|
||||
func (s *GenericSublist[T]) MatchesSingleFilter() (string, bool) {
|
||||
if s == nil {
|
||||
return _EMPTY_, false
|
||||
}
|
||||
s.RLock()
|
||||
defer s.RUnlock()
|
||||
return singleFilter(s.root, _EMPTY_)
|
||||
}
|
||||
|
||||
func singleFilter[T comparable](l *level[T], filter string) (string, bool) {
|
||||
if l == nil {
|
||||
return filter, filter != _EMPTY_
|
||||
}
|
||||
if len(l.nodes) > 1 {
|
||||
return _EMPTY_, false
|
||||
}
|
||||
var next *node[T]
|
||||
branches := 0
|
||||
if l.pwc != nil {
|
||||
next = l.pwc
|
||||
branches++
|
||||
}
|
||||
if l.fwc != nil {
|
||||
next = l.fwc
|
||||
branches++
|
||||
}
|
||||
for _, n := range l.nodes {
|
||||
next = n
|
||||
branches++
|
||||
}
|
||||
if branches != 1 {
|
||||
return _EMPTY_, false
|
||||
}
|
||||
for _, subj := range next.subs {
|
||||
filter = subj
|
||||
break
|
||||
}
|
||||
if next.next == nil {
|
||||
return filter, filter != _EMPTY_
|
||||
}
|
||||
if filter != _EMPTY_ {
|
||||
if next.next.numNodes() > 0 {
|
||||
return _EMPTY_, false
|
||||
}
|
||||
return filter, true
|
||||
}
|
||||
return singleFilter(next.next, filter)
|
||||
}
|
||||
|
||||
func (s *GenericSublist[T]) match(subject string, cb func(T), doLock bool) {
|
||||
tsa := [32]string{}
|
||||
tokens := tsa[:0]
|
||||
|
||||
+63
-423
@@ -25,13 +25,13 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime/debug"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/minio/highwayhash"
|
||||
"github.com/nats-io/nats-server/v2/server/gsl"
|
||||
"github.com/nats-io/nats-server/v2/server/sysmem"
|
||||
"github.com/nats-io/nats-server/v2/server/tpm"
|
||||
"github.com/nats-io/nkeys"
|
||||
@@ -103,22 +103,26 @@ type JetStreamAPIStats struct {
|
||||
// This is for internal accounting for JetStream for this server.
|
||||
type jetStream struct {
|
||||
// These are here first because of atomics on 32bit systems.
|
||||
apiInflight int64
|
||||
apiTotal int64
|
||||
apiErrors int64
|
||||
memReserved int64
|
||||
storeReserved int64
|
||||
memUsed int64
|
||||
storeUsed int64
|
||||
queueLimit int64
|
||||
clustered int32
|
||||
mu sync.RWMutex
|
||||
srv *Server
|
||||
config JetStreamConfig
|
||||
cluster *jetStreamCluster
|
||||
accounts map[string]*jsAccount
|
||||
apiSubs *Sublist
|
||||
started time.Time
|
||||
apiInflight int64
|
||||
apiTotal int64
|
||||
apiErrors int64
|
||||
memMax int64
|
||||
memReserved int64 // Requires JS lock to be held.
|
||||
memUsed int64
|
||||
storeMax int64
|
||||
storeReserved int64 // Requires JS lock to be held.
|
||||
storeUsed int64
|
||||
queueLimit int64
|
||||
infoQueueLimit int64
|
||||
clustered int32
|
||||
mu sync.RWMutex
|
||||
srv *Server
|
||||
config JetStreamConfig
|
||||
cluster *jetStreamCluster
|
||||
accounts map[string]*jsAccount
|
||||
apiSubs *Sublist
|
||||
infoSubs *gsl.SimpleSublist // Subjects for info-specific queue.
|
||||
started time.Time
|
||||
|
||||
// System level request to purge a stream move
|
||||
accountPurge *subscription
|
||||
@@ -150,14 +154,12 @@ type jsaStorage struct {
|
||||
// an internal sub for a stream, so we will direct link to the stream
|
||||
// and walk backwards as needed vs multiple hash lookups and locks, etc.
|
||||
type jsAccount struct {
|
||||
mu sync.RWMutex
|
||||
js *jetStream
|
||||
account *Account
|
||||
storeDir string
|
||||
inflight sync.Map
|
||||
streams map[string]*stream
|
||||
templates map[string]*streamTemplate // Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
store TemplateStore // Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
mu sync.RWMutex
|
||||
js *jetStream
|
||||
account *Account
|
||||
storeDir string
|
||||
inflight sync.Map
|
||||
streams map[string]*stream
|
||||
|
||||
// From server
|
||||
sendq *ipQueue[*pubMsg]
|
||||
@@ -415,15 +417,19 @@ func (s *Server) initJetStreamEncryption() (err error) {
|
||||
|
||||
// enableJetStream will start up the JetStream subsystem.
|
||||
func (s *Server) enableJetStream(cfg JetStreamConfig) error {
|
||||
js := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache()}
|
||||
js := &jetStream{srv: s, config: cfg, accounts: make(map[string]*jsAccount), apiSubs: NewSublistNoCache(), infoSubs: gsl.NewSimpleSublist()}
|
||||
s.gcbMu.Lock()
|
||||
if s.gcbOutMax = s.getOpts().JetStreamMaxCatchup; s.gcbOutMax == 0 {
|
||||
s.gcbOutMax = defaultMaxTotalCatchupOutBytes
|
||||
}
|
||||
s.gcbMu.Unlock()
|
||||
|
||||
atomic.StoreInt64(&js.memMax, cfg.MaxMemory)
|
||||
atomic.StoreInt64(&js.storeMax, cfg.MaxStore)
|
||||
|
||||
// TODO: Not currently reloadable.
|
||||
atomic.StoreInt64(&js.queueLimit, s.getOpts().JetStreamRequestQueueLimit)
|
||||
atomic.StoreInt64(&js.infoQueueLimit, s.getOpts().JetStreamInfoQueueLimit)
|
||||
|
||||
s.js.Store(js)
|
||||
|
||||
@@ -1058,8 +1064,10 @@ func (s *Server) shutdownJetStream() {
|
||||
func (s *Server) JetStreamConfig() *JetStreamConfig {
|
||||
var c *JetStreamConfig
|
||||
if js := s.getJetStream(); js != nil {
|
||||
js.mu.RLock()
|
||||
copy := js.config
|
||||
c = &(copy)
|
||||
js.mu.RUnlock()
|
||||
}
|
||||
return c
|
||||
}
|
||||
@@ -1219,54 +1227,6 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
|
||||
s.Debugf("Recovering JetStream state for account %q", a.Name)
|
||||
}
|
||||
|
||||
// Check templates first since messsage sets will need proper ownership.
|
||||
// FIXME(dlc) - Make this consistent.
|
||||
tdir := filepath.Join(jsa.storeDir, tmplsDir)
|
||||
if stat, err := os.Stat(tdir); err == nil && stat.IsDir() {
|
||||
key := sha256.Sum256([]byte("templates"))
|
||||
hh, err := highwayhash.NewDigest64(key[:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fis, _ := os.ReadDir(tdir)
|
||||
for _, fi := range fis {
|
||||
metafile := filepath.Join(tdir, fi.Name(), JetStreamMetaFile)
|
||||
metasum := filepath.Join(tdir, fi.Name(), JetStreamMetaFileSum)
|
||||
buf, err := os.ReadFile(metafile)
|
||||
if err != nil {
|
||||
s.Warnf(" Error reading StreamTemplate metafile %q: %v", metasum, err)
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(metasum); os.IsNotExist(err) {
|
||||
s.Warnf(" Missing StreamTemplate checksum for %q", metasum)
|
||||
continue
|
||||
}
|
||||
sum, err := os.ReadFile(metasum)
|
||||
if err != nil {
|
||||
s.Warnf(" Error reading StreamTemplate checksum %q: %v", metasum, err)
|
||||
continue
|
||||
}
|
||||
hh.Reset()
|
||||
hh.Write(buf)
|
||||
var hb [highwayhash.Size64]byte
|
||||
checksum := hex.EncodeToString(hh.Sum(hb[:0]))
|
||||
if checksum != string(sum) {
|
||||
s.Warnf(" StreamTemplate checksums do not match %q vs %q", sum, checksum)
|
||||
continue
|
||||
}
|
||||
var cfg StreamTemplateConfig
|
||||
if err := json.Unmarshal(buf, &cfg); err != nil {
|
||||
s.Warnf(" Error unmarshalling StreamTemplate metafile: %v", err)
|
||||
continue
|
||||
}
|
||||
cfg.Config.Name = _EMPTY_
|
||||
if _, err := a.addStreamTemplate(&cfg); err != nil {
|
||||
s.Warnf(" Error recreating StreamTemplate %q: %v", cfg.Name, err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Remember if we should be encrypted and what cipher we think we should use.
|
||||
encrypted := s.getOpts().JetStreamKey != _EMPTY_
|
||||
sc := s.getOpts().JetStreamCipher
|
||||
@@ -1510,15 +1470,6 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
|
||||
return nil
|
||||
}
|
||||
|
||||
if cfg.Template != _EMPTY_ {
|
||||
jsa.mu.Lock()
|
||||
err := jsa.addStreamNameToTemplate(cfg.Template, cfg.Name)
|
||||
jsa.mu.Unlock()
|
||||
if err != nil {
|
||||
s.Warnf(" Error adding stream %q to template %q: %v", cfg.Name, cfg.Template, err)
|
||||
}
|
||||
}
|
||||
|
||||
// We had a bug that set a default de dupe window on mirror, despite that being not a valid config
|
||||
fixCfgMirrorWithDedupWindow(&cfg.StreamConfig)
|
||||
|
||||
@@ -1587,6 +1538,7 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
|
||||
batchId string
|
||||
batchSeq uint64
|
||||
commit bool
|
||||
commitEob bool
|
||||
batchStoreDir string
|
||||
store StreamStore
|
||||
state StreamState
|
||||
@@ -1604,19 +1556,30 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
|
||||
}
|
||||
// We've observed a partial batch write. Write the remainder of the batch.
|
||||
batchSeq++
|
||||
_, batchStoreDir = getBatchStoreDir(mset, batchId)
|
||||
_, batchStoreDir = getBatchStoreDir(jsa.storeDir, cfg.Name, batchId)
|
||||
if _, err = os.Stat(batchStoreDir); err != nil {
|
||||
s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v",
|
||||
mset.accName(), mset.name(), batchSeq, err)
|
||||
goto SKIP
|
||||
}
|
||||
store, err = newBatchStore(mset, batchId)
|
||||
store, err = newBatchStore(mset, batchId, cfg.Replicas, cfg.Storage, jsa.storeDir, cfg.Name)
|
||||
if err != nil {
|
||||
s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: %v",
|
||||
mset.accName(), mset.name(), batchSeq, err)
|
||||
goto SKIP
|
||||
}
|
||||
store.FastState(&state)
|
||||
sm, err = store.LoadMsg(state.LastSeq, &smv)
|
||||
if err != nil || sm == nil {
|
||||
s.Errorf(" Failed restoring partial batch write for stream '%s > %s' at sequence %d: last msg not found %d",
|
||||
mset.accName(), mset.name(), batchSeq, state.LastSeq)
|
||||
goto SKIP
|
||||
}
|
||||
commitEob = bytes.Equal(sliceHeader(JSBatchCommit, sm.hdr), []byte("eob"))
|
||||
// If the commit ends with an "End Of Batch" message, we don't store this.
|
||||
if commitEob {
|
||||
state.LastSeq--
|
||||
}
|
||||
s.Noticef(" Restoring partial batch write for stream '%s > %s' (seq %d to %d)",
|
||||
mset.accName(), mset.name(), batchSeq, state.LastSeq)
|
||||
// Loop through items that weren't persisted yet.
|
||||
@@ -1627,7 +1590,12 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
|
||||
mset.accName(), mset.name(), seq, err)
|
||||
break
|
||||
}
|
||||
mset.processJetStreamMsg(sm.subj, _EMPTY_, sm.hdr, sm.msg, 0, 0, nil, false, true)
|
||||
hdr := sm.hdr
|
||||
// If committed by EOB, the last message must get the normal commit header.
|
||||
if commitEob && seq == state.LastSeq {
|
||||
hdr = genHeader(hdr, JSBatchCommit, "1")
|
||||
}
|
||||
mset.processJetStreamMsg(sm.subj, _EMPTY_, hdr, sm.msg, 0, 0, nil, false, true)
|
||||
}
|
||||
store.Delete(true)
|
||||
SKIP:
|
||||
@@ -2342,14 +2310,14 @@ func (jsa *jsAccount) sendClusterUsageUpdate() {
|
||||
func (js *jetStream) wouldExceedLimits(storeType StorageType, sz int) bool {
|
||||
var (
|
||||
total *int64
|
||||
max int64
|
||||
max *int64
|
||||
)
|
||||
if storeType == MemoryStorage {
|
||||
total, max = &js.memUsed, js.config.MaxMemory
|
||||
total, max = &js.memUsed, &js.memMax
|
||||
} else {
|
||||
total, max = &js.storeUsed, js.config.MaxStore
|
||||
total, max = &js.storeUsed, &js.storeMax
|
||||
}
|
||||
return (atomic.LoadInt64(total) + int64(sz)) > max
|
||||
return (atomic.LoadInt64(total) + int64(sz)) > atomic.LoadInt64(max)
|
||||
}
|
||||
|
||||
func (js *jetStream) limitsExceeded(storeType StorageType) bool {
|
||||
@@ -2519,7 +2487,6 @@ func (jsa *jsAccount) acc() *Account {
|
||||
// Delete the JetStream resources.
|
||||
func (jsa *jsAccount) delete() {
|
||||
var streams []*stream
|
||||
var ts []string
|
||||
|
||||
jsa.mu.Lock()
|
||||
// The update timer and subs need to be protected by usageMu lock
|
||||
@@ -2538,20 +2505,11 @@ func (jsa *jsAccount) delete() {
|
||||
for _, ms := range jsa.streams {
|
||||
streams = append(streams, ms)
|
||||
}
|
||||
acc := jsa.account
|
||||
for _, t := range jsa.templates {
|
||||
ts = append(ts, t.Name)
|
||||
}
|
||||
jsa.templates = nil
|
||||
jsa.mu.Unlock()
|
||||
|
||||
for _, mset := range streams {
|
||||
mset.stop(false, false)
|
||||
}
|
||||
|
||||
for _, t := range ts {
|
||||
acc.deleteStreamTemplate(t)
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup the jetstream account for a given account.
|
||||
@@ -2763,325 +2721,6 @@ func (a *Account) checkForJetStream() (*Server, *jsAccount, error) {
|
||||
return s, jsa, nil
|
||||
}
|
||||
|
||||
// StreamTemplateConfig allows a configuration to auto-create streams based on this template when a message
|
||||
// is received that matches. Each new stream will use the config as the template config to create them.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type StreamTemplateConfig struct {
|
||||
Name string `json:"name"`
|
||||
Config *StreamConfig `json:"config"`
|
||||
MaxStreams uint32 `json:"max_streams"`
|
||||
}
|
||||
|
||||
// StreamTemplateInfo
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type StreamTemplateInfo struct {
|
||||
Config *StreamTemplateConfig `json:"config"`
|
||||
Streams []string `json:"streams"`
|
||||
}
|
||||
|
||||
// streamTemplate
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type streamTemplate struct {
|
||||
mu sync.Mutex
|
||||
tc *client
|
||||
jsa *jsAccount
|
||||
*StreamTemplateConfig
|
||||
streams []string
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (t *StreamTemplateConfig) deepCopy() *StreamTemplateConfig {
|
||||
copy := *t
|
||||
cfg := *t.Config
|
||||
copy.Config = &cfg
|
||||
return ©
|
||||
}
|
||||
|
||||
// addStreamTemplate will add a stream template to this account that allows auto-creation of streams.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (a *Account) addStreamTemplate(tc *StreamTemplateConfig) (*streamTemplate, error) {
|
||||
s, jsa, err := a.checkForJetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if tc.Config.Name != "" {
|
||||
return nil, fmt.Errorf("template config name should be empty")
|
||||
}
|
||||
if len(tc.Name) > JSMaxNameLen {
|
||||
return nil, fmt.Errorf("template name is too long, maximum allowed is %d", JSMaxNameLen)
|
||||
}
|
||||
|
||||
// FIXME(dlc) - Hacky
|
||||
tcopy := tc.deepCopy()
|
||||
tcopy.Config.Name = "_"
|
||||
cfg, apiErr := s.checkStreamCfg(tcopy.Config, a, false)
|
||||
if apiErr != nil {
|
||||
return nil, apiErr
|
||||
}
|
||||
tcopy.Config = &cfg
|
||||
t := &streamTemplate{
|
||||
StreamTemplateConfig: tcopy,
|
||||
tc: s.createInternalJetStreamClient(),
|
||||
jsa: jsa,
|
||||
}
|
||||
t.tc.registerWithAccount(a)
|
||||
|
||||
jsa.mu.Lock()
|
||||
if jsa.templates == nil {
|
||||
jsa.templates = make(map[string]*streamTemplate)
|
||||
// Create the appropriate store
|
||||
if cfg.Storage == FileStorage {
|
||||
jsa.store = newTemplateFileStore(jsa.storeDir)
|
||||
} else {
|
||||
jsa.store = newTemplateMemStore()
|
||||
}
|
||||
} else if _, ok := jsa.templates[tcopy.Name]; ok {
|
||||
jsa.mu.Unlock()
|
||||
return nil, fmt.Errorf("template with name %q already exists", tcopy.Name)
|
||||
}
|
||||
jsa.templates[tcopy.Name] = t
|
||||
jsa.mu.Unlock()
|
||||
|
||||
// FIXME(dlc) - we can not overlap subjects between templates. Need to have test.
|
||||
|
||||
// Setup the internal subscriptions to trap the messages.
|
||||
if err := t.createTemplateSubscriptions(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := jsa.store.Store(t); err != nil {
|
||||
t.delete()
|
||||
return nil, err
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (t *streamTemplate) createTemplateSubscriptions() error {
|
||||
if t == nil {
|
||||
return fmt.Errorf("no template")
|
||||
}
|
||||
if t.tc == nil {
|
||||
return fmt.Errorf("template not enabled")
|
||||
}
|
||||
c := t.tc
|
||||
if !c.srv.EventsEnabled() {
|
||||
return ErrNoSysAccount
|
||||
}
|
||||
sid := 1
|
||||
for _, subject := range t.Config.Subjects {
|
||||
// Now create the subscription
|
||||
if _, err := c.processSub([]byte(subject), nil, []byte(strconv.Itoa(sid)), t.processInboundTemplateMsg, false); err != nil {
|
||||
c.acc.deleteStreamTemplate(t.Name)
|
||||
return err
|
||||
}
|
||||
sid++
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (t *streamTemplate) processInboundTemplateMsg(_ *subscription, pc *client, acc *Account, subject, reply string, msg []byte) {
|
||||
if t == nil || t.jsa == nil {
|
||||
return
|
||||
}
|
||||
jsa := t.jsa
|
||||
cn := canonicalName(subject)
|
||||
|
||||
jsa.mu.Lock()
|
||||
// If we already are registered then we can just return here.
|
||||
if _, ok := jsa.streams[cn]; ok {
|
||||
jsa.mu.Unlock()
|
||||
return
|
||||
}
|
||||
jsa.mu.Unlock()
|
||||
|
||||
// Check if we are at the maximum and grab some variables.
|
||||
t.mu.Lock()
|
||||
c := t.tc
|
||||
cfg := *t.Config
|
||||
cfg.Template = t.Name
|
||||
atLimit := len(t.streams) >= int(t.MaxStreams)
|
||||
if !atLimit {
|
||||
t.streams = append(t.streams, cn)
|
||||
}
|
||||
t.mu.Unlock()
|
||||
|
||||
if atLimit {
|
||||
c.RateLimitWarnf("JetStream could not create stream for account %q on subject %q, at limit", acc.Name, subject)
|
||||
return
|
||||
}
|
||||
|
||||
// We need to create the stream here.
|
||||
// Change the config from the template and only use literal subject.
|
||||
cfg.Name = cn
|
||||
cfg.Subjects = []string{subject}
|
||||
mset, err := acc.addStream(&cfg)
|
||||
if err != nil {
|
||||
acc.validateStreams(t)
|
||||
c.RateLimitWarnf("JetStream could not create stream for account %q on subject %q: %v", acc.Name, subject, err)
|
||||
return
|
||||
}
|
||||
|
||||
// Process this message directly by invoking mset.
|
||||
mset.processInboundJetStreamMsg(nil, pc, acc, subject, reply, msg)
|
||||
}
|
||||
|
||||
// lookupStreamTemplate looks up the names stream template.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (a *Account) lookupStreamTemplate(name string) (*streamTemplate, error) {
|
||||
_, jsa, err := a.checkForJetStream()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
jsa.mu.Lock()
|
||||
defer jsa.mu.Unlock()
|
||||
if jsa.templates == nil {
|
||||
return nil, fmt.Errorf("template not found")
|
||||
}
|
||||
t, ok := jsa.templates[name]
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("template not found")
|
||||
}
|
||||
return t, nil
|
||||
}
|
||||
|
||||
// This function will check all named streams and make sure they are valid.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (a *Account) validateStreams(t *streamTemplate) {
|
||||
t.mu.Lock()
|
||||
var vstreams []string
|
||||
for _, sname := range t.streams {
|
||||
if _, err := a.lookupStream(sname); err == nil {
|
||||
vstreams = append(vstreams, sname)
|
||||
}
|
||||
}
|
||||
t.streams = vstreams
|
||||
t.mu.Unlock()
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (t *streamTemplate) delete() error {
|
||||
if t == nil {
|
||||
return fmt.Errorf("nil stream template")
|
||||
}
|
||||
|
||||
t.mu.Lock()
|
||||
jsa := t.jsa
|
||||
c := t.tc
|
||||
t.tc = nil
|
||||
defer func() {
|
||||
if c != nil {
|
||||
c.closeConnection(ClientClosed)
|
||||
}
|
||||
}()
|
||||
t.mu.Unlock()
|
||||
|
||||
if jsa == nil {
|
||||
return NewJSNotEnabledForAccountError()
|
||||
}
|
||||
|
||||
jsa.mu.Lock()
|
||||
if jsa.templates == nil {
|
||||
jsa.mu.Unlock()
|
||||
return fmt.Errorf("template not found")
|
||||
}
|
||||
if _, ok := jsa.templates[t.Name]; !ok {
|
||||
jsa.mu.Unlock()
|
||||
return fmt.Errorf("template not found")
|
||||
}
|
||||
delete(jsa.templates, t.Name)
|
||||
acc := jsa.account
|
||||
jsa.mu.Unlock()
|
||||
|
||||
// Remove streams associated with this template.
|
||||
var streams []*stream
|
||||
t.mu.Lock()
|
||||
for _, name := range t.streams {
|
||||
if mset, err := acc.lookupStream(name); err == nil {
|
||||
streams = append(streams, mset)
|
||||
}
|
||||
}
|
||||
t.mu.Unlock()
|
||||
|
||||
if jsa.store != nil {
|
||||
if err := jsa.store.Delete(t); err != nil {
|
||||
return fmt.Errorf("error deleting template from store: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
var lastErr error
|
||||
for _, mset := range streams {
|
||||
if err := mset.delete(); err != nil {
|
||||
lastErr = err
|
||||
}
|
||||
}
|
||||
return lastErr
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (a *Account) deleteStreamTemplate(name string) error {
|
||||
t, err := a.lookupStreamTemplate(name)
|
||||
if err != nil {
|
||||
return NewJSStreamTemplateNotFoundError()
|
||||
}
|
||||
return t.delete()
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (a *Account) templates() []*streamTemplate {
|
||||
var ts []*streamTemplate
|
||||
_, jsa, err := a.checkForJetStream()
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
jsa.mu.Lock()
|
||||
for _, t := range jsa.templates {
|
||||
// FIXME(dlc) - Copy?
|
||||
ts = append(ts, t)
|
||||
}
|
||||
jsa.mu.Unlock()
|
||||
|
||||
return ts
|
||||
}
|
||||
|
||||
// Will add a stream to a template, this is for recovery.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (jsa *jsAccount) addStreamNameToTemplate(tname, mname string) error {
|
||||
if jsa.templates == nil {
|
||||
return fmt.Errorf("template not found")
|
||||
}
|
||||
t, ok := jsa.templates[tname]
|
||||
if !ok {
|
||||
return fmt.Errorf("template not found")
|
||||
}
|
||||
// We found template.
|
||||
t.mu.Lock()
|
||||
t.streams = append(t.streams, mname)
|
||||
t.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
// This will check if a template owns this stream.
|
||||
// jsAccount lock should be held
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (jsa *jsAccount) checkTemplateOwnership(tname, sname string) bool {
|
||||
if jsa.templates == nil {
|
||||
return false
|
||||
}
|
||||
t, ok := jsa.templates[tname]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
// We found template, make sure we are in streams.
|
||||
for _, streamName := range t.streams {
|
||||
if sname == streamName {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
type Number interface {
|
||||
int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64 | float32 | float64
|
||||
}
|
||||
@@ -3107,10 +2746,11 @@ func isValidName(name string) bool {
|
||||
return !strings.ContainsAny(name, " \t\r\n\f.*>")
|
||||
}
|
||||
|
||||
// CanonicalName will replace all token separators '.' with '_'.
|
||||
// This can be used when naming streams or consumers with multi-token subjects.
|
||||
func canonicalName(name string) string {
|
||||
return strings.ReplaceAll(name, ".", "_")
|
||||
func isValidAssetName(name string) bool {
|
||||
if name == _EMPTY_ {
|
||||
return false
|
||||
}
|
||||
return !strings.ContainsAny(name, " \t\r\n\f.*>\\/")
|
||||
}
|
||||
|
||||
// To throttle the out of resources errors.
|
||||
|
||||
+175
-331
@@ -20,6 +20,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
@@ -46,29 +47,6 @@ const (
|
||||
// Will return JSON response.
|
||||
JSApiAccountInfo = "$JS.API.INFO"
|
||||
|
||||
// JSApiTemplateCreate is the endpoint to create new stream templates.
|
||||
// Will return JSON response.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
JSApiTemplateCreate = "$JS.API.STREAM.TEMPLATE.CREATE.*"
|
||||
JSApiTemplateCreateT = "$JS.API.STREAM.TEMPLATE.CREATE.%s"
|
||||
|
||||
// JSApiTemplates is the endpoint to list all stream template names for this account.
|
||||
// Will return JSON response.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
JSApiTemplates = "$JS.API.STREAM.TEMPLATE.NAMES"
|
||||
|
||||
// JSApiTemplateInfo is for obtaining general information about a named stream template.
|
||||
// Will return JSON response.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
JSApiTemplateInfo = "$JS.API.STREAM.TEMPLATE.INFO.*"
|
||||
JSApiTemplateInfoT = "$JS.API.STREAM.TEMPLATE.INFO.%s"
|
||||
|
||||
// JSApiTemplateDelete is the endpoint to delete stream templates.
|
||||
// Will return JSON response.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
JSApiTemplateDelete = "$JS.API.STREAM.TEMPLATE.DELETE.*"
|
||||
JSApiTemplateDeleteT = "$JS.API.STREAM.TEMPLATE.DELETE.%s"
|
||||
|
||||
// JSApiStreamCreate is the endpoint to create new streams.
|
||||
// Will return JSON response.
|
||||
JSApiStreamCreate = "$JS.API.STREAM.CREATE.*"
|
||||
@@ -177,6 +155,9 @@ const (
|
||||
// JSApiRequestNextT is the prefix for the request next message(s) for a consumer in worker/pull mode.
|
||||
JSApiRequestNextT = "$JS.API.CONSUMER.MSG.NEXT.%s.%s"
|
||||
|
||||
// JSApiConsumerResetT is the prefix for resetting a given consumer to a new starting sequence.
|
||||
JSApiConsumerResetT = "$JS.API.CONSUMER.RESET.%s.%s"
|
||||
|
||||
// JSApiConsumerUnpinT is the prefix for unpinning subscription for a given consumer.
|
||||
JSApiConsumerUnpin = "$JS.API.CONSUMER.UNPIN.*.*"
|
||||
JSApiConsumerUnpinT = "$JS.API.CONSUMER.UNPIN.%s.%s"
|
||||
@@ -237,13 +218,15 @@ const (
|
||||
// jsAckT is the template for the ack message stream coming back from a consumer
|
||||
// when they ACK/NAK, etc a message.
|
||||
jsAckT = "$JS.ACK.%s.%s"
|
||||
jsAckTv2 = "$JS.ACK.%s.%s.%s.%s"
|
||||
jsAckPre = "$JS.ACK."
|
||||
jsAckPreLen = len(jsAckPre)
|
||||
|
||||
// jsFlowControl is for flow control subjects.
|
||||
jsFlowControlPre = "$JS.FC."
|
||||
// jsFlowControl is for FC responses.
|
||||
jsFlowControl = "$JS.FC.%s.%s.*"
|
||||
jsFlowControl = "$JS.FC.%s.%s.*"
|
||||
jsFlowControlV2 = "$JS.FC.%s.%s.%s.%s.*"
|
||||
|
||||
// JSAdvisoryPrefix is a prefix for all JetStream advisories.
|
||||
JSAdvisoryPrefix = "$JS.EVENT.ADVISORY"
|
||||
@@ -787,50 +770,19 @@ type JSApiConsumerGetNextRequest struct {
|
||||
PriorityGroup
|
||||
}
|
||||
|
||||
// JSApiStreamTemplateCreateResponse for creating templates.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type JSApiStreamTemplateCreateResponse struct {
|
||||
// JSApiConsumerResetRequest is for resetting a consumer to a specific sequence.
|
||||
type JSApiConsumerResetRequest struct {
|
||||
Seq uint64 `json:"seq,omitempty"`
|
||||
}
|
||||
|
||||
// JSApiConsumerResetResponse is a superset of JSApiConsumerCreateResponse, but including an explicit ResetSeq.
|
||||
type JSApiConsumerResetResponse struct {
|
||||
ApiResponse
|
||||
*StreamTemplateInfo
|
||||
*ConsumerInfo
|
||||
ResetSeq uint64 `json:"reset_seq"`
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
const JSApiStreamTemplateCreateResponseType = "io.nats.jetstream.api.v1.stream_template_create_response"
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type JSApiStreamTemplateDeleteResponse struct {
|
||||
ApiResponse
|
||||
Success bool `json:"success,omitempty"`
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
const JSApiStreamTemplateDeleteResponseType = "io.nats.jetstream.api.v1.stream_template_delete_response"
|
||||
|
||||
// JSApiStreamTemplateInfoResponse for information about stream templates.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type JSApiStreamTemplateInfoResponse struct {
|
||||
ApiResponse
|
||||
*StreamTemplateInfo
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
const JSApiStreamTemplateInfoResponseType = "io.nats.jetstream.api.v1.stream_template_info_response"
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type JSApiStreamTemplatesRequest struct {
|
||||
ApiPagedRequest
|
||||
}
|
||||
|
||||
// JSApiStreamTemplateNamesResponse list of templates
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
type JSApiStreamTemplateNamesResponse struct {
|
||||
ApiResponse
|
||||
ApiPaged
|
||||
Templates []string `json:"streams"`
|
||||
}
|
||||
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
const JSApiStreamTemplateNamesResponseType = "io.nats.jetstream.api.v1.stream_template_names_response"
|
||||
const JSApiConsumerResetResponseType = "io.nats.jetstream.api.v1.consumer_reset_response"
|
||||
|
||||
// Structure that holds state for a JetStream API request that is processed
|
||||
// in a separate long-lived go routine. This is to avoid blocking connections.
|
||||
@@ -911,11 +863,19 @@ func (js *jetStream) apiDispatch(sub *subscription, c *client, acc *Account, sub
|
||||
// Copy the state. Note the JSAPI only uses the hdr index to piece apart the
|
||||
// header from the msg body. No other references are needed.
|
||||
// Check pending and warn if getting backed up.
|
||||
pending, _ := s.jsAPIRoutedReqs.push(&jsAPIRoutedReq{jsub, sub, acc, subject, reply, copyBytes(rmsg), c.pa})
|
||||
limit := atomic.LoadInt64(&js.queueLimit)
|
||||
var queue *ipQueue[*jsAPIRoutedReq]
|
||||
var limit int64
|
||||
if js.infoSubs.HasInterest(subject) {
|
||||
queue = s.jsAPIRoutedInfoReqs
|
||||
limit = atomic.LoadInt64(&js.infoQueueLimit)
|
||||
} else {
|
||||
queue = s.jsAPIRoutedReqs
|
||||
limit = atomic.LoadInt64(&js.queueLimit)
|
||||
}
|
||||
pending, _ := queue.push(&jsAPIRoutedReq{jsub, sub, acc, subject, reply, copyBytes(rmsg), c.pa})
|
||||
if pending >= int(limit) {
|
||||
s.rateLimitFormatWarnf("JetStream API queue limit reached, dropping %d requests", pending)
|
||||
drained := int64(s.jsAPIRoutedReqs.drain())
|
||||
s.rateLimitFormatWarnf("%s limit reached, dropping %d requests", queue.name, pending)
|
||||
drained := int64(queue.drain())
|
||||
atomic.AddInt64(&js.apiInflight, -drained)
|
||||
|
||||
s.publishAdvisory(nil, JSAdvisoryAPILimitReached, JSAPILimitReachedAdvisory{
|
||||
@@ -935,29 +895,45 @@ func (s *Server) processJSAPIRoutedRequests() {
|
||||
defer s.grWG.Done()
|
||||
|
||||
s.mu.RLock()
|
||||
queue := s.jsAPIRoutedReqs
|
||||
queue, infoqueue := s.jsAPIRoutedReqs, s.jsAPIRoutedInfoReqs
|
||||
client := &client{srv: s, kind: JETSTREAM}
|
||||
s.mu.RUnlock()
|
||||
|
||||
js := s.getJetStream()
|
||||
|
||||
processFromQueue := func(ipq *ipQueue[*jsAPIRoutedReq]) {
|
||||
// Only pop one item at a time here, otherwise if the system is recovering
|
||||
// from queue buildup, then one worker will pull off all the tasks and the
|
||||
// others will be starved of work.
|
||||
if r, ok := ipq.popOne(); ok && r != nil {
|
||||
client.pa = r.pa
|
||||
start := time.Now()
|
||||
r.jsub.icb(r.sub, client, r.acc, r.subject, r.reply, r.msg)
|
||||
if dur := time.Since(start); dur >= readLoopReportThreshold {
|
||||
s.Warnf("Internal subscription on %q took too long: %v", r.subject, dur)
|
||||
}
|
||||
atomic.AddInt64(&js.apiInflight, -1)
|
||||
}
|
||||
}
|
||||
|
||||
for {
|
||||
// First select case is prioritizing queue, we will only fall through
|
||||
// to the second select case that considers infoqueue if queue is empty.
|
||||
// This effectively means infos are deprioritized.
|
||||
select {
|
||||
case <-queue.ch:
|
||||
// Only pop one item at a time here, otherwise if the system is recovering
|
||||
// from queue buildup, then one worker will pull off all the tasks and the
|
||||
// others will be starved of work.
|
||||
for r, ok := queue.popOne(); ok && r != nil; r, ok = queue.popOne() {
|
||||
client.pa = r.pa
|
||||
start := time.Now()
|
||||
r.jsub.icb(r.sub, client, r.acc, r.subject, r.reply, r.msg)
|
||||
if dur := time.Since(start); dur >= readLoopReportThreshold {
|
||||
s.Warnf("Internal subscription on %q took too long: %v", r.subject, dur)
|
||||
}
|
||||
atomic.AddInt64(&js.apiInflight, -1)
|
||||
}
|
||||
processFromQueue(queue)
|
||||
case <-s.quitCh:
|
||||
return
|
||||
default:
|
||||
select {
|
||||
case <-infoqueue.ch:
|
||||
processFromQueue(infoqueue)
|
||||
case <-queue.ch:
|
||||
processFromQueue(queue)
|
||||
case <-s.quitCh:
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -976,7 +952,8 @@ func (s *Server) setJetStreamExportSubs() error {
|
||||
if mp > maxProcs {
|
||||
mp = maxProcs
|
||||
}
|
||||
s.jsAPIRoutedReqs = newIPQueue[*jsAPIRoutedReq](s, "Routed JS API Requests")
|
||||
s.jsAPIRoutedReqs = newIPQueue[*jsAPIRoutedReq](s, "JetStream API queue")
|
||||
s.jsAPIRoutedInfoReqs = newIPQueue[*jsAPIRoutedReq](s, "JetStream API info queue")
|
||||
for i := 0; i < mp; i++ {
|
||||
s.startGoRoutine(s.processJSAPIRoutedRequests)
|
||||
}
|
||||
@@ -992,20 +969,13 @@ func (s *Server) setJetStreamExportSubs() error {
|
||||
}
|
||||
|
||||
// API handles themselves.
|
||||
// infopairs are deprioritized compared to pairs in processJSAPIRoutedRequests.
|
||||
pairs := []struct {
|
||||
subject string
|
||||
handler msgHandler
|
||||
}{
|
||||
{JSApiAccountInfo, s.jsAccountInfoRequest},
|
||||
{JSApiTemplateCreate, s.jsTemplateCreateRequest},
|
||||
{JSApiTemplates, s.jsTemplateNamesRequest},
|
||||
{JSApiTemplateInfo, s.jsTemplateInfoRequest},
|
||||
{JSApiTemplateDelete, s.jsTemplateDeleteRequest},
|
||||
{JSApiStreamCreate, s.jsStreamCreateRequest},
|
||||
{JSApiStreamUpdate, s.jsStreamUpdateRequest},
|
||||
{JSApiStreams, s.jsStreamNamesRequest},
|
||||
{JSApiStreamList, s.jsStreamListRequest},
|
||||
{JSApiStreamInfo, s.jsStreamInfoRequest},
|
||||
{JSApiStreamDelete, s.jsStreamDeleteRequest},
|
||||
{JSApiStreamPurge, s.jsStreamPurgeRequest},
|
||||
{JSApiStreamSnapshot, s.jsStreamSnapshotRequest},
|
||||
@@ -1018,23 +988,40 @@ func (s *Server) setJetStreamExportSubs() error {
|
||||
{JSApiConsumerCreateEx, s.jsConsumerCreateRequest},
|
||||
{JSApiConsumerCreate, s.jsConsumerCreateRequest},
|
||||
{JSApiDurableCreate, s.jsConsumerCreateRequest},
|
||||
{JSApiConsumers, s.jsConsumerNamesRequest},
|
||||
{JSApiConsumerList, s.jsConsumerListRequest},
|
||||
{JSApiConsumerInfo, s.jsConsumerInfoRequest},
|
||||
{JSApiConsumerDelete, s.jsConsumerDeleteRequest},
|
||||
{JSApiConsumerPause, s.jsConsumerPauseRequest},
|
||||
{JSApiConsumerUnpin, s.jsConsumerUnpinRequest},
|
||||
}
|
||||
infopairs := []struct {
|
||||
subject string
|
||||
handler msgHandler
|
||||
}{
|
||||
{JSApiAccountInfo, s.jsAccountInfoRequest},
|
||||
{JSApiStreams, s.jsStreamNamesRequest},
|
||||
{JSApiStreamList, s.jsStreamListRequest},
|
||||
{JSApiStreamInfo, s.jsStreamInfoRequest},
|
||||
{JSApiConsumers, s.jsConsumerNamesRequest},
|
||||
{JSApiConsumerList, s.jsConsumerListRequest},
|
||||
{JSApiConsumerInfo, s.jsConsumerInfoRequest},
|
||||
}
|
||||
|
||||
js.mu.Lock()
|
||||
defer js.mu.Unlock()
|
||||
|
||||
for _, p := range pairs {
|
||||
// As well as populating js.apiSubs for the dispatch function to use, we
|
||||
// will also populate js.infoSubs, so that the dispatch function can
|
||||
// decide quickly whether or not the request is an info request or not.
|
||||
for _, p := range append(infopairs, pairs...) {
|
||||
sub := &subscription{subject: []byte(p.subject), icb: p.handler}
|
||||
if err := js.apiSubs.Insert(sub); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, p := range infopairs {
|
||||
if err := js.infoSubs.Insert(p.subject, struct{}{}); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
@@ -1239,7 +1226,7 @@ func (s *Server) unmarshalRequest(c *client, acc *Account, subject string, msg [
|
||||
|
||||
c.RateLimitWarnf("Invalid JetStream request '%s > %s': %s", acc, subject, err)
|
||||
|
||||
if s.JetStreamConfig().Strict {
|
||||
if js := s.getJetStream(); js != nil && js.config.Strict {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1345,10 +1332,6 @@ func (s *Server) jsAccountInfoRequest(sub *subscription, c *client, _ *Account,
|
||||
}
|
||||
|
||||
// Helpers for token extraction.
|
||||
func templateNameFromSubject(subject string) string {
|
||||
return tokenAt(subject, 6)
|
||||
}
|
||||
|
||||
func streamNameFromSubject(subject string) string {
|
||||
return tokenAt(subject, 5)
|
||||
}
|
||||
@@ -1357,223 +1340,6 @@ func consumerNameFromSubject(subject string) string {
|
||||
return tokenAt(subject, 6)
|
||||
}
|
||||
|
||||
// Request to create a new template.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (s *Server) jsTemplateCreateRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
|
||||
if err != nil {
|
||||
s.Warnf(badAPIRequestT, msg)
|
||||
return
|
||||
}
|
||||
|
||||
var resp = JSApiStreamTemplateCreateResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateCreateResponseType}}
|
||||
if errorOnRequiredApiLevel(hdr) {
|
||||
resp.Error = NewJSRequiredApiLevelError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
if !acc.JetStreamEnabled() {
|
||||
resp.Error = NewJSNotEnabledForAccountError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
|
||||
// Not supported for now.
|
||||
if s.JetStreamIsClustered() {
|
||||
resp.Error = NewJSClusterUnSupportFeatureError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
|
||||
var cfg StreamTemplateConfig
|
||||
if err := s.unmarshalRequest(c, acc, subject, msg, &cfg); err != nil {
|
||||
resp.Error = NewJSInvalidJSONError(err)
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
templateName := templateNameFromSubject(subject)
|
||||
if templateName != cfg.Name {
|
||||
resp.Error = NewJSTemplateNameNotMatchSubjectError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
|
||||
t, err := acc.addStreamTemplate(&cfg)
|
||||
if err != nil {
|
||||
resp.Error = NewJSStreamTemplateCreateError(err, Unless(err))
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
tcfg := t.StreamTemplateConfig.deepCopy()
|
||||
streams := t.streams
|
||||
if streams == nil {
|
||||
streams = []string{}
|
||||
}
|
||||
t.mu.Unlock()
|
||||
resp.StreamTemplateInfo = &StreamTemplateInfo{Config: tcfg, Streams: streams}
|
||||
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
|
||||
}
|
||||
|
||||
// Request for the list of all template names.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (s *Server) jsTemplateNamesRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
|
||||
if err != nil {
|
||||
s.Warnf(badAPIRequestT, msg)
|
||||
return
|
||||
}
|
||||
|
||||
var resp = JSApiStreamTemplateNamesResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateNamesResponseType}}
|
||||
if errorOnRequiredApiLevel(hdr) {
|
||||
resp.Error = NewJSRequiredApiLevelError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
if !acc.JetStreamEnabled() {
|
||||
resp.Error = NewJSNotEnabledForAccountError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
|
||||
// Not supported for now.
|
||||
if s.JetStreamIsClustered() {
|
||||
resp.Error = NewJSClusterUnSupportFeatureError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
|
||||
var offset int
|
||||
if isJSONObjectOrArray(msg) {
|
||||
var req JSApiStreamTemplatesRequest
|
||||
if err := s.unmarshalRequest(c, acc, subject, msg, &req); err != nil {
|
||||
resp.Error = NewJSInvalidJSONError(err)
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
offset = req.Offset
|
||||
}
|
||||
|
||||
ts := acc.templates()
|
||||
slices.SortFunc(ts, func(i, j *streamTemplate) int {
|
||||
return cmp.Compare(i.StreamTemplateConfig.Name, j.StreamTemplateConfig.Name)
|
||||
})
|
||||
|
||||
tcnt := len(ts)
|
||||
if offset > tcnt {
|
||||
offset = tcnt
|
||||
}
|
||||
|
||||
for _, t := range ts[offset:] {
|
||||
t.mu.Lock()
|
||||
name := t.Name
|
||||
t.mu.Unlock()
|
||||
resp.Templates = append(resp.Templates, name)
|
||||
if len(resp.Templates) >= JSApiNamesLimit {
|
||||
break
|
||||
}
|
||||
}
|
||||
resp.Total = tcnt
|
||||
resp.Limit = JSApiNamesLimit
|
||||
resp.Offset = offset
|
||||
if resp.Templates == nil {
|
||||
resp.Templates = []string{}
|
||||
}
|
||||
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
|
||||
}
|
||||
|
||||
// Request for information about a stream template.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (s *Server) jsTemplateInfoRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
|
||||
if err != nil {
|
||||
s.Warnf(badAPIRequestT, msg)
|
||||
return
|
||||
}
|
||||
|
||||
var resp = JSApiStreamTemplateInfoResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateInfoResponseType}}
|
||||
if errorOnRequiredApiLevel(hdr) {
|
||||
resp.Error = NewJSRequiredApiLevelError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
if !acc.JetStreamEnabled() {
|
||||
resp.Error = NewJSNotEnabledForAccountError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
if !isEmptyRequest(msg) {
|
||||
resp.Error = NewJSNotEmptyRequestError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
name := templateNameFromSubject(subject)
|
||||
t, err := acc.lookupStreamTemplate(name)
|
||||
if err != nil {
|
||||
resp.Error = NewJSStreamTemplateNotFoundError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
t.mu.Lock()
|
||||
cfg := t.StreamTemplateConfig.deepCopy()
|
||||
streams := t.streams
|
||||
if streams == nil {
|
||||
streams = []string{}
|
||||
}
|
||||
t.mu.Unlock()
|
||||
|
||||
resp.StreamTemplateInfo = &StreamTemplateInfo{Config: cfg, Streams: streams}
|
||||
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
|
||||
}
|
||||
|
||||
// Request to delete a stream template.
|
||||
// Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
func (s *Server) jsTemplateDeleteRequest(sub *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
ci, acc, hdr, msg, err := s.getRequestInfo(c, rmsg)
|
||||
if err != nil {
|
||||
s.Warnf(badAPIRequestT, msg)
|
||||
return
|
||||
}
|
||||
|
||||
var resp = JSApiStreamTemplateDeleteResponse{ApiResponse: ApiResponse{Type: JSApiStreamTemplateDeleteResponseType}}
|
||||
if errorOnRequiredApiLevel(hdr) {
|
||||
resp.Error = NewJSRequiredApiLevelError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
if !acc.JetStreamEnabled() {
|
||||
resp.Error = NewJSNotEnabledForAccountError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
if !isEmptyRequest(msg) {
|
||||
resp.Error = NewJSNotEmptyRequestError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
name := templateNameFromSubject(subject)
|
||||
err = acc.deleteStreamTemplate(name)
|
||||
if err != nil {
|
||||
resp.Error = NewJSStreamTemplateDeleteError(err, Unless(err))
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
resp.Success = true
|
||||
s.sendAPIResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(resp))
|
||||
}
|
||||
|
||||
func (s *Server) jsonResponse(v any) string {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
@@ -2109,7 +1875,7 @@ func (s *Server) jsStreamInfoRequest(sub *subscription, c *client, a *Account, s
|
||||
if cc != nil {
|
||||
// Check to make sure the stream is assigned.
|
||||
js.mu.RLock()
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, streamName)
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, streamName)
|
||||
var offline bool
|
||||
if sa != nil {
|
||||
clusterWideConsCount = len(sa.consumers)
|
||||
@@ -2338,7 +2104,7 @@ func (s *Server) jsStreamLeaderStepDownRequest(sub *subscription, c *client, _ *
|
||||
}
|
||||
|
||||
js.mu.RLock()
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, name)
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, name)
|
||||
js.mu.RUnlock()
|
||||
|
||||
if isLeader && sa == nil {
|
||||
@@ -2455,7 +2221,7 @@ func (s *Server) jsConsumerLeaderStepDownRequest(sub *subscription, c *client, _
|
||||
consumer := tokenAt(subject, 7)
|
||||
|
||||
js.mu.RLock()
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
|
||||
js.mu.RUnlock()
|
||||
|
||||
if isLeader && sa == nil {
|
||||
@@ -3456,7 +3222,7 @@ func (s *Server) jsMsgDeleteRequest(sub *subscription, c *client, _ *Account, su
|
||||
}
|
||||
|
||||
js.mu.RLock()
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
|
||||
js.mu.RUnlock()
|
||||
|
||||
if isLeader && sa == nil {
|
||||
@@ -3581,7 +3347,7 @@ func (s *Server) jsMsgGetRequest(sub *subscription, c *client, _ *Account, subje
|
||||
}
|
||||
|
||||
js.mu.RLock()
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
|
||||
js.mu.RUnlock()
|
||||
|
||||
if isLeader && sa == nil {
|
||||
@@ -3876,7 +3642,7 @@ func (s *Server) jsStreamPurgeRequest(sub *subscription, c *client, _ *Account,
|
||||
}
|
||||
|
||||
js.mu.RLock()
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignment(acc.Name, stream)
|
||||
isLeader, sa := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, stream)
|
||||
js.mu.RUnlock()
|
||||
|
||||
if isLeader && sa == nil {
|
||||
@@ -4048,6 +3814,13 @@ func (s *Server) jsStreamRestoreRequest(sub *subscription, c *client, _ *Account
|
||||
return
|
||||
}
|
||||
|
||||
// Check for path like separators in the name.
|
||||
if strings.ContainsAny(stream, `\/`) {
|
||||
resp.Error = NewJSStreamNameContainsPathSeparatorsError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
|
||||
if s.JetStreamIsClustered() {
|
||||
s.jsClusteredStreamRestoreRequest(ci, acc, &req, subject, reply, rmsg)
|
||||
return
|
||||
@@ -4597,11 +4370,49 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
|
||||
isClustered := s.JetStreamIsClustered()
|
||||
|
||||
// Determine if we should proceed here when we are in clustered mode.
|
||||
direct := req.Config.Direct
|
||||
if isClustered {
|
||||
if req.Config.Direct {
|
||||
// Check to see if we have this stream and are the stream leader.
|
||||
if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {
|
||||
return
|
||||
if direct {
|
||||
// If it's just a direct consumer, check for stream leader.
|
||||
if !req.Config.Sourcing {
|
||||
// Check to see if we have this stream and are the stream leader.
|
||||
if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
// Otherwise, we either need this to be answered by the stream or meta leader.
|
||||
var cc *jetStreamCluster
|
||||
js, cc = s.getJetStreamCluster()
|
||||
if js == nil || cc == nil {
|
||||
return
|
||||
}
|
||||
js.mu.RLock()
|
||||
sa := js.streamAssignmentOrInflight(acc.Name, streamNameFromSubject(subject))
|
||||
if sa == nil {
|
||||
js.mu.RUnlock()
|
||||
return
|
||||
}
|
||||
// If the stream is WQ or Interest, we need the meta leader to answer.
|
||||
if sa.Config.Retention != LimitsPolicy {
|
||||
direct = false
|
||||
}
|
||||
js.mu.RUnlock()
|
||||
if direct {
|
||||
// Check to see if we have this stream and are the stream leader.
|
||||
if !acc.JetStreamIsStreamLeader(streamNameFromSubject(subject)) {
|
||||
return
|
||||
}
|
||||
} else {
|
||||
if js.isLeaderless() {
|
||||
resp.Error = NewJSClusterNotAvailError()
|
||||
s.sendAPIErrResponse(ci, acc, subject, reply, string(msg), s.jsonResponse(&resp))
|
||||
return
|
||||
}
|
||||
// Make sure we are meta leader.
|
||||
if !s.JetStreamIsLeader() {
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
var cc *jetStreamCluster
|
||||
@@ -4645,6 +4456,7 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
|
||||
// Legacy ephemeral.
|
||||
rt = ccLegacyEphemeral
|
||||
streamName = streamNameFromSubject(subject)
|
||||
consumerName = req.Config.Name
|
||||
} else {
|
||||
// New style and durable legacy.
|
||||
if tokenAt(subject, 4) == "DURABLE" {
|
||||
@@ -4736,7 +4548,7 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
|
||||
return
|
||||
}
|
||||
|
||||
if isClustered && !req.Config.Direct {
|
||||
if isClustered && !direct {
|
||||
s.jsClusteredConsumerRequest(ci, acc, subject, reply, rmsg, req.Stream, &req.Config, req.Action, req.Pedantic)
|
||||
return
|
||||
}
|
||||
@@ -4760,6 +4572,23 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
|
||||
return
|
||||
}
|
||||
|
||||
// If the consumer is a direct sourcing consumer, we need to "upgrade"
|
||||
// it to be durable without AckNone if not a Limits-based stream.
|
||||
if req.Config.Direct && req.Config.Sourcing && req.Config.Name != _EMPTY_ {
|
||||
if !isClustered && stream.isInterestRetention() {
|
||||
req.Config.Direct = false
|
||||
req.Config.Durable = req.Config.Name
|
||||
req.Config.AckPolicy = AckFlowControl
|
||||
req.Config.AckWait = 0
|
||||
req.Config.MaxDeliver = 0
|
||||
req.Config.InactiveThreshold = 0
|
||||
} else {
|
||||
// Otherwise, need to append a randomized suffix since the source uses a stable name.
|
||||
req.Config.Name = fmt.Sprintf("%s-%s", req.Config.Name, createConsumerName())
|
||||
consumerName = req.Config.Name
|
||||
}
|
||||
}
|
||||
|
||||
if o := stream.lookupConsumer(consumerName); o != nil {
|
||||
if o.offlineReason != _EMPTY_ {
|
||||
resp.Error = NewJSConsumerOfflineReasonError(errors.New(o.offlineReason))
|
||||
@@ -4770,6 +4599,12 @@ func (s *Server) jsConsumerCreateRequest(sub *subscription, c *client, a *Accoun
|
||||
// it back to whatever the current configured value is.
|
||||
o.mu.RLock()
|
||||
req.Config.PauseUntil = o.cfg.PauseUntil
|
||||
// If a durable sourcing consumer is used, we need to reset the deliver policy.
|
||||
if req.Config.Sourcing && req.Config.Durable != _EMPTY_ {
|
||||
req.Config.DeliverPolicy = o.cfg.DeliverPolicy
|
||||
req.Config.OptStartSeq = o.cfg.OptStartSeq
|
||||
req.Config.OptStartTime = o.cfg.OptStartTime
|
||||
}
|
||||
o.mu.RUnlock()
|
||||
}
|
||||
|
||||
@@ -5079,7 +4914,7 @@ func (s *Server) jsConsumerInfoRequest(sub *subscription, c *client, _ *Account,
|
||||
groupCreated := meta.Created()
|
||||
|
||||
js.mu.RLock()
|
||||
isLeader, sa, ca := cc.isLeader(), js.streamAssignment(acc.Name, streamName), js.consumerAssignment(acc.Name, streamName, consumerName)
|
||||
isLeader, sa, ca := cc.isLeader(), js.streamAssignmentOrInflight(acc.Name, streamName), js.consumerAssignmentOrInflight(acc.Name, streamName, consumerName)
|
||||
var rg *raftGroup
|
||||
var offline, isMember bool
|
||||
if ca != nil {
|
||||
@@ -5404,8 +5239,11 @@ func (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account
|
||||
return
|
||||
}
|
||||
|
||||
nca := *ca
|
||||
nca := ca.clone()
|
||||
// We're only holding the read lock and release below,
|
||||
// we need a copy to prevent concurrent reads/writes.
|
||||
ncfg := *ca.Config
|
||||
ncfg.Metadata = maps.Clone(ncfg.Metadata)
|
||||
nca.Config = &ncfg
|
||||
meta := cc.meta
|
||||
js.mu.RUnlock()
|
||||
@@ -5420,7 +5258,7 @@ func (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account
|
||||
// Only PauseUntil is updated above, so reuse config for both.
|
||||
setStaticConsumerMetadata(nca.Config)
|
||||
|
||||
eca := encodeAddConsumerAssignment(&nca)
|
||||
eca := encodeAddConsumerAssignment(nca)
|
||||
meta.Propose(eca)
|
||||
|
||||
resp.PauseUntil = pauseUTC
|
||||
@@ -5453,7 +5291,13 @@ func (s *Server) jsConsumerPauseRequest(sub *subscription, c *client, _ *Account
|
||||
return
|
||||
}
|
||||
|
||||
// We're only holding the read lock and release below,
|
||||
// we need a copy to prevent concurrent reads/writes.
|
||||
obs.mu.RLock()
|
||||
ncfg := obs.cfg
|
||||
ncfg.Metadata = maps.Clone(ncfg.Metadata)
|
||||
obs.mu.RUnlock()
|
||||
|
||||
pauseUTC := req.PauseUntil.UTC()
|
||||
if !pauseUTC.IsZero() {
|
||||
ncfg.PauseUntil = &pauseUTC
|
||||
|
||||
+455
-48
@@ -21,6 +21,7 @@ import (
|
||||
"math/big"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
@@ -29,60 +30,105 @@ import (
|
||||
|
||||
var (
|
||||
// Tracks the total inflight batches, across all streams and accounts that enable batching.
|
||||
globalInflightBatches atomic.Int32
|
||||
globalInflightAtomicBatches atomic.Int64
|
||||
globalInflightFastBatches atomic.Int64
|
||||
)
|
||||
|
||||
type batching struct {
|
||||
mu sync.Mutex
|
||||
group map[string]*batchGroup
|
||||
mu sync.Mutex
|
||||
atomic map[string]*atomicBatch
|
||||
fast map[string]*fastBatch
|
||||
}
|
||||
|
||||
type batchGroup struct {
|
||||
lseq uint64
|
||||
store StreamStore
|
||||
timer *time.Timer
|
||||
type atomicBatch struct {
|
||||
timer *time.Timer // Inactivity timer for the batch.
|
||||
lseq uint64 // The highest sequence for this batch.
|
||||
store StreamStore // Where the batch is staged before committing.
|
||||
}
|
||||
|
||||
type fastBatch struct {
|
||||
timer *time.Timer // Inactivity timer for the batch.
|
||||
lseq uint64 // The highest sequence for this batch.
|
||||
sseq uint64 // Last persisted stream sequence.
|
||||
pseq uint64 // Last persisted batch sequence (is always lower or equal to lseq).
|
||||
fseq uint64 // Sequence of when we last sent a flow message (is always lower or equal to pseq).
|
||||
pending uint32 // Number of pending messages in the batch waiting to be persisted.
|
||||
ackMessages uint16 // Ack will be sent every N messages.
|
||||
maxAckMessages uint16 // Maximum ackMessages value the client allows.
|
||||
reply string // The last reply subject seen when persisting a message.
|
||||
gapOk bool // Whether a gap is okay, if not, the batch would be rejected.
|
||||
commit bool // If the batch is committed.
|
||||
}
|
||||
|
||||
// newAtomicBatch creates an atomic batch publish object.
|
||||
// Lock should be held.
|
||||
func (batches *batching) newBatchGroup(mset *stream, batchId string) (*batchGroup, error) {
|
||||
store, err := newBatchStore(mset, batchId)
|
||||
func (batches *batching) newAtomicBatch(mset *stream, batchId string, replicas int, storage StorageType, storeDir, streamName string) (*atomicBatch, error) {
|
||||
store, err := newBatchStore(mset, batchId, replicas, storage, storeDir, streamName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
b := &batchGroup{store: store}
|
||||
b := &atomicBatch{store: store}
|
||||
b.setupCleanupTimer(mset, batchId, batches)
|
||||
return b, nil
|
||||
}
|
||||
|
||||
// setupCleanupTimer sets up a timer to clean up the batch after a timeout.
|
||||
func (b *atomicBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) {
|
||||
// Create a timer to clean up after timeout.
|
||||
timeout := streamMaxBatchTimeout
|
||||
if maxBatchTimeout := mset.srv.getOpts().JetStreamLimits.MaxBatchTimeout; maxBatchTimeout > 0 {
|
||||
timeout = maxBatchTimeout
|
||||
}
|
||||
timeout := getCleanupTimeout(mset)
|
||||
b.timer = time.AfterFunc(timeout, func() {
|
||||
b.cleanup(batchId, batches)
|
||||
mset.sendStreamBatchAbandonedAdvisory(batchId, BatchTimeout)
|
||||
})
|
||||
return b, nil
|
||||
}
|
||||
|
||||
func getBatchStoreDir(mset *stream, batchId string) (string, string) {
|
||||
mset.mu.RLock()
|
||||
jsa, name := mset.jsa, mset.cfg.Name
|
||||
mset.mu.RUnlock()
|
||||
// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch.
|
||||
// Returns whether the timer was reset without it having expired before.
|
||||
func (b *atomicBatch) resetCleanupTimer(mset *stream) bool {
|
||||
timeout := getCleanupTimeout(mset)
|
||||
return b.timer.Reset(timeout)
|
||||
}
|
||||
|
||||
jsa.mu.RLock()
|
||||
sd := jsa.storeDir
|
||||
jsa.mu.RUnlock()
|
||||
// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches.
|
||||
func (b *atomicBatch) cleanup(batchId string, batches *batching) {
|
||||
batches.mu.Lock()
|
||||
defer batches.mu.Unlock()
|
||||
b.cleanupLocked(batchId, batches)
|
||||
}
|
||||
|
||||
// Lock should be held.
|
||||
func (b *atomicBatch) cleanupLocked(batchId string, batches *batching) {
|
||||
if b.timer == nil {
|
||||
return
|
||||
}
|
||||
globalInflightAtomicBatches.Add(-1)
|
||||
b.timer.Stop()
|
||||
b.store.Delete(true)
|
||||
delete(batches.atomic, batchId)
|
||||
// Reset so that another invocation doesn't double-account.
|
||||
b.timer = nil
|
||||
}
|
||||
|
||||
// Lock should be held.
|
||||
func (b *atomicBatch) stopLocked() {
|
||||
if b.timer == nil {
|
||||
return
|
||||
}
|
||||
globalInflightAtomicBatches.Add(-1)
|
||||
b.timer.Stop()
|
||||
b.store.Stop()
|
||||
// Reset so that another invocation doesn't double-account.
|
||||
b.timer = nil
|
||||
}
|
||||
|
||||
func getBatchStoreDir(storeDir, streamName, batchId string) (string, string) {
|
||||
bname := getHash(batchId)
|
||||
return bname, filepath.Join(sd, streamsDir, name, batchesDir, bname)
|
||||
return bname, filepath.Join(storeDir, streamsDir, streamName, batchesDir, bname)
|
||||
}
|
||||
|
||||
func newBatchStore(mset *stream, batchId string) (StreamStore, error) {
|
||||
mset.mu.RLock()
|
||||
replicas, storage := mset.cfg.Replicas, mset.cfg.Storage
|
||||
mset.mu.RUnlock()
|
||||
|
||||
func newBatchStore(mset *stream, batchId string, replicas int, storage StorageType, storeDir, streamName string) (StreamStore, error) {
|
||||
if replicas == 1 && storage == FileStorage {
|
||||
bname, storeDir := getBatchStoreDir(mset, batchId)
|
||||
bname, storeDir := getBatchStoreDir(storeDir, streamName, batchId)
|
||||
fcfg := FileStoreConfig{AsyncFlush: true, BlockSize: defaultLargeBlockSize, StoreDir: storeDir}
|
||||
s := mset.srv
|
||||
prf := s.jsKeyGen(s.getOpts().JetStreamKey, mset.acc.Name)
|
||||
@@ -101,34 +147,264 @@ func newBatchStore(mset *stream, batchId string) (StreamStore, error) {
|
||||
// If the timer has already cleaned up the batch, we can't commit.
|
||||
// Otherwise, we ensure the timer does not clean up the batch in the meantime.
|
||||
// Lock should be held.
|
||||
func (b *batchGroup) readyForCommit() bool {
|
||||
func (b *atomicBatch) readyForCommit() *BatchAbandonReason {
|
||||
if !b.timer.Stop() {
|
||||
return &BatchTimeout
|
||||
}
|
||||
if b.store.FlushAllPending() != nil {
|
||||
return &BatchIncomplete
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// newFastBatch creates a fast batch publish object and registers it in batches.fast.
|
||||
// Lock should be held.
|
||||
func (batches *batching) newFastBatch(mset *stream, batchId string, gapOk bool, maxAckMessages uint16) *fastBatch {
|
||||
b := &fastBatch{gapOk: gapOk, maxAckMessages: maxAckMessages}
|
||||
if batches.fast == nil {
|
||||
batches.fast = make(map[string]*fastBatch, 1)
|
||||
}
|
||||
batches.fast[batchId] = b
|
||||
batches.fastBatchInit(b)
|
||||
b.setupCleanupTimer(mset, batchId, batches)
|
||||
return b
|
||||
}
|
||||
|
||||
// fastBatchInit (re)initializes the ackMessages field for a fast batch.
|
||||
// The batch must already be registered in batches.fast.
|
||||
// Lock should be held.
|
||||
func (batches *batching) fastBatchInit(b *fastBatch) {
|
||||
// If it's the only batch, just allow what the client wants, otherwise we'll
|
||||
// need to coordinate and slowly ramp up this publisher.
|
||||
// TODO(mvv): fast ingest's initial flow value improvements?
|
||||
ackMessages := min(500, b.maxAckMessages)
|
||||
if len(batches.fast) > 1 {
|
||||
ackMessages = 1
|
||||
}
|
||||
b.ackMessages = ackMessages
|
||||
}
|
||||
|
||||
// fastBatchReset resets the fast batch to an empty state and sends a flow control message.
|
||||
// Lock should be held.
|
||||
func (batches *batching) fastBatchReset(mset *stream, batchId string, b *fastBatch) {
|
||||
// If the timer already stopped before we could commit, we clean it up.
|
||||
if b.timer == nil || (!b.commit && !b.timer.Stop()) {
|
||||
b.cleanupLocked(batchId, batches)
|
||||
return
|
||||
}
|
||||
// Otherwise, reset the state.
|
||||
batches.fastBatchInit(b)
|
||||
b.timer.Reset(getCleanupTimeout(mset))
|
||||
b.commit = false
|
||||
b.pending = 0
|
||||
b.fseq, b.lseq = b.pseq, b.pseq
|
||||
b.sendFlowControl(b.fseq, mset, b.reply)
|
||||
}
|
||||
|
||||
// fastBatchRegisterSequences registers the highest stored batch and stream sequence and returns
|
||||
// whether a PubAck should be sent if the batch has been committed.
|
||||
// If this is called on a follower, it only registers the highest stream and persisted batch sequences.
|
||||
// Lock should be held.
|
||||
func (batches *batching) fastBatchRegisterSequences(mset *stream, reply string, streamSeq uint64, isLeader bool, batch *FastBatch) bool {
|
||||
b, ok := batches.fast[batch.id]
|
||||
if !ok || !isLeader {
|
||||
// If this batch has committed, we can clean it up.
|
||||
if batch.commit {
|
||||
if b != nil {
|
||||
b.cleanupLocked(batch.id, batches)
|
||||
}
|
||||
return false
|
||||
}
|
||||
// Otherwise, even as a follower, we record the latest state of this batch.
|
||||
if b == nil || !b.resetCleanupTimer(mset) {
|
||||
if b != nil {
|
||||
// The timer couldn't be reset, this means the timer already runs and is likely
|
||||
// waiting to acquire the lock. We reset the timer here so it doesn't clean up
|
||||
// this batch that we're about to overwrite.
|
||||
b.timer = nil
|
||||
} else {
|
||||
// If this is a new batch for us, even though we're a follower, we still need
|
||||
// to account toward the global inflight limit.
|
||||
globalInflightFastBatches.Add(1)
|
||||
}
|
||||
// We'll need a copy as we'll use it as a key and later for cleanup.
|
||||
batchId := copyString(batch.id)
|
||||
b = batches.newFastBatch(mset, batchId, batch.gapOk, batch.flow)
|
||||
}
|
||||
b.sseq = streamSeq
|
||||
b.pseq, b.lseq = batch.seq, batch.seq
|
||||
b.reply = reply
|
||||
return false
|
||||
}
|
||||
b.store.FlushAllPending()
|
||||
b.reply = reply
|
||||
if b.pending > 0 {
|
||||
b.pending--
|
||||
}
|
||||
b.sseq = streamSeq
|
||||
// Store last persisted batch sequence.
|
||||
// If we have no remaining pending writes, we might have had duplicate messages
|
||||
// and need to send additional flow control messages.
|
||||
var skipped bool
|
||||
if b.pending == 0 {
|
||||
skipped = true
|
||||
b.pseq = b.lseq
|
||||
} else {
|
||||
b.pseq = batch.seq
|
||||
}
|
||||
// If the PubAck needs to be sent now as a result of a commit.
|
||||
if b.lseq == b.pseq && b.commit {
|
||||
b.cleanupLocked(batch.id, batches)
|
||||
// If we skipped ahead due to duplicate messages, send the PubAck with the highest sequence.
|
||||
if skipped {
|
||||
var buf [256]byte
|
||||
pubAck := append(buf[:0], mset.pubAck...)
|
||||
response := append(pubAck, strconv.FormatUint(b.sseq, 10)...)
|
||||
response = append(response, fmt.Sprintf(",\"batch\":%q,\"count\":%d}", batch.id, b.lseq)...)
|
||||
if len(reply) > 0 {
|
||||
mset.outq.sendMsg(reply, response)
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
b.checkFlowControl(mset, reply, batches)
|
||||
return false
|
||||
}
|
||||
|
||||
// checkFlowControl checks whether a flow control message should be sent.
|
||||
// If so, it updates the flow values to speed up or slow down the publisher if needed.
|
||||
// Returns whether a flow control message was sent.
|
||||
// Lock should be held.
|
||||
func (b *fastBatch) checkFlowControl(mset *stream, reply string, batches *batching) bool {
|
||||
am := uint64(b.ackMessages)
|
||||
if b.pseq < b.fseq+am {
|
||||
return false
|
||||
}
|
||||
// Instead of sending multiple flow control messages, skip ahead to only send the last.
|
||||
steps := (b.pseq - b.fseq) / am
|
||||
b.fseq += steps * am
|
||||
|
||||
// TODO(mvv): fast ingest's dynamic flow value improvements?
|
||||
// This is currently just a simple value to have a working version. Should take average
|
||||
// message sizes into account and compare how much this client is contributing to the
|
||||
// ingest IPQ total size and messages and have publishers share based on that.
|
||||
maxAckMessages := uint16(500 / len(batches.fast))
|
||||
if maxAckMessages < 1 {
|
||||
maxAckMessages = 1
|
||||
}
|
||||
// Limit to the client's allowed maximum.
|
||||
if maxAckMessages > b.maxAckMessages {
|
||||
maxAckMessages = b.maxAckMessages
|
||||
}
|
||||
|
||||
if b.ackMessages < maxAckMessages {
|
||||
// Ramp up.
|
||||
b.ackMessages *= 2
|
||||
if b.ackMessages > maxAckMessages {
|
||||
b.ackMessages = maxAckMessages
|
||||
}
|
||||
} else if b.ackMessages > maxAckMessages {
|
||||
// Slow down.
|
||||
b.ackMessages /= 2
|
||||
if b.ackMessages <= maxAckMessages {
|
||||
b.ackMessages = maxAckMessages
|
||||
}
|
||||
}
|
||||
|
||||
// Finally, send the flow control message.
|
||||
b.sendFlowControl(b.fseq, mset, reply)
|
||||
return true
|
||||
}
|
||||
|
||||
// sendFlowControl sends a fast batch flow control message for the current highest sequence.
|
||||
// Lock should be held.
|
||||
func (b *fastBatch) sendFlowControl(batchSeq uint64, mset *stream, reply string) {
|
||||
if len(reply) == 0 {
|
||||
return
|
||||
}
|
||||
response, _ := BatchFlowAck{Sequence: batchSeq, Messages: b.ackMessages}.MarshalJSON()
|
||||
mset.outq.sendMsg(reply, response)
|
||||
}
|
||||
|
||||
// fastBatchCommit ends the batch and commits the data up to that point. If all messages
|
||||
// have already been persisted, a PubAck is sent immediately. Otherwise, it will be sent
|
||||
// after the last message has been persisted.
|
||||
// Lock should be held.
|
||||
func (batches *batching) fastBatchCommit(b *fastBatch, batchId string, mset *stream, reply string) bool {
|
||||
// Either we commit now, or we clean up later, so stop the timer.
|
||||
if b.timer == nil || (!b.commit && !b.timer.Stop()) {
|
||||
// Shouldn't be possible for the timer to already be stopped if we haven't committed yet,
|
||||
// since we pre-check being able to reset the timer. But guard against it anyhow.
|
||||
return true
|
||||
}
|
||||
// Mark that this batch commits.
|
||||
b.commit = true
|
||||
// If the whole batch has been persisted, we can respond with the PubAck now.
|
||||
if b.lseq == b.pseq {
|
||||
b.cleanupLocked(batchId, batches)
|
||||
var buf [256]byte
|
||||
pubAck := append(buf[:0], mset.pubAck...)
|
||||
response := append(pubAck, strconv.FormatUint(b.sseq, 10)...)
|
||||
response = append(response, fmt.Sprintf(",\"batch\":%q,\"count\":%d}", batchId, b.lseq)...)
|
||||
if len(reply) > 0 {
|
||||
mset.outq.sendMsg(reply, response)
|
||||
}
|
||||
return true
|
||||
}
|
||||
// Otherwise, we need to wait and the PubAck will be sent when the last message is persisted.
|
||||
return false
|
||||
}
|
||||
|
||||
// setupCleanupTimer sets up a timer to clean up the batch after a timeout.
|
||||
func (b *fastBatch) setupCleanupTimer(mset *stream, batchId string, batches *batching) {
|
||||
// Create a timer to clean up after timeout.
|
||||
timeout := getCleanupTimeout(mset)
|
||||
b.timer = time.AfterFunc(timeout, func() {
|
||||
b.cleanup(batchId, batches)
|
||||
})
|
||||
}
|
||||
|
||||
// resetCleanupTimer resets the cleanup timer, allowing to extend the lifetime of the batch.
|
||||
// Returns whether the timer was reset without it having expired before.
|
||||
func (b *fastBatch) resetCleanupTimer(mset *stream) bool {
|
||||
if b.commit {
|
||||
return true
|
||||
}
|
||||
if b.timer == nil {
|
||||
return false
|
||||
}
|
||||
timeout := getCleanupTimeout(mset)
|
||||
return b.timer.Reset(timeout)
|
||||
}
|
||||
|
||||
// cleanup deletes underlying resources associated with the batch and unregisters it from the stream's batches.
|
||||
func (b *batchGroup) cleanup(batchId string, batches *batching) {
|
||||
func (b *fastBatch) cleanup(batchId string, batches *batching) {
|
||||
batches.mu.Lock()
|
||||
defer batches.mu.Unlock()
|
||||
b.cleanupLocked(batchId, batches)
|
||||
}
|
||||
|
||||
// Lock should be held.
|
||||
func (b *batchGroup) cleanupLocked(batchId string, batches *batching) {
|
||||
globalInflightBatches.Add(-1)
|
||||
func (b *fastBatch) cleanupLocked(batchId string, batches *batching) {
|
||||
// If the timer is nil, it means this batch has been replaced with a new one.
|
||||
// This can happen on a follower depending on timing.
|
||||
if b.timer == nil {
|
||||
return
|
||||
}
|
||||
globalInflightFastBatches.Add(-1)
|
||||
b.timer.Stop()
|
||||
b.store.Delete(true)
|
||||
delete(batches.group, batchId)
|
||||
delete(batches.fast, batchId)
|
||||
// Reset so that another invocation doesn't double-account.
|
||||
b.timer = nil
|
||||
}
|
||||
|
||||
// Lock should be held.
|
||||
func (b *batchGroup) stopLocked() {
|
||||
globalInflightBatches.Add(-1)
|
||||
b.timer.Stop()
|
||||
b.store.Stop()
|
||||
// getCleanupTimeout returns the timeout for the batch, taking into account the server's limits.
|
||||
func getCleanupTimeout(mset *stream) time.Duration {
|
||||
timeout := streamMaxBatchTimeout
|
||||
if maxBatchTimeout := mset.srv.getOpts().JetStreamLimits.MaxBatchTimeout; maxBatchTimeout > 0 {
|
||||
timeout = maxBatchTimeout
|
||||
}
|
||||
return timeout
|
||||
}
|
||||
|
||||
// batchStagedDiff stages all changes for consistency checks until commit.
|
||||
@@ -136,6 +412,7 @@ type batchStagedDiff struct {
|
||||
msgIds map[string]struct{}
|
||||
counter map[string]*msgCounterRunningTotal
|
||||
inflight map[string]*inflightSubjectRunningTotal
|
||||
inflightTransform map[uint64]string
|
||||
expectedPerSubject map[string]*batchExpectedPerSubject
|
||||
}
|
||||
|
||||
@@ -180,6 +457,16 @@ func (diff *batchStagedDiff) commit(mset *stream) {
|
||||
}
|
||||
}
|
||||
|
||||
// Track inflight subject transforms.
|
||||
if len(diff.inflightTransform) > 0 {
|
||||
if mset.inflightTransform == nil {
|
||||
mset.inflightTransform = make(map[uint64]string, len(diff.inflightTransform))
|
||||
}
|
||||
for clseq, subj := range diff.inflightTransform {
|
||||
mset.inflightTransform[clseq] = subj
|
||||
}
|
||||
}
|
||||
|
||||
// Track sequence and subject.
|
||||
if len(diff.expectedPerSubject) > 0 {
|
||||
if mset.expectedPerSubjectSequence == nil {
|
||||
@@ -238,7 +525,7 @@ func (batch *batchApply) rejectBatchState(mset *stream) {
|
||||
// mset.mu lock must NOT be held or used.
|
||||
// mset.clMu lock must be held.
|
||||
func checkMsgHeadersPreClusteredProposal(
|
||||
diff *batchStagedDiff, mset *stream, subject string, hdr []byte, msg []byte, sourced bool, name string,
|
||||
diff *batchStagedDiff, mset *stream, subject, rsubject string, hdr []byte, msg []byte, sourced bool, name string,
|
||||
jsa *jsAccount, allowRollup, denyPurge, allowTTL, allowMsgCounter, allowMsgSchedules bool,
|
||||
discard DiscardPolicy, discardNewPer bool, maxMsgSize int, maxMsgs int64, maxMsgsPer int64, maxBytes int64,
|
||||
) ([]byte, []byte, uint64, *ApiError, error) {
|
||||
@@ -515,8 +802,9 @@ func checkMsgHeadersPreClusteredProposal(
|
||||
}
|
||||
|
||||
// Message scheduling.
|
||||
if schedule, ok := getMessageSchedule(hdr); !ok {
|
||||
apiErr := NewJSMessageSchedulesPatternInvalidError()
|
||||
if sourced {
|
||||
// noop, sourced messages were already validated by the origin stream.
|
||||
} else if schedule, apiErr := getMessageSchedule(hdr); apiErr != nil {
|
||||
if !allowMsgSchedules {
|
||||
apiErr = NewJSMessageSchedulesDisabledError()
|
||||
}
|
||||
@@ -528,22 +816,40 @@ func checkMsgHeadersPreClusteredProposal(
|
||||
} else if scheduleTtl, ok := getMessageScheduleTTL(hdr); !ok {
|
||||
apiErr := NewJSMessageSchedulesTTLInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
} else if scheduleRollup := getMessageScheduleRollup(hdr); scheduleRollup != _EMPTY_ && scheduleRollup != JSMsgRollupSubject {
|
||||
apiErr := NewJSMessageSchedulesRollupInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
} else if scheduleTtl != _EMPTY_ && !allowTTL {
|
||||
return hdr, msg, 0, NewJSMessageTTLDisabledError(), errMsgTTLDisabled
|
||||
} else if scheduleTarget := getMessageScheduleTarget(hdr); scheduleTarget == _EMPTY_ ||
|
||||
!IsValidPublishSubject(scheduleTarget) || SubjectsCollide(scheduleTarget, subject) {
|
||||
!IsValidPublishSubject(scheduleTarget) || scheduleTarget == subject {
|
||||
apiErr := NewJSMessageSchedulesTargetInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
} else if scheduleSource := getMessageScheduleSource(hdr); scheduleSource != _EMPTY_ &&
|
||||
(scheduleSource == scheduleTarget || scheduleSource == subject || !IsValidPublishSubject(scheduleSource)) {
|
||||
apiErr := NewJSMessageSchedulesSourceInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
} else {
|
||||
mset.cfgMu.RLock()
|
||||
match := slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool {
|
||||
return SubjectsCollide(subj, scheduleTarget)
|
||||
})
|
||||
mset.cfgMu.RUnlock()
|
||||
if !match {
|
||||
mset.cfgMu.RUnlock()
|
||||
apiErr := NewJSMessageSchedulesTargetInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
}
|
||||
if scheduleSource != _EMPTY_ {
|
||||
match = slices.ContainsFunc(mset.cfg.Subjects, func(subj string) bool {
|
||||
return SubjectsCollide(subj, scheduleSource)
|
||||
})
|
||||
if !match {
|
||||
mset.cfgMu.RUnlock()
|
||||
apiErr := NewJSMessageSchedulesSourceInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
}
|
||||
}
|
||||
mset.cfgMu.RUnlock()
|
||||
|
||||
// Add a rollup sub header if it doesn't already exist.
|
||||
// Otherwise, it must exist already as a rollup on the subject.
|
||||
@@ -555,10 +861,32 @@ func checkMsgHeadersPreClusteredProposal(
|
||||
}
|
||||
}
|
||||
}
|
||||
if scheduleNext := sliceHeader(JSScheduleNext, hdr); len(scheduleNext) > 0 && !sourced {
|
||||
// Clients may only use Nats-Schedule-Next to purge a schedule.
|
||||
if bytesToString(scheduleNext) != JSScheduleNextPurge {
|
||||
apiErr := NewJSMessageSchedulesSchedulerInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
}
|
||||
// Nats-Scheduler must accompany the purge and:
|
||||
// - it must NOT be empty.
|
||||
// - it must NOT match the publish subject.
|
||||
if scheduler := sliceHeader(JSScheduler, hdr); len(scheduler) == 0 ||
|
||||
bytesToString(scheduler) == subject || !IsValidPublishSubject(bytesToString(scheduler)) {
|
||||
apiErr := NewJSMessageSchedulesSchedulerInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
} else if !allowMsgSchedules {
|
||||
apiErr := NewJSMessageSchedulesDisabledError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
}
|
||||
} else if !sourced && len(sliceHeader(JSScheduler, hdr)) > 0 {
|
||||
// Clients may only use Nats-Scheduler alongside Nats-Schedule-Next.
|
||||
apiErr := NewJSMessageSchedulesSchedulerInvalidError()
|
||||
return hdr, msg, 0, apiErr, apiErr
|
||||
}
|
||||
|
||||
// Check for any rollups.
|
||||
if rollup := getRollup(hdr); rollup != _EMPTY_ {
|
||||
if !allowRollup || denyPurge {
|
||||
if (!allowRollup || denyPurge) && !sourced {
|
||||
err := errors.New("rollup not permitted")
|
||||
return hdr, msg, 0, NewJSStreamRollupFailedError(err), err
|
||||
}
|
||||
@@ -607,6 +935,19 @@ func checkMsgHeadersPreClusteredProposal(
|
||||
diff.inflight[subject] = i
|
||||
}
|
||||
|
||||
// Subject transform.
|
||||
if subject != rsubject {
|
||||
// The 'subject' is a transformed subject used for consistency checks.
|
||||
// But since we propose the original (raw) subject to our peers, we need
|
||||
// to store the transformed subject separately for when we apply.
|
||||
// TODO(mvv): since subject transforms are handled by each replica individually, this has a
|
||||
// potential for desync given out-of-order stream subject transform updates.
|
||||
if diff.inflightTransform == nil {
|
||||
diff.inflightTransform = make(map[uint64]string, 1)
|
||||
}
|
||||
diff.inflightTransform[mset.clseq] = subject
|
||||
}
|
||||
|
||||
// Check if we have discard new with max msgs or bytes.
|
||||
// We need to deny here otherwise we'd need to bump CLFS, and it could succeed on some
|
||||
// peers and not others depending on consumer ack state (if interest policy).
|
||||
@@ -639,7 +980,8 @@ func checkMsgHeadersPreClusteredProposal(
|
||||
}
|
||||
|
||||
// Similarly, check DiscardNew per-subject threshold to not need to bump CLFS.
|
||||
if discardNewPer && maxMsgsPer > 0 {
|
||||
// Allow rollup messages through since they will purge after storing.
|
||||
if discardNewPer && maxMsgsPer > 0 && len(sliceHeader(JSMsgRollup, hdr)) == 0 {
|
||||
// Get the current total for this subject.
|
||||
totalMsgsForSubject := mset.store.SubjectsTotals(subject)[subject]
|
||||
// Add inflight count in this batch and for this stream.
|
||||
@@ -656,3 +998,68 @@ func checkMsgHeadersPreClusteredProposal(
|
||||
|
||||
return hdr, msg, 0, nil, nil
|
||||
}
|
||||
|
||||
// recalculateClusteredSeq initializes or updates mset.clseq, for example after a leader change.
|
||||
// This is reused for normal clustered publishing into a stream, and for atomic and fast batch publishing.
|
||||
// mset.clMu lock must be held.
|
||||
func recalculateClusteredSeq(mset *stream, needStreamLock bool) (lseq uint64) {
|
||||
// Need to unlock and re-acquire the locks in the proper order.
|
||||
mset.clMu.Unlock()
|
||||
// Locking order is stream -> batchMu -> clMu
|
||||
if needStreamLock {
|
||||
mset.mu.RLock()
|
||||
}
|
||||
batch := mset.batchApply
|
||||
var batchCount uint64
|
||||
if batch != nil {
|
||||
batch.mu.Lock()
|
||||
batchCount = batch.count
|
||||
}
|
||||
mset.clMu.Lock()
|
||||
// Re-capture
|
||||
lseq = mset.lseq
|
||||
mset.clseq = lseq + mset.clfs + batchCount
|
||||
// Keep hold of the mset.clMu, but unlock the others.
|
||||
if batch != nil {
|
||||
batch.mu.Unlock()
|
||||
}
|
||||
if needStreamLock {
|
||||
mset.mu.RUnlock()
|
||||
}
|
||||
return lseq
|
||||
}
|
||||
|
||||
// commitSingleMsg commits and proposes a single message to the node.
|
||||
// This is reused both for normal publishing into a stream, and for fast batch publishing.
|
||||
// mset.clMu lock must be held.
|
||||
func commitSingleMsg(
|
||||
diff *batchStagedDiff, mset *stream, subject string, reply string, hdr []byte, msg []byte, name string,
|
||||
jsa *jsAccount, mt *msgTrace, node RaftNode, replicas int, lseq uint64,
|
||||
) error {
|
||||
// Do proposal.
|
||||
esm := encodeStreamMsgAllowCompress(subject, reply, hdr, msg, mset.clseq, time.Now().UnixNano(), false)
|
||||
if err := node.Propose(esm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var mtKey uint64
|
||||
if mt != nil {
|
||||
mtKey = mset.clseq
|
||||
if mset.mt == nil {
|
||||
mset.mt = make(map[uint64]*msgTrace)
|
||||
}
|
||||
mset.mt[mtKey] = mt
|
||||
}
|
||||
|
||||
diff.commit(mset)
|
||||
mset.clseq++
|
||||
mset.trackReplicationTraffic(node, len(esm), replicas)
|
||||
|
||||
// Check to see if we are being overrun.
|
||||
// TODO(dlc) - Make this a limit where we drop messages to protect ourselves, but allow to be configured.
|
||||
if mset.clseq-(lseq+mset.clfs) > streamLagWarnThreshold {
|
||||
lerr := fmt.Errorf("JetStream stream '%s > %s' has high message lag", jsa.acc().Name, name)
|
||||
mset.srv.RateLimitWarnf("%s", lerr.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
+640
-295
File diff suppressed because it is too large
Load Diff
+300
@@ -29,12 +29,30 @@ const (
|
||||
// JSAtomicPublishTooLargeBatchErrF atomic publish batch is too large: {size}
|
||||
JSAtomicPublishTooLargeBatchErrF ErrorIdentifier = 10199
|
||||
|
||||
// JSAtomicPublishTooManyInflight atomic publish too many inflight
|
||||
JSAtomicPublishTooManyInflight ErrorIdentifier = 10210
|
||||
|
||||
// JSAtomicPublishUnsupportedHeaderBatchErr atomic publish unsupported header used: {header}
|
||||
JSAtomicPublishUnsupportedHeaderBatchErr ErrorIdentifier = 10177
|
||||
|
||||
// JSBadRequestErr bad request
|
||||
JSBadRequestErr ErrorIdentifier = 10003
|
||||
|
||||
// JSBatchPublishDisabledErr batch publish is disabled
|
||||
JSBatchPublishDisabledErr ErrorIdentifier = 10205
|
||||
|
||||
// JSBatchPublishInvalidBatchIDErr batch publish ID is invalid
|
||||
JSBatchPublishInvalidBatchIDErr ErrorIdentifier = 10207
|
||||
|
||||
// JSBatchPublishInvalidPatternErr batch publish pattern is invalid
|
||||
JSBatchPublishInvalidPatternErr ErrorIdentifier = 10206
|
||||
|
||||
// JSBatchPublishTooManyInflight batch publish too many inflight
|
||||
JSBatchPublishTooManyInflight ErrorIdentifier = 10211
|
||||
|
||||
// JSBatchPublishUnknownBatchIDErr batch publish ID unknown
|
||||
JSBatchPublishUnknownBatchIDErr ErrorIdentifier = 10208
|
||||
|
||||
// JSClusterIncompleteErr incomplete results
|
||||
JSClusterIncompleteErr ErrorIdentifier = 10004
|
||||
|
||||
@@ -71,6 +89,21 @@ const (
|
||||
// JSClusterUnSupportFeatureErr not currently supported in clustered mode
|
||||
JSClusterUnSupportFeatureErr ErrorIdentifier = 10036
|
||||
|
||||
// JSConsumerAckFCRequiresFCErr flow control ack policy requires flow control
|
||||
JSConsumerAckFCRequiresFCErr ErrorIdentifier = 10219
|
||||
|
||||
// JSConsumerAckFCRequiresMaxAckPendingErr flow control ack policy requires max ack pending
|
||||
JSConsumerAckFCRequiresMaxAckPendingErr ErrorIdentifier = 10220
|
||||
|
||||
// JSConsumerAckFCRequiresNoAckWaitErr flow control ack policy requires unset ack wait
|
||||
JSConsumerAckFCRequiresNoAckWaitErr ErrorIdentifier = 10221
|
||||
|
||||
// JSConsumerAckFCRequiresNoMaxDeliverErr flow control ack policy requires unset max deliver
|
||||
JSConsumerAckFCRequiresNoMaxDeliverErr ErrorIdentifier = 10222
|
||||
|
||||
// JSConsumerAckFCRequiresPushErr flow control ack policy requires a push based consumer
|
||||
JSConsumerAckFCRequiresPushErr ErrorIdentifier = 10218
|
||||
|
||||
// JSConsumerAckPolicyInvalidErr consumer ack policy invalid
|
||||
JSConsumerAckPolicyInvalidErr ErrorIdentifier = 10181
|
||||
|
||||
@@ -167,6 +200,9 @@ const (
|
||||
// JSConsumerInvalidPriorityGroupErr Provided priority group does not exist for this consumer
|
||||
JSConsumerInvalidPriorityGroupErr ErrorIdentifier = 10160
|
||||
|
||||
// JSConsumerInvalidResetErr invalid reset: {err}
|
||||
JSConsumerInvalidResetErr ErrorIdentifier = 10204
|
||||
|
||||
// JSConsumerInvalidSamplingErrF failed to parse consumer sampling configuration: {err}
|
||||
JSConsumerInvalidSamplingErrF ErrorIdentifier = 10095
|
||||
|
||||
@@ -317,21 +353,36 @@ const (
|
||||
// JSMessageSchedulesRollupInvalidErr message schedules invalid rollup
|
||||
JSMessageSchedulesRollupInvalidErr ErrorIdentifier = 10192
|
||||
|
||||
// JSMessageSchedulesSchedulerInvalidErr message schedules invalid scheduler
|
||||
JSMessageSchedulesSchedulerInvalidErr ErrorIdentifier = 10212
|
||||
|
||||
// JSMessageSchedulesSourceInvalidErr message schedules source is invalid
|
||||
JSMessageSchedulesSourceInvalidErr ErrorIdentifier = 10203
|
||||
|
||||
// JSMessageSchedulesTTLInvalidErr message schedules invalid per-message TTL
|
||||
JSMessageSchedulesTTLInvalidErr ErrorIdentifier = 10191
|
||||
|
||||
// JSMessageSchedulesTargetInvalidErr message schedules target is invalid
|
||||
JSMessageSchedulesTargetInvalidErr ErrorIdentifier = 10190
|
||||
|
||||
// JSMessageSchedulesTimeZoneInvalidErr message schedules time zone is invalid
|
||||
JSMessageSchedulesTimeZoneInvalidErr ErrorIdentifier = 10223
|
||||
|
||||
// JSMessageTTLDisabledErr per-message TTL is disabled
|
||||
JSMessageTTLDisabledErr ErrorIdentifier = 10166
|
||||
|
||||
// JSMessageTTLInvalidErr invalid per-message TTL
|
||||
JSMessageTTLInvalidErr ErrorIdentifier = 10165
|
||||
|
||||
// JSMirrorConsumerRequiresAckFCErr stream mirror consumer requires flow control ack policy
|
||||
JSMirrorConsumerRequiresAckFCErr ErrorIdentifier = 10214
|
||||
|
||||
// JSMirrorConsumerSetupFailedErrF generic mirror consumer setup failure string ({err})
|
||||
JSMirrorConsumerSetupFailedErrF ErrorIdentifier = 10029
|
||||
|
||||
// JSMirrorDurableConsumerCfgInvalid stream mirror consumer config is invalid
|
||||
JSMirrorDurableConsumerCfgInvalid ErrorIdentifier = 10213
|
||||
|
||||
// JSMirrorInvalidStreamName mirrored stream name is invalid
|
||||
JSMirrorInvalidStreamName ErrorIdentifier = 10142
|
||||
|
||||
@@ -353,6 +404,9 @@ const (
|
||||
// JSMirrorWithAtomicPublishErr stream mirrors can not also use atomic publishing
|
||||
JSMirrorWithAtomicPublishErr ErrorIdentifier = 10198
|
||||
|
||||
// JSMirrorWithBatchPublishErr stream mirrors can not also use batch publishing
|
||||
JSMirrorWithBatchPublishErr ErrorIdentifier = 10209
|
||||
|
||||
// JSMirrorWithCountersErr stream mirrors can not also calculate counters
|
||||
JSMirrorWithCountersErr ErrorIdentifier = 10173
|
||||
|
||||
@@ -416,12 +470,21 @@ const (
|
||||
// JSSnapshotDeliverSubjectInvalidErr deliver subject not valid
|
||||
JSSnapshotDeliverSubjectInvalidErr ErrorIdentifier = 10015
|
||||
|
||||
// JSSourceConsumerRequiresAckFCErr stream source consumer requires flow control ack policy
|
||||
JSSourceConsumerRequiresAckFCErr ErrorIdentifier = 10217
|
||||
|
||||
// JSSourceConsumerSetupFailedErrF General source consumer setup failure string ({err})
|
||||
JSSourceConsumerSetupFailedErrF ErrorIdentifier = 10045
|
||||
|
||||
// JSSourceDuplicateDetected source stream, filter and transform (plus external if present) must form a unique combination (duplicate source configuration detected)
|
||||
JSSourceDuplicateDetected ErrorIdentifier = 10140
|
||||
|
||||
// JSSourceDurableConsumerCfgInvalid stream source consumer config is invalid
|
||||
JSSourceDurableConsumerCfgInvalid ErrorIdentifier = 10215
|
||||
|
||||
// JSSourceDurableConsumerDuplicateDetected duplicate stream source consumer detected
|
||||
JSSourceDurableConsumerDuplicateDetected ErrorIdentifier = 10216
|
||||
|
||||
// JSSourceInvalidStreamName sourced stream name is invalid
|
||||
JSSourceInvalidStreamName ErrorIdentifier = 10141
|
||||
|
||||
@@ -619,8 +682,14 @@ var (
|
||||
JSAtomicPublishInvalidBatchIDErr: {Code: 400, ErrCode: 10179, Description: "atomic publish batch ID is invalid"},
|
||||
JSAtomicPublishMissingSeqErr: {Code: 400, ErrCode: 10175, Description: "atomic publish sequence is missing"},
|
||||
JSAtomicPublishTooLargeBatchErrF: {Code: 400, ErrCode: 10199, Description: "atomic publish batch is too large: {size}"},
|
||||
JSAtomicPublishTooManyInflight: {Code: 429, ErrCode: 10210, Description: "atomic publish too many inflight"},
|
||||
JSAtomicPublishUnsupportedHeaderBatchErr: {Code: 400, ErrCode: 10177, Description: "atomic publish unsupported header used: {header}"},
|
||||
JSBadRequestErr: {Code: 400, ErrCode: 10003, Description: "bad request"},
|
||||
JSBatchPublishDisabledErr: {Code: 400, ErrCode: 10205, Description: "batch publish is disabled"},
|
||||
JSBatchPublishInvalidBatchIDErr: {Code: 400, ErrCode: 10207, Description: "batch publish ID is invalid"},
|
||||
JSBatchPublishInvalidPatternErr: {Code: 400, ErrCode: 10206, Description: "batch publish pattern is invalid"},
|
||||
JSBatchPublishTooManyInflight: {Code: 429, ErrCode: 10211, Description: "batch publish too many inflight"},
|
||||
JSBatchPublishUnknownBatchIDErr: {Code: 400, ErrCode: 10208, Description: "batch publish ID unknown"},
|
||||
JSClusterIncompleteErr: {Code: 503, ErrCode: 10004, Description: "incomplete results"},
|
||||
JSClusterNoPeersErrF: {Code: 400, ErrCode: 10005, Description: "{err}"},
|
||||
JSClusterNotActiveErr: {Code: 500, ErrCode: 10006, Description: "JetStream not in clustered mode"},
|
||||
@@ -633,6 +702,11 @@ var (
|
||||
JSClusterServerNotMemberErr: {Code: 400, ErrCode: 10044, Description: "server is not a member of the cluster"},
|
||||
JSClusterTagsErr: {Code: 400, ErrCode: 10011, Description: "tags placement not supported for operation"},
|
||||
JSClusterUnSupportFeatureErr: {Code: 503, ErrCode: 10036, Description: "not currently supported in clustered mode"},
|
||||
JSConsumerAckFCRequiresFCErr: {Code: 400, ErrCode: 10219, Description: "flow control ack policy requires flow control"},
|
||||
JSConsumerAckFCRequiresMaxAckPendingErr: {Code: 400, ErrCode: 10220, Description: "flow control ack policy requires max ack pending"},
|
||||
JSConsumerAckFCRequiresNoAckWaitErr: {Code: 400, ErrCode: 10221, Description: "flow control ack policy requires unset ack wait"},
|
||||
JSConsumerAckFCRequiresNoMaxDeliverErr: {Code: 400, ErrCode: 10222, Description: "flow control ack policy requires unset max deliver"},
|
||||
JSConsumerAckFCRequiresPushErr: {Code: 400, ErrCode: 10218, Description: "flow control ack policy requires a push based consumer"},
|
||||
JSConsumerAckPolicyInvalidErr: {Code: 400, ErrCode: 10181, Description: "consumer ack policy invalid"},
|
||||
JSConsumerAckWaitNegativeErr: {Code: 400, ErrCode: 10183, Description: "consumer ack wait needs to be positive"},
|
||||
JSConsumerAlreadyExists: {Code: 400, ErrCode: 10148, Description: "consumer already exists"},
|
||||
@@ -665,6 +739,7 @@ var (
|
||||
JSConsumerInvalidGroupNameErr: {Code: 400, ErrCode: 10162, Description: "Valid priority group name must match A-Z, a-z, 0-9, -_/=)+ and may not exceed 16 characters"},
|
||||
JSConsumerInvalidPolicyErrF: {Code: 400, ErrCode: 10094, Description: "{err}"},
|
||||
JSConsumerInvalidPriorityGroupErr: {Code: 400, ErrCode: 10160, Description: "Provided priority group does not exist for this consumer"},
|
||||
JSConsumerInvalidResetErr: {Code: 400, ErrCode: 10204, Description: "invalid reset: {err}"},
|
||||
JSConsumerInvalidSamplingErrF: {Code: 400, ErrCode: 10095, Description: "failed to parse consumer sampling configuration: {err}"},
|
||||
JSConsumerMaxDeliverBackoffErr: {Code: 400, ErrCode: 10116, Description: "max deliver is required to be > length of backoff values"},
|
||||
JSConsumerMaxPendingAckExcessErrF: {Code: 400, ErrCode: 10121, Description: "consumer max ack pending exceeds system limit of {limit}"},
|
||||
@@ -715,11 +790,16 @@ var (
|
||||
JSMessageSchedulesDisabledErr: {Code: 400, ErrCode: 10188, Description: "message schedules is disabled"},
|
||||
JSMessageSchedulesPatternInvalidErr: {Code: 400, ErrCode: 10189, Description: "message schedules pattern is invalid"},
|
||||
JSMessageSchedulesRollupInvalidErr: {Code: 400, ErrCode: 10192, Description: "message schedules invalid rollup"},
|
||||
JSMessageSchedulesSchedulerInvalidErr: {Code: 400, ErrCode: 10212, Description: "message schedules invalid scheduler"},
|
||||
JSMessageSchedulesSourceInvalidErr: {Code: 400, ErrCode: 10203, Description: "message schedules source is invalid"},
|
||||
JSMessageSchedulesTTLInvalidErr: {Code: 400, ErrCode: 10191, Description: "message schedules invalid per-message TTL"},
|
||||
JSMessageSchedulesTargetInvalidErr: {Code: 400, ErrCode: 10190, Description: "message schedules target is invalid"},
|
||||
JSMessageSchedulesTimeZoneInvalidErr: {Code: 400, ErrCode: 10223, Description: "message schedules time zone is invalid"},
|
||||
JSMessageTTLDisabledErr: {Code: 400, ErrCode: 10166, Description: "per-message TTL is disabled"},
|
||||
JSMessageTTLInvalidErr: {Code: 400, ErrCode: 10165, Description: "invalid per-message TTL"},
|
||||
JSMirrorConsumerRequiresAckFCErr: {Code: 400, ErrCode: 10214, Description: "stream mirror consumer requires flow control ack policy"},
|
||||
JSMirrorConsumerSetupFailedErrF: {Code: 500, ErrCode: 10029, Description: "{err}"},
|
||||
JSMirrorDurableConsumerCfgInvalid: {Code: 400, ErrCode: 10213, Description: "stream mirror consumer config is invalid"},
|
||||
JSMirrorInvalidStreamName: {Code: 400, ErrCode: 10142, Description: "mirrored stream name is invalid"},
|
||||
JSMirrorInvalidSubjectFilter: {Code: 400, ErrCode: 10151, Description: "mirror transform source: {err}"},
|
||||
JSMirrorInvalidTransformDestination: {Code: 400, ErrCode: 10154, Description: "mirror transform: {err}"},
|
||||
@@ -727,6 +807,7 @@ var (
|
||||
JSMirrorMultipleFiltersNotAllowed: {Code: 400, ErrCode: 10150, Description: "mirror with multiple subject transforms cannot also have a single subject filter"},
|
||||
JSMirrorOverlappingSubjectFilters: {Code: 400, ErrCode: 10152, Description: "mirror subject filters can not overlap"},
|
||||
JSMirrorWithAtomicPublishErr: {Code: 400, ErrCode: 10198, Description: "stream mirrors can not also use atomic publishing"},
|
||||
JSMirrorWithBatchPublishErr: {Code: 400, ErrCode: 10209, Description: "stream mirrors can not also use batch publishing"},
|
||||
JSMirrorWithCountersErr: {Code: 400, ErrCode: 10173, Description: "stream mirrors can not also calculate counters"},
|
||||
JSMirrorWithFirstSeqErr: {Code: 400, ErrCode: 10143, Description: "stream mirrors can not have first sequence configured"},
|
||||
JSMirrorWithMsgSchedulesErr: {Code: 400, ErrCode: 10186, Description: "stream mirrors can not also schedule messages"},
|
||||
@@ -748,8 +829,11 @@ var (
|
||||
JSRestoreSubscribeFailedErrF: {Code: 500, ErrCode: 10042, Description: "JetStream unable to subscribe to restore snapshot {subject}: {err}"},
|
||||
JSSequenceNotFoundErrF: {Code: 400, ErrCode: 10043, Description: "sequence {seq} not found"},
|
||||
JSSnapshotDeliverSubjectInvalidErr: {Code: 400, ErrCode: 10015, Description: "deliver subject not valid"},
|
||||
JSSourceConsumerRequiresAckFCErr: {Code: 400, ErrCode: 10217, Description: "stream source consumer requires flow control ack policy"},
|
||||
JSSourceConsumerSetupFailedErrF: {Code: 500, ErrCode: 10045, Description: "{err}"},
|
||||
JSSourceDuplicateDetected: {Code: 400, ErrCode: 10140, Description: "duplicate source configuration detected"},
|
||||
JSSourceDurableConsumerCfgInvalid: {Code: 400, ErrCode: 10215, Description: "stream source consumer config is invalid"},
|
||||
JSSourceDurableConsumerDuplicateDetected: {Code: 400, ErrCode: 10216, Description: "duplicate stream source consumer detected"},
|
||||
JSSourceInvalidStreamName: {Code: 400, ErrCode: 10141, Description: "sourced stream name is invalid"},
|
||||
JSSourceInvalidSubjectFilter: {Code: 400, ErrCode: 10145, Description: "source transform source: {err}"},
|
||||
JSSourceInvalidTransformDestination: {Code: 400, ErrCode: 10146, Description: "source transform: {err}"},
|
||||
@@ -923,6 +1007,16 @@ func NewJSAtomicPublishTooLargeBatchError(size interface{}, opts ...ErrorOption)
|
||||
}
|
||||
}
|
||||
|
||||
// NewJSAtomicPublishTooManyInflightError creates a new JSAtomicPublishTooManyInflight error: "atomic publish too many inflight"
|
||||
func NewJSAtomicPublishTooManyInflightError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSAtomicPublishTooManyInflight]
|
||||
}
|
||||
|
||||
// NewJSAtomicPublishUnsupportedHeaderBatchError creates a new JSAtomicPublishUnsupportedHeaderBatchErr error: "atomic publish unsupported header used: {header}"
|
||||
func NewJSAtomicPublishUnsupportedHeaderBatchError(header interface{}, opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -949,6 +1043,56 @@ func NewJSBadRequestError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSBadRequestErr]
|
||||
}
|
||||
|
||||
// NewJSBatchPublishDisabledError creates a new JSBatchPublishDisabledErr error: "batch publish is disabled"
|
||||
func NewJSBatchPublishDisabledError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSBatchPublishDisabledErr]
|
||||
}
|
||||
|
||||
// NewJSBatchPublishInvalidBatchIDError creates a new JSBatchPublishInvalidBatchIDErr error: "batch publish ID is invalid"
|
||||
func NewJSBatchPublishInvalidBatchIDError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSBatchPublishInvalidBatchIDErr]
|
||||
}
|
||||
|
||||
// NewJSBatchPublishInvalidPatternError creates a new JSBatchPublishInvalidPatternErr error: "batch publish pattern is invalid"
|
||||
func NewJSBatchPublishInvalidPatternError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSBatchPublishInvalidPatternErr]
|
||||
}
|
||||
|
||||
// NewJSBatchPublishTooManyInflightError creates a new JSBatchPublishTooManyInflight error: "batch publish too many inflight"
|
||||
func NewJSBatchPublishTooManyInflightError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSBatchPublishTooManyInflight]
|
||||
}
|
||||
|
||||
// NewJSBatchPublishUnknownBatchIDError creates a new JSBatchPublishUnknownBatchIDErr error: "batch publish ID unknown"
|
||||
func NewJSBatchPublishUnknownBatchIDError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSBatchPublishUnknownBatchIDErr]
|
||||
}
|
||||
|
||||
// NewJSClusterIncompleteError creates a new JSClusterIncompleteErr error: "incomplete results"
|
||||
func NewJSClusterIncompleteError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -1075,6 +1219,56 @@ func NewJSClusterUnSupportFeatureError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSClusterUnSupportFeatureErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerAckFCRequiresFCError creates a new JSConsumerAckFCRequiresFCErr error: "flow control ack policy requires flow control"
|
||||
func NewJSConsumerAckFCRequiresFCError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSConsumerAckFCRequiresFCErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerAckFCRequiresMaxAckPendingError creates a new JSConsumerAckFCRequiresMaxAckPendingErr error: "flow control ack policy requires max ack pending"
|
||||
func NewJSConsumerAckFCRequiresMaxAckPendingError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSConsumerAckFCRequiresMaxAckPendingErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerAckFCRequiresNoAckWaitError creates a new JSConsumerAckFCRequiresNoAckWaitErr error: "flow control ack policy requires unset ack wait"
|
||||
func NewJSConsumerAckFCRequiresNoAckWaitError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSConsumerAckFCRequiresNoAckWaitErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerAckFCRequiresNoMaxDeliverError creates a new JSConsumerAckFCRequiresNoMaxDeliverErr error: "flow control ack policy requires unset max deliver"
|
||||
func NewJSConsumerAckFCRequiresNoMaxDeliverError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSConsumerAckFCRequiresNoMaxDeliverErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerAckFCRequiresPushError creates a new JSConsumerAckFCRequiresPushErr error: "flow control ack policy requires a push based consumer"
|
||||
func NewJSConsumerAckFCRequiresPushError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSConsumerAckFCRequiresPushErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerAckPolicyInvalidError creates a new JSConsumerAckPolicyInvalidErr error: "consumer ack policy invalid"
|
||||
func NewJSConsumerAckPolicyInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -1419,6 +1613,22 @@ func NewJSConsumerInvalidPriorityGroupError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSConsumerInvalidPriorityGroupErr]
|
||||
}
|
||||
|
||||
// NewJSConsumerInvalidResetError creates a new JSConsumerInvalidResetErr error: "invalid reset: {err}"
|
||||
func NewJSConsumerInvalidResetError(err error, opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
e := ApiErrors[JSConsumerInvalidResetErr]
|
||||
args := e.toReplacerArgs([]interface{}{"{err}", err})
|
||||
return &ApiError{
|
||||
Code: e.Code,
|
||||
ErrCode: e.ErrCode,
|
||||
Description: strings.NewReplacer(args...).Replace(e.Description),
|
||||
}
|
||||
}
|
||||
|
||||
// NewJSConsumerInvalidSamplingError creates a new JSConsumerInvalidSamplingErrF error: "failed to parse consumer sampling configuration: {err}"
|
||||
func NewJSConsumerInvalidSamplingError(err error, opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -1967,6 +2177,26 @@ func NewJSMessageSchedulesRollupInvalidError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSMessageSchedulesRollupInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSMessageSchedulesSchedulerInvalidError creates a new JSMessageSchedulesSchedulerInvalidErr error: "message schedules invalid scheduler"
|
||||
func NewJSMessageSchedulesSchedulerInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSMessageSchedulesSchedulerInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSMessageSchedulesSourceInvalidError creates a new JSMessageSchedulesSourceInvalidErr error: "message schedules source is invalid"
|
||||
func NewJSMessageSchedulesSourceInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSMessageSchedulesSourceInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSMessageSchedulesTTLInvalidError creates a new JSMessageSchedulesTTLInvalidErr error: "message schedules invalid per-message TTL"
|
||||
func NewJSMessageSchedulesTTLInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -1987,6 +2217,16 @@ func NewJSMessageSchedulesTargetInvalidError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSMessageSchedulesTargetInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSMessageSchedulesTimeZoneInvalidError creates a new JSMessageSchedulesTimeZoneInvalidErr error: "message schedules time zone is invalid"
|
||||
func NewJSMessageSchedulesTimeZoneInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSMessageSchedulesTimeZoneInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSMessageTTLDisabledError creates a new JSMessageTTLDisabledErr error: "per-message TTL is disabled"
|
||||
func NewJSMessageTTLDisabledError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -2007,6 +2247,16 @@ func NewJSMessageTTLInvalidError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSMessageTTLInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSMirrorConsumerRequiresAckFCError creates a new JSMirrorConsumerRequiresAckFCErr error: "stream mirror consumer requires flow control ack policy"
|
||||
func NewJSMirrorConsumerRequiresAckFCError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSMirrorConsumerRequiresAckFCErr]
|
||||
}
|
||||
|
||||
// NewJSMirrorConsumerSetupFailedError creates a new JSMirrorConsumerSetupFailedErrF error: "{err}"
|
||||
func NewJSMirrorConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -2023,6 +2273,16 @@ func NewJSMirrorConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiErr
|
||||
}
|
||||
}
|
||||
|
||||
// NewJSMirrorDurableConsumerCfgInvalidError creates a new JSMirrorDurableConsumerCfgInvalid error: "stream mirror consumer config is invalid"
|
||||
func NewJSMirrorDurableConsumerCfgInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSMirrorDurableConsumerCfgInvalid]
|
||||
}
|
||||
|
||||
// NewJSMirrorInvalidStreamNameError creates a new JSMirrorInvalidStreamName error: "mirrored stream name is invalid"
|
||||
func NewJSMirrorInvalidStreamNameError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -2105,6 +2365,16 @@ func NewJSMirrorWithAtomicPublishError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSMirrorWithAtomicPublishErr]
|
||||
}
|
||||
|
||||
// NewJSMirrorWithBatchPublishError creates a new JSMirrorWithBatchPublishErr error: "stream mirrors can not also use batch publishing"
|
||||
func NewJSMirrorWithBatchPublishError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSMirrorWithBatchPublishErr]
|
||||
}
|
||||
|
||||
// NewJSMirrorWithCountersError creates a new JSMirrorWithCountersErr error: "stream mirrors can not also calculate counters"
|
||||
func NewJSMirrorWithCountersError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -2339,6 +2609,16 @@ func NewJSSnapshotDeliverSubjectInvalidError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSSnapshotDeliverSubjectInvalidErr]
|
||||
}
|
||||
|
||||
// NewJSSourceConsumerRequiresAckFCError creates a new JSSourceConsumerRequiresAckFCErr error: "stream source consumer requires flow control ack policy"
|
||||
func NewJSSourceConsumerRequiresAckFCError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSSourceConsumerRequiresAckFCErr]
|
||||
}
|
||||
|
||||
// NewJSSourceConsumerSetupFailedError creates a new JSSourceConsumerSetupFailedErrF error: "{err}"
|
||||
func NewJSSourceConsumerSetupFailedError(err error, opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
@@ -2365,6 +2645,26 @@ func NewJSSourceDuplicateDetectedError(opts ...ErrorOption) *ApiError {
|
||||
return ApiErrors[JSSourceDuplicateDetected]
|
||||
}
|
||||
|
||||
// NewJSSourceDurableConsumerCfgInvalidError creates a new JSSourceDurableConsumerCfgInvalid error: "stream source consumer config is invalid"
|
||||
func NewJSSourceDurableConsumerCfgInvalidError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSSourceDurableConsumerCfgInvalid]
|
||||
}
|
||||
|
||||
// NewJSSourceDurableConsumerDuplicateDetectedError creates a new JSSourceDurableConsumerDuplicateDetected error: "duplicate stream source consumer detected"
|
||||
func NewJSSourceDurableConsumerDuplicateDetectedError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
if ae, ok := eopts.err.(*ApiError); ok {
|
||||
return ae
|
||||
}
|
||||
|
||||
return ApiErrors[JSSourceDurableConsumerDuplicateDetected]
|
||||
}
|
||||
|
||||
// NewJSSourceInvalidStreamNameError creates a new JSSourceInvalidStreamName error: "sourced stream name is invalid"
|
||||
func NewJSSourceInvalidStreamNameError(opts ...ErrorOption) *ApiError {
|
||||
eopts := parseOpts(opts)
|
||||
|
||||
+7
-7
@@ -71,10 +71,9 @@ const (
|
||||
// JSStreamActionAdvisory indicates that a stream was created, edited or deleted
|
||||
type JSStreamActionAdvisory struct {
|
||||
TypedEvent
|
||||
Stream string `json:"stream"`
|
||||
Action ActionAdvisoryType `json:"action"`
|
||||
Template string `json:"template,omitempty"` // Deprecated: stream templates are deprecated and will be removed in a future version.
|
||||
Domain string `json:"domain,omitempty"`
|
||||
Stream string `json:"stream"`
|
||||
Action ActionAdvisoryType `json:"action"`
|
||||
Domain string `json:"domain,omitempty"`
|
||||
}
|
||||
|
||||
const JSStreamActionAdvisoryType = "io.nats.jetstream.advisory.v1.stream_action"
|
||||
@@ -269,9 +268,10 @@ type JSStreamBatchAbandonedAdvisory struct {
|
||||
type BatchAbandonReason string
|
||||
|
||||
var (
|
||||
BatchTimeout BatchAbandonReason = "timeout"
|
||||
BatchLarge BatchAbandonReason = "large"
|
||||
BatchIncomplete BatchAbandonReason = "incomplete"
|
||||
BatchTimeout BatchAbandonReason = "timeout"
|
||||
BatchLarge BatchAbandonReason = "large"
|
||||
BatchIncomplete BatchAbandonReason = "incomplete"
|
||||
BatchRequirementsNotMet BatchAbandonReason = "unsupported"
|
||||
)
|
||||
|
||||
// JSConsumerLeaderElectedAdvisoryType is sent when the system elects a leader for a consumer.
|
||||
|
||||
+11
-1
@@ -17,7 +17,7 @@ import "strconv"
|
||||
|
||||
const (
|
||||
// JSApiLevel is the maximum supported JetStream API level for this server.
|
||||
JSApiLevel int = 3
|
||||
JSApiLevel int = 4
|
||||
|
||||
JSRequiredLevelMetadataKey = "_nats.req.level"
|
||||
JSServerVersionMetadataKey = "_nats.ver"
|
||||
@@ -82,6 +82,11 @@ func setStaticStreamMetadata(cfg *StreamConfig) {
|
||||
requires(2)
|
||||
}
|
||||
|
||||
// Fast batch publishing was added in v2.14 and requires API level 4.
|
||||
if cfg.AllowBatchPublish {
|
||||
requires(4)
|
||||
}
|
||||
|
||||
cfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel)
|
||||
}
|
||||
|
||||
@@ -158,6 +163,11 @@ func setStaticConsumerMetadata(cfg *ConsumerConfig) {
|
||||
requires(1)
|
||||
}
|
||||
|
||||
// Added in 2.14
|
||||
if cfg.AckPolicy == AckFlowControl {
|
||||
requires(4)
|
||||
}
|
||||
|
||||
cfg.Metadata[JSRequiredLevelMetadataKey] = strconv.Itoa(requiredApiLevel)
|
||||
}
|
||||
|
||||
|
||||
+44
-14
@@ -202,12 +202,15 @@ func validateSrc(claims *jwt.UserClaims, host string) bool {
|
||||
}
|
||||
|
||||
func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {
|
||||
return validateTimesAt(claims, time.Now())
|
||||
}
|
||||
|
||||
func validateTimesAt(claims *jwt.UserClaims, now time.Time) (bool, time.Duration) {
|
||||
if claims == nil {
|
||||
return false, time.Duration(0)
|
||||
} else if len(claims.Times) == 0 {
|
||||
return true, time.Duration(0)
|
||||
}
|
||||
now := time.Now()
|
||||
loc := time.Local
|
||||
if claims.Locale != "" {
|
||||
var err error
|
||||
@@ -216,10 +219,11 @@ func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {
|
||||
}
|
||||
now = now.In(loc)
|
||||
}
|
||||
|
||||
var ok bool
|
||||
var validFor time.Duration
|
||||
|
||||
for _, timeRange := range claims.Times {
|
||||
y, m, d := now.Date()
|
||||
m = m - 1
|
||||
d = d - 1
|
||||
start, err := time.ParseInLocation("15:04:05", timeRange.Start, loc)
|
||||
if err != nil {
|
||||
return false, time.Duration(0) // parsing not expected to fail at this point
|
||||
@@ -228,17 +232,43 @@ func validateTimes(claims *jwt.UserClaims) (bool, time.Duration) {
|
||||
if err != nil {
|
||||
return false, time.Duration(0) // parsing not expected to fail at this point
|
||||
}
|
||||
if start.After(end) {
|
||||
start = start.AddDate(y, int(m), d)
|
||||
d++ // the intent is to be the next day
|
||||
} else {
|
||||
start = start.AddDate(y, int(m), d)
|
||||
|
||||
y, m, d := now.Date()
|
||||
start = time.Date(y, m, d, start.Hour(), start.Minute(), start.Second(), 0, loc)
|
||||
end = time.Date(y, m, d, end.Hour(), end.Minute(), end.Second(), 0, loc)
|
||||
|
||||
inRange, expires := validateTimeRangeAt(start, end, now)
|
||||
if inRange && (!ok || expires > validFor) {
|
||||
ok = true
|
||||
validFor = expires
|
||||
}
|
||||
if start.Before(now) {
|
||||
end = end.AddDate(y, int(m), d)
|
||||
if end.After(now) {
|
||||
return true, end.Sub(now)
|
||||
}
|
||||
}
|
||||
return ok, validFor
|
||||
}
|
||||
|
||||
// Returns true if now is within `start` and `end`, and
|
||||
// how much time is left until `end`.
|
||||
// False if `now` is not within range.
|
||||
func validateTimeRangeAt(start, end, now time.Time) (bool, time.Duration) {
|
||||
// Now falls within range.
|
||||
// For example 11:00-22:00 at 13:00
|
||||
if start.Before(now) && end.After(now) {
|
||||
return true, end.Sub(now)
|
||||
}
|
||||
|
||||
// Range crosses midnight.
|
||||
if start.After(end) {
|
||||
// Now is after midnight.
|
||||
// For example 22:00-06:00 at 05:00.
|
||||
if end.After(now) {
|
||||
return true, end.Sub(now)
|
||||
}
|
||||
|
||||
// Now is before midnight.
|
||||
// For example 22:00-06:00 at 23:30.
|
||||
end = end.AddDate(0, 0, 1)
|
||||
if start.Before(now) && end.After(now) {
|
||||
return true, end.Sub(now)
|
||||
}
|
||||
}
|
||||
return false, time.Duration(0)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user