chore: bump github.com/nats-io/nats-server/v2

Fixes various CVEs (see https://github.com/nats-io/nats-server/releases/tag/v2.12.6
for details)
This commit is contained in:
Ralf Haferkamp
2026-03-30 12:22:23 +02:00
committed by Ralf Haferkamp
parent 8cf768bf80
commit 9c4ac18e47
241 changed files with 19192 additions and 64831 deletions
+17 -17
View File
@@ -1,6 +1,6 @@
module github.com/opencloud-eu/opencloud
go 1.24.6
go 1.25.0
require (
dario.cat/mergo v1.0.2
@@ -54,8 +54,8 @@ require (
github.com/mitchellh/mapstructure v1.5.0
github.com/mna/pigeon v1.3.0
github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826
github.com/nats-io/nats-server/v2 v2.12.2
github.com/nats-io/nats.go v1.47.0
github.com/nats-io/nats-server/v2 v2.12.6
github.com/nats-io/nats.go v1.49.0
github.com/oklog/run v1.2.0
github.com/olekukonko/tablewriter v1.1.1
github.com/onsi/ginkgo v1.16.5
@@ -101,14 +101,14 @@ require (
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0
go.opentelemetry.io/otel/sdk v1.38.0
go.opentelemetry.io/otel/trace v1.38.0
golang.org/x/crypto v0.45.0
golang.org/x/crypto v0.49.0
golang.org/x/exp v0.0.0-20250210185358-939b2ce775ac
golang.org/x/image v0.33.0
golang.org/x/net v0.47.0
golang.org/x/net v0.51.0
golang.org/x/oauth2 v0.33.0
golang.org/x/sync v0.18.0
golang.org/x/term v0.37.0
golang.org/x/text v0.31.0
golang.org/x/sync v0.20.0
golang.org/x/term v0.41.0
golang.org/x/text v0.35.0
google.golang.org/genproto/googleapis/api v0.0.0-20251022142026-3a174f9686a8
google.golang.org/grpc v1.77.0
google.golang.org/protobuf v1.36.10
@@ -134,7 +134,7 @@ require (
github.com/ajg/form v1.5.1 // indirect
github.com/alexedwards/argon2id v1.0.0 // indirect
github.com/amoghe/go-crypt v0.0.0-20220222110647-20eada5f5964 // indirect
github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op // indirect
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op // indirect
github.com/armon/go-radix v1.0.0 // indirect
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@@ -240,7 +240,7 @@ require (
github.com/golang/snappy v0.0.4 // indirect
github.com/gomodule/redigo v1.9.3 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/google/go-tpm v0.9.6 // indirect
github.com/google/go-tpm v0.9.8 // indirect
github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect
github.com/google/renameio/v2 v2.0.1 // indirect
github.com/gookit/goutil v0.7.1 // indirect
@@ -259,7 +259,7 @@ require (
github.com/json-iterator/go v1.1.12 // indirect
github.com/juliangruber/go-intersect v1.1.0 // indirect
github.com/kevinburke/ssh_config v1.2.0 // indirect
github.com/klauspost/compress v1.18.1 // indirect
github.com/klauspost/compress v1.18.4 // indirect
github.com/klauspost/cpuid/v2 v2.2.11 // indirect
github.com/klauspost/crc32 v1.3.0 // indirect
github.com/kovidgoyal/go-parallel v1.1.1 // indirect
@@ -305,8 +305,8 @@ require (
github.com/morikuni/aec v1.0.0 // indirect
github.com/mschoch/smat v0.2.0 // indirect
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/nats-io/jwt/v2 v2.8.0 // indirect
github.com/nats-io/nkeys v0.4.11 // indirect
github.com/nats-io/jwt/v2 v2.8.1 // indirect
github.com/nats-io/nkeys v0.4.15 // indirect
github.com/nats-io/nuid v1.0.1 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/olekukonko/cat v0.0.0-20250911104152-50322a0618f6 // indirect
@@ -384,10 +384,10 @@ require (
go.uber.org/zap v1.27.0 // indirect
go.yaml.in/yaml/v2 v2.4.3 // indirect
go.yaml.in/yaml/v3 v3.0.4 // indirect
golang.org/x/mod v0.29.0 // indirect
golang.org/x/sys v0.38.0 // indirect
golang.org/x/time v0.14.0 // indirect
golang.org/x/tools v0.38.0 // indirect
golang.org/x/mod v0.33.0 // indirect
golang.org/x/sys v0.42.0 // indirect
golang.org/x/time v0.15.0 // indirect
golang.org/x/tools v0.42.0 // indirect
google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20251022142026-3a174f9686a8 // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
+32 -32
View File
@@ -119,8 +119,8 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNg
github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8=
github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4=
github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0=
github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op h1:kpBdlEPbRvff0mDD1gk7o9BhI16b9p5yYAXRlidpqJE=
github.com/antithesishq/antithesis-sdk-go v0.6.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0 h1:jfIu9sQUG6Ig+0+Ap1h4unLjW6YQJpKZVmUzxsD4E/Q=
github.com/arbovm/levenshtein v0.0.0-20160628152529-48b4e1c0c4d0/go.mod h1:t2tdKJDJF9BV14lnkjHmOQgcvEKgtqs5a1N3LNdJhGE=
@@ -572,8 +572,8 @@ github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/go-tika v0.3.1 h1:l+jr10hDhZjcgxFRfcQChRLo1bPXQeLFluMyvDhXTTA=
github.com/google/go-tika v0.3.1/go.mod h1:DJh5N8qxXIl85QkqmXknd+PeeRkUOTbvwyYf7ieDz6c=
github.com/google/go-tpm v0.9.6 h1:Ku42PT4LmjDu1H5C5ISWLlpI1mj+Zq7sPGKoRw2XROA=
github.com/google/go-tpm v0.9.6/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/google/go-tpm v0.9.8 h1:slArAR9Ft+1ybZu0lBwpSmpwhRXaa85hWtMinMyRAWo=
github.com/google/go-tpm v0.9.8/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@@ -725,8 +725,8 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
github.com/klauspost/compress v1.18.4 h1:RPhnKRAQ4Fh8zU2FY/6ZFDwTVTxgJ/EMydqSTzE9a2c=
github.com/klauspost/compress v1.18.4/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU=
github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
@@ -908,14 +908,14 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8m
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8=
github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
github.com/nats-io/nats-server/v2 v2.12.2 h1:4TEQd0Y4zvcW0IsVxjlXnRso1hBkQl3TS0BI+SxgPhE=
github.com/nats-io/nats-server/v2 v2.12.2/go.mod h1:j1AAttYeu7WnvD8HLJ+WWKNMSyxsqmZ160pNtCQRMyE=
github.com/nats-io/nats.go v1.47.0 h1:YQdADw6J/UfGUd2Oy6tn4Hq6YHxCaJrVKayxxFqYrgM=
github.com/nats-io/nats.go v1.47.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/jwt/v2 v2.8.1 h1:V0xpGuD/N8Mi+fQNDynXohVvp7ZztevW5io8CUWlPmU=
github.com/nats-io/jwt/v2 v2.8.1/go.mod h1:nWnOEEiVMiKHQpnAy4eXlizVEtSfzacZ1Q43LIRavZg=
github.com/nats-io/nats-server/v2 v2.12.6 h1:Egbx9Vl7Ch8wTtpXPGqbehkZ+IncKqShUxvrt1+Enc8=
github.com/nats-io/nats-server/v2 v2.12.6/go.mod h1:4HPlrvtmSO3yd7KcElDNMx9kv5EBJBnJJzQPptXlheo=
github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4=
github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms=
@@ -1359,8 +1359,8 @@ golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
golang.org/x/crypto v0.49.0 h1:+Ng2ULVvLHnJ/ZFEq4KdcDd/cfjrrjjNSXNzxg0Y4U4=
golang.org/x/crypto v0.49.0/go.mod h1:ErX4dUh2UM+CFYiXZRTcMpEcN8b/1gxEuv3nODoYtCA=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1402,8 +1402,8 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -1457,8 +1457,8 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo=
golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
@@ -1485,8 +1485,8 @@ golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y=
golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4=
golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
@@ -1570,8 +1570,8 @@ golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
@@ -1583,8 +1583,8 @@ golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U=
golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk=
golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU=
golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254=
golang.org/x/term v0.41.0 h1:QCgPso/Q3RTJx2Th4bDLqML4W6iJiaXFq2/ftQF13YU=
golang.org/x/term v0.41.0/go.mod h1:3pfBgksrReYfZ5lvYM0kSO0LIkAl4Yl2bXOkKP7Ec2A=
golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -1599,16 +1599,16 @@ golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI=
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
golang.org/x/text v0.35.0 h1:JOVx6vVDFokkpaq1AEptVzLTpDe9KGpj5tR4/X+ybL8=
golang.org/x/text v0.35.0/go.mod h1:khi/HExzZJ2pGnjenulevKNX1W67CUy0AsXcNubPGCA=
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U=
golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno=
golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1663,8 +1663,8 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k=
golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0=
golang.org/x/tools/godoc v0.1.0-deprecated h1:o+aZ1BOj6Hsx/GBdJO/s815sqftjSnrZZwyYTHODvtk=
golang.org/x/tools/godoc v0.1.0-deprecated/go.mod h1:qM63CriJ961IHWmnWa9CjZnBndniPt4a3CK0PVB9bIg=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+65 -2
View File
@@ -17,13 +17,18 @@
// [test properties]: https://antithesis.com/docs/using_antithesis/properties/
// [workload]: https://antithesis.com/docs/getting_started/first_test/
// [antithesis-go-generator]: https://antithesis.com/docs/using_antithesis/sdk/go/instrumentor/
// [triage report]: https://antithesis.com/docs/reports/triage/
// [triage report]: https://antithesis.com/docs/reports/
// [here]: https://antithesis.com/docs/using_antithesis/sdk/fallback/
// [Sometimes assertions]: https://antithesis.com/docs/best_practices/sometimes_assertions/
//
// [details]: https://antithesis.com/docs/reports/triage/#details
// [details]: https://antithesis.com/docs/reports/
package assert
import (
"encoding/json"
"fmt"
)
type assertInfo struct {
Location *locationInfo `json:"location"`
Details map[string]any `json:"details"`
@@ -36,6 +41,64 @@ type assertInfo struct {
Condition bool `json:"condition"`
}
// Create a custom json marshaler for assertInfo so that we can force Errors to be marshaled with their error details.
// Without this, custom errors are marshaled as an empty object because the default json marshaler doesn't include the error
// (because it's a method - not an exported struct field).
func (f assertInfo) MarshalJSON() ([]byte, error) {
type alias assertInfo // prevent infinite recursion
a := alias(f)
if a.Details != nil {
a.Details = normalizeMap(a.Details)
}
return json.Marshal(a)
}
type jsonError struct {
innerError error
}
func (e jsonError) MarshalJSON() ([]byte, error) {
// Marshal this as the debug output string instead of e.Error(). These should be equivalent, but Sprintf correctly
// handles nil values for us (which otherwise are annoying to defend against due to this - https://go.dev/doc/faq#nil_error)
return json.Marshal(fmt.Sprintf("%+v", e.innerError))
}
// Recursively replace any `error` with jsonError while doing a deep copy.
// Most of the logic is in the normalize method below. This method exists to localize the type assertions
// and provide a function that takes in/out a map instead of any.
func normalizeMap(v map[string]any) map[string]any {
return normalize(v).(map[string]any)
}
func normalize(input any) any {
// This switch will miss some cases (pointers, structs, non-any types), but should catch a very large proportion of real error interfaces
// in real details objects. We can augment this if we find other cases common enough to support.
switch inputTyped := input.(type) {
case error:
// Check if the underlying error implements json.Marshaler, so that if the error
// already knows who to marshal itself, we don't override that.
if _, ok := inputTyped.(json.Marshaler); ok {
return inputTyped
} else {
return jsonError{inputTyped}
}
case map[string]any:
out := make(map[string]any, len(inputTyped))
for k, v := range inputTyped {
out[k] = normalize(v)
}
return out
case []any:
out := make([]any, len(inputTyped))
for i := range inputTyped {
out[i] = normalize(inputTyped[i])
}
return out
default:
return input
}
}
type wrappedAssertInfo struct {
A *assertInfo `json:"antithesis_assert"`
}
+4 -146
View File
@@ -4,67 +4,11 @@ package internal
import (
"encoding/json"
"fmt"
"log"
"math/rand"
"os"
"unsafe"
)
// --------------------------------------------------------------------------------
// To build and run an executable with this package
//
// CC=clang CGO_ENABLED=1 go run ./main.go
// --------------------------------------------------------------------------------
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
//
// The commented lines below, and the `import "C"` line which must directly follow
// the commented lines are used by CGO. They are load-bearing, and should not be
// changed without first understanding how CGO uses them.
//
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
// #cgo LDFLAGS: -ldl
//
// #include <dlfcn.h>
// #include <stdbool.h>
// #include <stdint.h>
// #include <stdlib.h>
//
// typedef void (*go_fuzz_json_data_fn)(const char *data, size_t size);
// void
// go_fuzz_json_data(void *f, const char *data, size_t size) {
// ((go_fuzz_json_data_fn)f)(data, size);
// }
//
// typedef void (*go_fuzz_flush_fn)(void);
// void
// go_fuzz_flush(void *f) {
// ((go_fuzz_flush_fn)f)();
// }
//
// typedef uint64_t (*go_fuzz_get_random_fn)(void);
// uint64_t
// go_fuzz_get_random(void *f) {
// return ((go_fuzz_get_random_fn)f)();
// }
//
// typedef bool (*go_notify_coverage_fn)(size_t);
// int
// go_notify_coverage(void *f, size_t edges) {
// bool b = ((go_notify_coverage_fn)f)(edges);
// return b ? 1 : 0;
// }
//
// typedef uint64_t (*go_init_coverage_fn)(size_t num_edges, const char *symbols);
// uint64_t
// go_init_coverage(void *f, size_t num_edges, const char *symbols) {
// return ((go_init_coverage_fn)f)(num_edges, symbols);
// }
//
import "C"
func Json_data(v any) error {
if data, err := json.Marshal(v); err != nil {
return err
@@ -95,45 +39,10 @@ type libHandler interface {
const (
errorLogLinePrefix = "[* antithesis-sdk-go *]"
defaultNativeLibraryPath = "/usr/lib/libvoidstar.so"
)
var handler libHandler
type voidstarHandler struct {
fuzzJsonData unsafe.Pointer
fuzzFlush unsafe.Pointer
fuzzGetRandom unsafe.Pointer
initCoverage unsafe.Pointer
notifyCoverage unsafe.Pointer
}
func (h *voidstarHandler) output(message string) {
msg_len := len(message)
if msg_len == 0 {
return
}
cstrMessage := C.CString(message)
defer C.free(unsafe.Pointer(cstrMessage))
C.go_fuzz_json_data(h.fuzzJsonData, cstrMessage, C.ulong(msg_len))
C.go_fuzz_flush(h.fuzzFlush)
}
func (h *voidstarHandler) random() uint64 {
return uint64(C.go_fuzz_get_random(h.fuzzGetRandom))
}
func (h *voidstarHandler) init_coverage(num_edge uint64, symbols string) uint64 {
cstrSymbols := C.CString(symbols)
defer C.free(unsafe.Pointer(cstrSymbols))
return uint64(C.go_init_coverage(h.initCoverage, C.ulong(num_edge), cstrSymbols))
}
func (h *voidstarHandler) notify(edge uint64) bool {
ival := int(C.go_notify_coverage(h.notifyCoverage, C.ulong(edge)))
return ival == 1
}
type localHandler struct {
outputFile *os.File // can be nil
}
@@ -160,63 +69,12 @@ func (h *localHandler) init_coverage(num_edges uint64, symbols string) uint64 {
return 0
}
// If we have a file at `defaultNativeLibraryPath`, we load the shared library
// (and panic on any error encountered during load).
// Otherwise fallback to the local handler.
func init() {
if _, err := os.Stat(defaultNativeLibraryPath); err == nil {
if handler, err = openSharedLib(defaultNativeLibraryPath); err != nil {
panic(err)
}
return
handler = init_in_antithesis()
if handler == nil {
// Otherwise fallback to the local handler.
handler = openLocalHandler()
}
handler = openLocalHandler()
}
// Attempt to load libvoidstar and some symbols from `path`
func openSharedLib(path string) (*voidstarHandler, error) {
cstrPath := C.CString(path)
defer C.free(unsafe.Pointer(cstrPath))
dlError := func(message string) error {
return fmt.Errorf("%s: (%s)", message, C.GoString(C.dlerror()))
}
sharedLib := C.dlopen(cstrPath, C.int(C.RTLD_NOW))
if sharedLib == nil {
return nil, dlError("Can not load the Antithesis native library")
}
loadFunc := func(name string) (symbol unsafe.Pointer, err error) {
cstrName := C.CString(name)
defer C.free(unsafe.Pointer(cstrName))
if symbol = C.dlsym(sharedLib, cstrName); symbol == nil {
err = dlError(fmt.Sprintf("Can not access symbol %s", name))
}
return
}
fuzzJsonData, err := loadFunc("fuzz_json_data")
if err != nil {
return nil, err
}
fuzzFlush, err := loadFunc("fuzz_flush")
if err != nil {
return nil, err
}
fuzzGetRandom, err := loadFunc("fuzz_get_random")
if err != nil {
return nil, err
}
notifyCoverage, err := loadFunc("notify_coverage")
if err != nil {
return nil, err
}
initCoverage, err := loadFunc("init_coverage_module")
if err != nil {
return nil, err
}
return &voidstarHandler{fuzzJsonData, fuzzFlush, fuzzGetRandom, initCoverage, notifyCoverage}, nil
}
// If `localOutputEnvVar` is set to a non-empty path, attempt to open that path and truncate the file
+1 -1
View File
@@ -3,7 +3,7 @@ package internal
// --------------------------------------------------------------------------------
// Versions
// --------------------------------------------------------------------------------
const SDK_Version = "0.4.3"
const SDK_Version = "0.6.0"
const Protocol_Version = "1.1.0"
// --------------------------------------------------------------------------------
@@ -0,0 +1,160 @@
//go:build enable_antithesis_sdk && linux && amd64 && cgo
package internal
import (
"fmt"
"unsafe"
"os"
)
// --------------------------------------------------------------------------------
// To build and run an executable with this package
//
// CC=clang CGO_ENABLED=1 go run ./main.go
// --------------------------------------------------------------------------------
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
//
// The commented lines below, and the `import "C"` line which must directly follow
// the commented lines are used by CGO. They are load-bearing, and should not be
// changed without first understanding how CGO uses them.
//
// \/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/\/
// #cgo LDFLAGS: -ldl
//
// #include <dlfcn.h>
// #include <stdbool.h>
// #include <stdint.h>
// #include <stdlib.h>
//
// typedef void (*go_fuzz_json_data_fn)(const char *data, size_t size);
// void
// go_fuzz_json_data(void *f, const char *data, size_t size) {
// ((go_fuzz_json_data_fn)f)(data, size);
// }
//
// typedef void (*go_fuzz_flush_fn)(void);
// void
// go_fuzz_flush(void *f) {
// ((go_fuzz_flush_fn)f)();
// }
//
// typedef uint64_t (*go_fuzz_get_random_fn)(void);
// uint64_t
// go_fuzz_get_random(void *f) {
// return ((go_fuzz_get_random_fn)f)();
// }
//
// typedef bool (*go_notify_coverage_fn)(size_t);
// int
// go_notify_coverage(void *f, size_t edges) {
// bool b = ((go_notify_coverage_fn)f)(edges);
// return b ? 1 : 0;
// }
//
// typedef uint64_t (*go_init_coverage_fn)(size_t num_edges, const char *symbols);
// uint64_t
// go_init_coverage(void *f, size_t num_edges, const char *symbols) {
// return ((go_init_coverage_fn)f)(num_edges, symbols);
// }
//
import "C"
const (
defaultNativeLibraryPath = "/usr/lib/libvoidstar.so"
)
type voidstarHandler struct {
fuzzJsonData unsafe.Pointer
fuzzFlush unsafe.Pointer
fuzzGetRandom unsafe.Pointer
initCoverage unsafe.Pointer
notifyCoverage unsafe.Pointer
}
func (h *voidstarHandler) output(message string) {
msg_len := len(message)
if msg_len == 0 {
return
}
cstrMessage := C.CString(message)
defer C.free(unsafe.Pointer(cstrMessage))
C.go_fuzz_json_data(h.fuzzJsonData, cstrMessage, C.ulong(msg_len))
C.go_fuzz_flush(h.fuzzFlush)
}
func (h *voidstarHandler) random() uint64 {
return uint64(C.go_fuzz_get_random(h.fuzzGetRandom))
}
func (h *voidstarHandler) init_coverage(num_edge uint64, symbols string) uint64 {
cstrSymbols := C.CString(symbols)
defer C.free(unsafe.Pointer(cstrSymbols))
return uint64(C.go_init_coverage(h.initCoverage, C.ulong(num_edge), cstrSymbols))
}
func (h *voidstarHandler) notify(edge uint64) bool {
ival := int(C.go_notify_coverage(h.notifyCoverage, C.ulong(edge)))
return ival == 1
}
// Attempt to load libvoidstar and some symbols from `path`
func openSharedLib(path string) (*voidstarHandler, error) {
cstrPath := C.CString(path)
defer C.free(unsafe.Pointer(cstrPath))
dlError := func(message string) error {
return fmt.Errorf("%s: (%s)", message, C.GoString(C.dlerror()))
}
sharedLib := C.dlopen(cstrPath, C.int(C.RTLD_NOW))
if sharedLib == nil {
return nil, dlError("Can not load the Antithesis native library")
}
loadFunc := func(name string) (symbol unsafe.Pointer, err error) {
cstrName := C.CString(name)
defer C.free(unsafe.Pointer(cstrName))
if symbol = C.dlsym(sharedLib, cstrName); symbol == nil {
err = dlError(fmt.Sprintf("Can not access symbol %s", name))
}
return
}
fuzzJsonData, err := loadFunc("fuzz_json_data")
if err != nil {
return nil, err
}
fuzzFlush, err := loadFunc("fuzz_flush")
if err != nil {
return nil, err
}
fuzzGetRandom, err := loadFunc("fuzz_get_random")
if err != nil {
return nil, err
}
notifyCoverage, err := loadFunc("notify_coverage")
if err != nil {
return nil, err
}
initCoverage, err := loadFunc("init_coverage_module")
if err != nil {
return nil, err
}
return &voidstarHandler{fuzzJsonData, fuzzFlush, fuzzGetRandom, initCoverage, notifyCoverage}, nil
}
// If we have a file at `defaultNativeLibraryPath`, we load the shared library
// (and panic on any error encountered during load).
func init_in_antithesis() libHandler {
if _, err := os.Stat(defaultNativeLibraryPath); err == nil {
handler, err := openSharedLib(defaultNativeLibraryPath)
if err != nil {
panic(err)
}
return handler
}
return nil
}
@@ -0,0 +1,7 @@
//go:build enable_antithesis_sdk && (!linux || !amd64 || !cgo)
package internal
func init_in_antithesis() libHandler {
return nil
}
+30 -8
View File
@@ -7,7 +7,7 @@ This package provides various compression algorithms.
* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams.
* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently.
* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped/zstd HTTP requests efficiently.
* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
[![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
@@ -26,6 +26,22 @@ This package will support the current Go version and 2 versions back.
Use the links above for more information on each.
# changelog
* Jan 16th, 2026 [1.18.3](https://github.com/klauspost/compress/releases/tag/v1.18.3)
* Downstream CVE-2025-61728. See [golang/go#77102](https://github.com/golang/go/issues/77102).
* Dec 1st, 2025 - [1.18.2](https://github.com/klauspost/compress/releases/tag/v1.18.2)
* flate: Fix invalid encoding on level 9 with single value input in https://github.com/klauspost/compress/pull/1115
* flate: reduce stateless allocations by @RXamzin in https://github.com/klauspost/compress/pull/1106
* Oct 20, 2025 - [1.18.1](https://github.com/klauspost/compress/releases/tag/v1.18.1) - RETRACTED
* zstd: Add simple zstd EncodeTo/DecodeTo functions https://github.com/klauspost/compress/pull/1079
* zstd: Fix incorrect buffer size in dictionary encodes https://github.com/klauspost/compress/pull/1059
* s2: check for cap, not len of buffer in EncodeBetter/Best by @vdarulis in https://github.com/klauspost/compress/pull/1080
* zlib: Avoiding extra allocation in zlib.reader.Reset by @travelpolicy in https://github.com/klauspost/compress/pull/1086
* gzhttp: remove redundant err check in zstdReader by @ryanfowler in https://github.com/klauspost/compress/pull/1090
* flate: Faster load+store https://github.com/klauspost/compress/pull/1104
* flate: Simplify matchlen https://github.com/klauspost/compress/pull/1101
* flate: Use exact sizes for huffman tables https://github.com/klauspost/compress/pull/1103
* Feb 19th, 2025 - [1.18.0](https://github.com/klauspost/compress/releases/tag/v1.18.0)
* Add unsafe little endian loaders https://github.com/klauspost/compress/pull/1036
@@ -36,6 +52,9 @@ Use the links above for more information on each.
* flate: Fix matchlen L5+L6 https://github.com/klauspost/compress/pull/1049
* flate: Cleanup & reduce casts https://github.com/klauspost/compress/pull/1050
<details>
<summary>See changes to v1.17.x</summary>
* Oct 11th, 2024 - [1.17.11](https://github.com/klauspost/compress/releases/tag/v1.17.11)
* zstd: Fix extra CRC written with multiple Close calls https://github.com/klauspost/compress/pull/1017
* s2: Don't use stack for index tables https://github.com/klauspost/compress/pull/1014
@@ -102,7 +121,8 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp
* s2: Do 2 overlapping match checks https://github.com/klauspost/compress/pull/839
* flate: Add amd64 assembly matchlen https://github.com/klauspost/compress/pull/837
* gzip: Copy bufio.Reader on Reset by @thatguystone in https://github.com/klauspost/compress/pull/860
</details>
<details>
<summary>See changes to v1.16.x</summary>
@@ -589,7 +609,7 @@ While the release has been extensively tested, it is recommended to testing when
# deflate usage
The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:
The packages are drop-in replacements for standard library [deflate](https://godoc.org/github.com/klauspost/compress/flate), [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip), and [zlib](https://godoc.org/github.com/klauspost/compress/zlib). Simply replace the import path to use them:
Typical speed is about 2x of the standard library packages.
@@ -600,17 +620,15 @@ Typical speed is about 2x of the standard library packages.
| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc) |
| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc) |
* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop-in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
The packages contains the same as the standard library, so you can use the godoc for that: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
The packages implement the same API as the standard library, so you can use the original godoc documentation: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
Currently there is only minor speedup on decompression (mostly CRC32 calculation).
Memory usage is typically 1MB for a Writer. stdlib is in the same range.
If you expect to have a lot of concurrently allocated Writers consider using
the stateless compress described below.
the stateless compression described below.
For compression performance, see: [this spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing).
@@ -669,3 +687,7 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv
# license
This code is licensed under the same conditions as the original Go code. See LICENSE file.
+3 -1
View File
@@ -421,7 +421,9 @@ func (d *compressor) deflateLazy() {
d.h = newHuffmanEncoder(maxFlateBlockTokens)
}
var tmp [256]uint16
for _, v := range d.window[s.index:d.windowEnd] {
toIndex := d.window[s.index:d.windowEnd]
toIndex = toIndex[:min(len(toIndex), maxFlateBlockTokens)]
for _, v := range toIndex {
tmp[v]++
}
d.h.generate(tmp[:], 15)
+5 -4
View File
@@ -646,7 +646,7 @@ func (w *huffmanBitWriter) writeBlockDynamic(tokens *tokens, eof bool, input []b
w.lastHeader = 0
}
numLiterals, numOffsets := w.indexTokens(tokens, fillReuse && !sync)
numLiterals, numOffsets := w.indexTokens(tokens, true)
extraBits := 0
ssize, storable := w.storedSize(input)
@@ -781,7 +781,7 @@ func (w *huffmanBitWriter) fillTokens() {
// literalFreq and offsetFreq, and generates literalEncoding
// and offsetEncoding.
// The number of literal and offset tokens is returned.
func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, numOffsets int) {
func (w *huffmanBitWriter) indexTokens(t *tokens, alwaysEOB bool) (numLiterals, numOffsets int) {
//copy(w.literalFreq[:], t.litHist[:])
*(*[256]uint16)(w.literalFreq[:]) = t.litHist
//copy(w.literalFreq[256:], t.extraHist[:])
@@ -791,9 +791,10 @@ func (w *huffmanBitWriter) indexTokens(t *tokens, filled bool) (numLiterals, num
if t.n == 0 {
return
}
if filled {
return maxNumLit, maxNumDist
if alwaysEOB {
w.literalFreq[endBlockMarker] = 1
}
// get the number of literals
numLiterals = len(w.literalFreq)
for w.literalFreq[numLiterals-1] == 0 {
+16 -4
View File
@@ -61,13 +61,19 @@ var bitWriterPool = sync.Pool{
},
}
// tokensPool contains tokens struct objects that can be reused
var tokensPool = sync.Pool{
New: func() any {
return &tokens{}
},
}
// StatelessDeflate allows compressing directly to a Writer without retaining state.
// When returning everything will be flushed.
// Up to 8KB of an optional dictionary can be given which is presumed to precede the block.
// Longer dictionaries will be truncated and will still produce valid output.
// Sending nil dictionary is perfectly fine.
func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
var dst tokens
bw := bitWriterPool.Get().(*huffmanBitWriter)
bw.reset(out)
defer func() {
@@ -91,6 +97,12 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
// For subsequent loops, keep shallow dict reference to avoid alloc+copy.
var inDict []byte
dst := tokensPool.Get().(*tokens)
dst.Reset()
defer func() {
tokensPool.Put(dst)
}()
for len(in) > 0 {
todo := in
if len(inDict) > 0 {
@@ -113,9 +125,9 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
}
// Compress
if len(inDict) == 0 {
statelessEnc(&dst, todo, int16(len(dict)))
statelessEnc(dst, todo, int16(len(dict)))
} else {
statelessEnc(&dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict)
statelessEnc(dst, inDict[:maxStatelessDict+len(todo)], maxStatelessDict)
}
isEof := eof && len(in) == 0
@@ -129,7 +141,7 @@ func StatelessDeflate(out io.Writer, in []byte, eof bool, dict []byte) error {
// If we removed less than 1/16th, huffman compress the block.
bw.writeBlockHuff(isEof, uncompressed, len(in) == 0)
} else {
bw.writeBlockDynamic(&dst, isEof, uncompressed, len(in) == 0)
bw.writeBlockDynamic(dst, isEof, uncompressed, len(in) == 0)
}
if len(in) > 0 {
// Retain a dict if we have more
+19 -9
View File
@@ -39,9 +39,6 @@ type Decoder struct {
frame *frameDec
// Custom dictionaries.
dicts map[uint32]*dict
// streamWg is the waitgroup for all streams
streamWg sync.WaitGroup
}
@@ -101,12 +98,10 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) {
d.current.err = ErrDecoderNilInput
}
// Transfer option dicts.
d.dicts = make(map[uint32]*dict, len(d.o.dicts))
for _, dc := range d.o.dicts {
d.dicts[dc.id] = dc
// Initialize dict map if needed.
if d.o.dicts == nil {
d.o.dicts = make(map[uint32]*dict)
}
d.o.dicts = nil
// Create decoders
d.decoders = make(chan *blockDec, d.o.concurrent)
@@ -238,6 +233,21 @@ func (d *Decoder) Reset(r io.Reader) error {
return nil
}
// ResetWithOptions will reset the decoder and apply the given options
// for the next stream or DecodeAll operation.
// Options are applied on top of the existing options.
// Some options cannot be changed on reset and will return an error.
func (d *Decoder) ResetWithOptions(r io.Reader, opts ...DOption) error {
d.o.resetOpt = true
defer func() { d.o.resetOpt = false }()
for _, o := range opts {
if err := o(&d.o); err != nil {
return err
}
}
return d.Reset(r)
}
// drainOutput will drain the output until errEndOfStream is sent.
func (d *Decoder) drainOutput() {
if d.current.cancel != nil {
@@ -930,7 +940,7 @@ decodeStream:
}
func (d *Decoder) setDict(frame *frameDec) (err error) {
dict, ok := d.dicts[frame.DictionaryID]
dict, ok := d.o.dicts[frame.DictionaryID]
if ok {
if debugDecoder {
println("setting dict", frame.DictionaryID)
+52 -8
View File
@@ -20,10 +20,11 @@ type decoderOptions struct {
concurrent int
maxDecodedSize uint64
maxWindowSize uint64
dicts []*dict
dicts map[uint32]*dict
ignoreChecksum bool
limitToCap bool
decodeBufsBelow int
resetOpt bool
}
func (o *decoderOptions) setDefault() {
@@ -42,8 +43,15 @@ func (o *decoderOptions) setDefault() {
// WithDecoderLowmem will set whether to use a lower amount of memory,
// but possibly have to allocate more while running.
// Cannot be changed with ResetWithOptions.
func WithDecoderLowmem(b bool) DOption {
return func(o *decoderOptions) error { o.lowMem = b; return nil }
return func(o *decoderOptions) error {
if o.resetOpt && b != o.lowMem {
return errors.New("WithDecoderLowmem cannot be changed on Reset")
}
o.lowMem = b
return nil
}
}
// WithDecoderConcurrency sets the number of created decoders.
@@ -53,18 +61,23 @@ func WithDecoderLowmem(b bool) DOption {
// inflight blocks.
// When decoding streams and setting maximum to 1,
// no async decoding will be done.
// The value supplied must be at least 0.
// When a value of 0 is provided GOMAXPROCS will be used.
// By default this will be set to 4 or GOMAXPROCS, whatever is lower.
// Cannot be changed with ResetWithOptions.
func WithDecoderConcurrency(n int) DOption {
return func(o *decoderOptions) error {
if n < 0 {
return errors.New("concurrency must be at least 1")
return errors.New("concurrency must be at least 0")
}
newVal := n
if n == 0 {
o.concurrent = runtime.GOMAXPROCS(0)
} else {
o.concurrent = n
newVal = runtime.GOMAXPROCS(0)
}
if o.resetOpt && newVal != o.concurrent {
return errors.New("WithDecoderConcurrency cannot be changed on Reset")
}
o.concurrent = newVal
return nil
}
}
@@ -73,6 +86,7 @@ func WithDecoderConcurrency(n int) DOption {
// non-streaming operations or maximum window size for streaming operations.
// This can be used to control memory usage of potentially hostile content.
// Maximum is 1 << 63 bytes. Default is 64GiB.
// Can be changed with ResetWithOptions.
func WithDecoderMaxMemory(n uint64) DOption {
return func(o *decoderOptions) error {
if n == 0 {
@@ -92,16 +106,20 @@ func WithDecoderMaxMemory(n uint64) DOption {
// "zstd --train" from the Zstandard reference implementation.
//
// If several dictionaries with the same ID are provided, the last one will be used.
// Can be changed with ResetWithOptions.
//
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
func WithDecoderDicts(dicts ...[]byte) DOption {
return func(o *decoderOptions) error {
if o.dicts == nil {
o.dicts = make(map[uint32]*dict)
}
for _, b := range dicts {
d, err := loadDict(b)
if err != nil {
return err
}
o.dicts = append(o.dicts, d)
o.dicts[d.id] = d
}
return nil
}
@@ -109,12 +127,16 @@ func WithDecoderDicts(dicts ...[]byte) DOption {
// WithDecoderDictRaw registers a dictionary that may be used by the decoder.
// The slice content can be arbitrary data.
// Can be changed with ResetWithOptions.
func WithDecoderDictRaw(id uint32, content []byte) DOption {
return func(o *decoderOptions) error {
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content))
}
o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}})
if o.dicts == nil {
o.dicts = make(map[uint32]*dict)
}
o.dicts[id] = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}
return nil
}
}
@@ -124,6 +146,7 @@ func WithDecoderDictRaw(id uint32, content []byte) DOption {
// The Decoder will likely allocate more memory based on the WithDecoderLowmem setting.
// If WithDecoderMaxMemory is set to a lower value, that will be used.
// Default is 512MB, Maximum is ~3.75 TB as per zstandard spec.
// Can be changed with ResetWithOptions.
func WithDecoderMaxWindow(size uint64) DOption {
return func(o *decoderOptions) error {
if size < MinWindowSize {
@@ -141,6 +164,7 @@ func WithDecoderMaxWindow(size uint64) DOption {
// or any size set in WithDecoderMaxMemory.
// This can be used to limit decoding to a specific maximum output size.
// Disabled by default.
// Can be changed with ResetWithOptions.
func WithDecodeAllCapLimit(b bool) DOption {
return func(o *decoderOptions) error {
o.limitToCap = b
@@ -153,17 +177,37 @@ func WithDecodeAllCapLimit(b bool) DOption {
// This typically uses less allocations but will have the full decompressed object in memory.
// Note that DecodeAllCapLimit will disable this, as well as giving a size of 0 or less.
// Default is 128KiB.
// Cannot be changed with ResetWithOptions.
func WithDecodeBuffersBelow(size int) DOption {
return func(o *decoderOptions) error {
if o.resetOpt && size != o.decodeBufsBelow {
return errors.New("WithDecodeBuffersBelow cannot be changed on Reset")
}
o.decodeBufsBelow = size
return nil
}
}
// IgnoreChecksum allows to forcibly ignore checksum checking.
// Can be changed with ResetWithOptions.
func IgnoreChecksum(b bool) DOption {
return func(o *decoderOptions) error {
o.ignoreChecksum = b
return nil
}
}
// WithDecoderDictDelete removes dictionaries by ID.
// If no ids are passed, all dictionaries are deleted.
// Should be used with ResetWithOptions.
func WithDecoderDictDelete(ids ...uint32) DOption {
return func(o *decoderOptions) error {
if len(ids) == 0 {
clear(o.dicts)
}
for _, id := range ids {
delete(o.dicts, id)
}
return nil
}
}
+16
View File
@@ -131,6 +131,22 @@ func (e *Encoder) Reset(w io.Writer) {
s.frameContentSize = 0
}
// ResetWithOptions will re-initialize the writer and apply the given options
// as a new, independent stream.
// Options are applied on top of the existing options.
// Some options cannot be changed on reset and will return an error.
func (e *Encoder) ResetWithOptions(w io.Writer, opts ...EOption) error {
e.o.resetOpt = true
defer func() { e.o.resetOpt = false }()
for _, o := range opts {
if err := o(&e.o); err != nil {
return err
}
}
e.Reset(w)
return nil
}
// ResetContentSize will reset and set a content size for the next stream.
// If the bytes written does not match the size given an error will be returned
// when calling Close().
+41 -3
View File
@@ -14,6 +14,7 @@ type EOption func(*encoderOptions) error
// options retains accumulated state of multiple options.
type encoderOptions struct {
resetOpt bool
concurrent int
level EncoderLevel
single *bool
@@ -71,19 +72,28 @@ func (o encoderOptions) encoder() encoder {
// WithEncoderCRC will add CRC value to output.
// Output will be 4 bytes larger.
// Can be changed with ResetWithOptions.
func WithEncoderCRC(b bool) EOption {
return func(o *encoderOptions) error { o.crc = b; return nil }
}
// WithEncoderConcurrency will set the concurrency,
// meaning the maximum number of encoders to run concurrently.
// The value supplied must be at least 1.
// The value supplied must be at least 0.
// When a value of 0 is provided GOMAXPROCS will be used.
// For streams, setting a value of 1 will disable async compression.
// By default this will be set to GOMAXPROCS.
// Cannot be changed with ResetWithOptions.
func WithEncoderConcurrency(n int) EOption {
return func(o *encoderOptions) error {
if n <= 0 {
return fmt.Errorf("concurrency must be at least 1")
if n < 0 {
return errors.New("concurrency must at least 0")
}
if n == 0 {
n = runtime.GOMAXPROCS(0)
}
if o.resetOpt && n != o.concurrent {
return errors.New("WithEncoderConcurrency cannot be changed on Reset")
}
o.concurrent = n
return nil
@@ -95,6 +105,7 @@ func WithEncoderConcurrency(n int) EOption {
// A larger value will enable better compression but allocate more memory and,
// for above-default values, take considerably longer.
// The default value is determined by the compression level and max 8MB.
// Cannot be changed with ResetWithOptions.
func WithWindowSize(n int) EOption {
return func(o *encoderOptions) error {
switch {
@@ -105,6 +116,9 @@ func WithWindowSize(n int) EOption {
case (n & (n - 1)) != 0:
return errors.New("window size must be a power of 2")
}
if o.resetOpt && n != o.windowSize {
return errors.New("WithWindowSize cannot be changed on Reset")
}
o.windowSize = n
o.customWindow = true
@@ -122,6 +136,7 @@ func WithWindowSize(n int) EOption {
// n must be > 0 and <= 1GB, 1<<30 bytes.
// The padded area will be filled with data from crypto/rand.Reader.
// If `EncodeAll` is used with data already in the destination, the total size will be multiple of this.
// Can be changed with ResetWithOptions.
func WithEncoderPadding(n int) EOption {
return func(o *encoderOptions) error {
if n <= 0 {
@@ -215,12 +230,16 @@ func (e EncoderLevel) String() string {
}
// WithEncoderLevel specifies a predefined compression level.
// Cannot be changed with ResetWithOptions.
func WithEncoderLevel(l EncoderLevel) EOption {
return func(o *encoderOptions) error {
switch {
case l <= speedNotSet || l >= speedLast:
return fmt.Errorf("unknown encoder level")
}
if o.resetOpt && l != o.level {
return errors.New("WithEncoderLevel cannot be changed on Reset")
}
o.level = l
if !o.customWindow {
switch o.level {
@@ -248,6 +267,7 @@ func WithEncoderLevel(l EncoderLevel) EOption {
// WithZeroFrames will encode 0 length input as full frames.
// This can be needed for compatibility with zstandard usage,
// but is not needed for this package.
// Can be changed with ResetWithOptions.
func WithZeroFrames(b bool) EOption {
return func(o *encoderOptions) error {
o.fullZero = b
@@ -259,6 +279,7 @@ func WithZeroFrames(b bool) EOption {
// Disabling this will skip incompressible data faster, but in cases with no matches but
// skewed character distribution compression is lost.
// Default value depends on the compression level selected.
// Can be changed with ResetWithOptions.
func WithAllLitEntropyCompression(b bool) EOption {
return func(o *encoderOptions) error {
o.customALEntropy = true
@@ -270,6 +291,7 @@ func WithAllLitEntropyCompression(b bool) EOption {
// WithNoEntropyCompression will always skip entropy compression of literals.
// This can be useful if content has matches, but unlikely to benefit from entropy
// compression. Usually the slight speed improvement is not worth enabling this.
// Can be changed with ResetWithOptions.
func WithNoEntropyCompression(b bool) EOption {
return func(o *encoderOptions) error {
o.noEntropy = b
@@ -287,6 +309,7 @@ func WithNoEntropyCompression(b bool) EOption {
// This is only a recommendation, each decoder is free to support higher or lower limits, depending on local limitations.
// If this is not specified, block encodes will automatically choose this based on the input size and the window size.
// This setting has no effect on streamed encodes.
// Can be changed with ResetWithOptions.
func WithSingleSegment(b bool) EOption {
return func(o *encoderOptions) error {
o.single = &b
@@ -298,8 +321,12 @@ func WithSingleSegment(b bool) EOption {
// slower encoding speed.
// This will not change the window size which is the primary function for reducing
// memory usage. See WithWindowSize.
// Cannot be changed with ResetWithOptions.
func WithLowerEncoderMem(b bool) EOption {
return func(o *encoderOptions) error {
if o.resetOpt && b != o.lowMem {
return errors.New("WithLowerEncoderMem cannot be changed on Reset")
}
o.lowMem = b
return nil
}
@@ -311,6 +338,7 @@ func WithLowerEncoderMem(b bool) EOption {
// "zstd --train" from the Zstandard reference implementation.
//
// The encoder *may* choose to use no dictionary instead for certain payloads.
// Can be changed with ResetWithOptions.
//
// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format
func WithEncoderDict(dict []byte) EOption {
@@ -328,6 +356,7 @@ func WithEncoderDict(dict []byte) EOption {
//
// The slice content may contain arbitrary data. It will be used as an initial
// history.
// Can be changed with ResetWithOptions.
func WithEncoderDictRaw(id uint32, content []byte) EOption {
return func(o *encoderOptions) error {
if bits.UintSize > 32 && uint(len(content)) > dictMaxLength {
@@ -337,3 +366,12 @@ func WithEncoderDictRaw(id uint32, content []byte) EOption {
return nil
}
}
// WithEncoderDictDelete clears the dictionary, so no dictionary will be used.
// Should be used with ResetWithOptions.
func WithEncoderDictDelete() EOption {
return func(o *encoderOptions) error {
o.dict = nil
return nil
}
}
+9 -6
View File
@@ -152,19 +152,22 @@ type Mapping map[Subject][]WeightedMapping
func (m *Mapping) Validate(vr *ValidationResults) {
for ubFrom, wm := range (map[Subject][]WeightedMapping)(*m) {
ubFrom.Validate(vr)
perCluster := make(map[string]uint8)
total := uint8(0)
perCluster := make(map[string]uint32)
total := uint32(0)
for _, e := range wm {
e.Subject.Validate(vr)
if e.GetWeight() > 100 {
vr.AddError("Mapping %q has a weight %d that exceeds 100", ubFrom, e.GetWeight())
}
if e.Cluster != "" {
t := perCluster[e.Cluster]
t += e.Weight
t += uint32(e.GetWeight())
perCluster[e.Cluster] = t
if t > 100 {
vr.AddError("Mapping %q in cluster %q exceeds 100%% among all of it's weighted to mappings", ubFrom, e.Cluster)
}
} else {
total += e.GetWeight()
total += uint32(e.GetWeight())
}
}
if total > 100 {
@@ -286,7 +289,7 @@ func (a *Account) Validate(acct *AccountClaims, vr *ValidationResults) {
tvr := CreateValidationResults()
a.Trace.Destination.Validate(tvr)
if !tvr.IsEmpty() {
vr.AddError(fmt.Sprintf("the account Trace.Destination %s", tvr.Issues[0].Description))
vr.AddError("the account Trace.Destination %s", tvr.Issues[0].Description)
}
if a.Trace.Destination.HasWildCards() {
vr.AddError("the account Trace.Destination subject %q is not a valid publish subject", a.Trace.Destination)
@@ -325,7 +328,7 @@ func (a *Account) Validate(acct *AccountClaims, vr *ValidationResults) {
a.Info.Validate(vr)
if err := a.ClusterTraffic.Valid(); err != nil {
vr.AddError(err.Error())
vr.AddError("%s", err.Error())
}
}
+15
View File
@@ -63,6 +63,9 @@ func formatJwt(kind string, jwtString string) ([]byte, error) {
func DecorateSeed(seed []byte) ([]byte, error) {
w := bytes.NewBuffer(nil)
ts := bytes.TrimSpace(seed)
if len(ts) < 2 {
return nil, errors.New("seed is too short")
}
pre := string(ts[0:2])
kind := ""
switch pre {
@@ -138,6 +141,18 @@ func FormatUserConfig(jwtString string, seed []byte) ([]byte, error) {
return nil, fmt.Errorf("nkey seed is not an user seed")
}
kp, err := nkeys.FromSeed(seed)
if err != nil {
return nil, err
}
pk, err := kp.PublicKey()
if err != nil {
return nil, err
}
if pk != gc.Claims().Subject {
return nil, fmt.Errorf("nkey seed does not match the jwt subject")
}
d, err := DecorateSeed(seed)
if err != nil {
return nil, err
+9
View File
@@ -26,6 +26,12 @@ import (
const libVersion = 2
// MaxTokenSize is the maximum size of a JWT token in bytes
const MaxTokenSize = 1024 * 1024 // 1MB
// ErrTokenTooLarge is returned when a token exceeds MaxTokenSize
var ErrTokenTooLarge = errors.New("token too large")
type identifier struct {
Type ClaimType `json:"type,omitempty"`
GenericFields `json:"nats,omitempty"`
@@ -56,6 +62,9 @@ type v1ClaimsDataDeletedFields struct {
// doesn't match the expected algorithm, or the claim is
// not valid or verification fails an error is returned.
func Decode(token string) (Claims, error) {
if len(token) > MaxTokenSize {
return nil, fmt.Errorf("token size %d exceeds maximum of %d bytes: %w", len(token), MaxTokenSize, ErrTokenTooLarge)
}
// must have 3 chunks
chunks := strings.Split(token, ".")
if len(chunks) != 3 {
+12 -7
View File
@@ -126,7 +126,8 @@ type Imports []*Import
// Validate checks if an import is valid for the wrapping account
func (i *Imports) Validate(acctPubKey string, vr *ValidationResults) {
toSet := make(map[Subject]struct{}, len(*i))
// Group subjects by account to check for overlaps only within the same account
subsByAcct := make(map[string]map[Subject]struct{}, len(*i))
for _, v := range *i {
if v == nil {
vr.AddError("null import is not allowed")
@@ -140,15 +141,19 @@ func (i *Imports) Validate(acctPubKey string, vr *ValidationResults) {
if sub == "" {
sub = v.Subject
}
for k := range toSet {
if sub.IsContainedIn(k) || k.IsContainedIn(sub) {
vr.AddError("overlapping subject namespace for %q and %q", sub, k)
// Check for overlapping subjects only within the same account
for subOther := range subsByAcct[v.Account] {
if sub.IsContainedIn(subOther) || subOther.IsContainedIn(sub) {
vr.AddError("overlapping subject namespace for %q and %q in same account %q", sub, subOther, v.Account)
}
}
if _, ok := toSet[sub]; ok {
vr.AddError("overlapping subject namespace for %q", v.To)
if subsByAcct[v.Account] == nil {
subsByAcct[v.Account] = make(map[Subject]struct{}, len(*i))
}
toSet[sub] = struct{}{}
if _, ok := subsByAcct[v.Account][sub]; ok {
vr.AddError("overlapping subject namespace for %q in account %q", sub, v.Account)
}
subsByAcct[v.Account][sub] = struct{}{}
}
v.Validate(acctPubKey, vr)
}
+2 -2
View File
@@ -71,12 +71,12 @@ func ParseServerVersion(version string) (int, int, int, error) {
// Validate checks the validity of the operators contents
func (o *Operator) Validate(vr *ValidationResults) {
if err := o.validateAccountServerURL(); err != nil {
vr.AddError(err.Error())
vr.AddError("%s", err.Error())
}
for _, v := range o.validateOperatorServiceURLs() {
if v != nil {
vr.AddError(v.Error())
vr.AddError("%s", v.Error())
}
}
+56 -39
View File
@@ -60,6 +60,9 @@ type parser struct {
// pedantic reports error when configuration is not correct.
pedantic bool
// Tracks environment variable references, to avoid cycles
envVarReferences map[string]bool
}
// Parse will return a map of keys to any, although concrete types
@@ -111,42 +114,28 @@ func ParseFileWithChecks(fp string) (map[string]any, error) {
return p.mapping, nil
}
// cleanupUsedEnvVars will recursively remove all already used
// environment variables which might be in the parsed tree.
func cleanupUsedEnvVars(m map[string]any) {
for k, v := range m {
t := v.(*token)
if t.usedVariable {
delete(m, k)
continue
}
// Cleanup any other env var that is still in the map.
if tm, ok := t.value.(map[string]any); ok {
cleanupUsedEnvVars(tm)
}
// configDigest returns a digest for the parsed config.
func configDigest(m map[string]any) (string, error) {
digest := sha256.New()
e := json.NewEncoder(digest)
if err := e.Encode(m); err != nil {
return _EMPTY_, err
}
return fmt.Sprintf("sha256:%x", digest.Sum(nil)), nil
}
// ParseFileWithChecksDigest returns the processed config and a digest
// that represents the configuration.
func ParseFileWithChecksDigest(fp string) (map[string]any, string, error) {
data, err := os.ReadFile(fp)
m, err := ParseFileWithChecks(fp)
if err != nil {
return nil, _EMPTY_, err
}
p, err := parse(string(data), fp, true)
digest, err := configDigest(m)
if err != nil {
return nil, _EMPTY_, err
}
// Filter out any environment variables before taking the digest.
cleanupUsedEnvVars(p.mapping)
digest := sha256.New()
e := json.NewEncoder(digest)
err = e.Encode(p.mapping)
if err != nil {
return nil, _EMPTY_, err
}
return p.mapping, fmt.Sprintf("sha256:%x", digest.Sum(nil)), nil
return m, digest, nil
}
type token struct {
@@ -180,16 +169,37 @@ func (t *token) Position() int {
return t.item.pos
}
func parse(data, fp string, pedantic bool) (p *parser, err error) {
p = &parser{
mapping: make(map[string]any),
lx: lex(data),
ctxs: make([]any, 0, 4),
keys: make([]string, 0, 4),
ikeys: make([]item, 0, 4),
fp: filepath.Dir(fp),
pedantic: pedantic,
func newParser(data, fp string, pedantic bool) *parser {
return &parser{
mapping: make(map[string]any),
lx: lex(data),
ctxs: make([]any, 0, 4),
keys: make([]string, 0, 4),
ikeys: make([]item, 0, 4),
fp: filepath.Dir(fp),
pedantic: pedantic,
envVarReferences: make(map[string]bool),
}
}
func parse(data, fp string, pedantic bool) (*parser, error) {
p := newParser(data, fp, pedantic)
if err := p.parse(fp); err != nil {
return nil, err
}
return p, nil
}
func parseEnv(data string, parent *parser) (*parser, error) {
p := newParser(data, "", false)
p.envVarReferences = parent.envVarReferences
if err := p.parse(""); err != nil {
return nil, err
}
return p, nil
}
func (p *parser) parse(fp string) error {
p.pushContext(p.mapping)
var prevItem item
@@ -199,16 +209,16 @@ func parse(data, fp string, pedantic bool) (p *parser, err error) {
// Here we allow the final character to be a bracket '}'
// in order to support JSON like configurations.
if prevItem.typ == itemKey && prevItem.val != mapEndString {
return nil, fmt.Errorf("config is invalid (%s:%d:%d)", fp, it.line, it.pos)
return fmt.Errorf("config is invalid (%s:%d:%d)", fp, it.line, it.pos)
}
break
}
prevItem = it
if err := p.processItem(it, fp); err != nil {
return nil, err
return err
}
}
return p, nil
return nil
}
func (p *parser) next() item {
@@ -453,11 +463,18 @@ func (p *parser) lookupVariable(varReference string) (any, bool, error) {
}
// If we are here, we have exhausted our context maps and still not found anything.
// Parse from the environment.
// Detect reference cycles
if p.envVarReferences[varReference] {
return nil, false, fmt.Errorf("variable reference cycle for '%s'", varReference)
}
p.envVarReferences[varReference] = true
defer delete(p.envVarReferences, varReference)
// Parse from the environment
if vStr, ok := os.LookupEnv(varReference); ok {
// Everything we get here will be a string value, so we need to process as a parser would.
if vmap, err := Parse(fmt.Sprintf("%s=%s", pkey, vStr)); err == nil {
v, ok := vmap[pkey]
if subp, err := parseEnv(fmt.Sprintf("%s=%s", pkey, vStr), p); err == nil {
v, ok := subp.mapping[pkey]
return v, ok, nil
} else {
return nil, false, err
+10 -10
View File
@@ -235,19 +235,19 @@ func (d *DN) RDNsMatch(other *DN) bool {
if len(d.RDNs) != len(other.RDNs) {
return false
}
CheckNextRDN:
matched := make([]bool, len(other.RDNs))
for _, irdn := range d.RDNs {
for _, ordn := range other.RDNs {
if (len(irdn.Attributes) == len(ordn.Attributes)) &&
(irdn.hasAllAttributes(ordn.Attributes) && ordn.hasAllAttributes(irdn.Attributes)) {
// Found the RDN, check if next one matches.
continue CheckNextRDN
found := false
for j, ordn := range other.RDNs {
if !matched[j] && irdn.Equal(ordn) {
matched[j] = true
found = true
break
}
}
// Could not find a matching individual RDN, auth fails.
return false
if !found {
return false
}
}
return true
}
+21 -10
View File
@@ -138,6 +138,12 @@ type sconns struct {
leafs int32
}
// clampInt64ToInt32 safely converts an int64 limit to int32,
// clamping values to the [math.MinInt32, math.MaxInt32] range.
func clampInt64ToInt32(v int64) int32 {
return int32(max(math.MinInt32, min(math.MaxInt32, v)))
}
// Import stream mapping struct
type streamImport struct {
acc *Account
@@ -299,6 +305,7 @@ func (a *Account) shallowCopy(na *Account) {
na.Nkey = a.Nkey
na.Issuer = a.Issuer
na.traceDest, na.traceDestSampling = a.traceDest, a.traceDestSampling
na.nrgAccount = a.nrgAccount
if a.imports.streams != nil {
na.imports.streams = make([]*streamImport, 0, len(a.imports.streams))
@@ -1609,10 +1616,12 @@ func (a *Account) checkServiceImportsForCycles(from string, visited map[string]b
}
// Push ourselves and check si.acc
visited[a.Name] = true
if subjectIsSubsetMatch(si.from, from) {
from = si.from
// Make a copy to not overwrite the passed value.
f := from
if subjectIsSubsetMatch(si.from, f) {
f = si.from
}
if err := si.acc.checkServiceImportsForCycles(from, visited); err != nil {
if err := si.acc.checkServiceImportsForCycles(f, visited); err != nil {
return err
}
a.mu.RLock()
@@ -1667,10 +1676,12 @@ func (a *Account) checkStreamImportsForCycles(to string, visited map[string]bool
}
// Push ourselves and check si.acc
visited[a.Name] = true
if subjectIsSubsetMatch(si.to, to) {
to = si.to
// Make a copy to not overwrite the passed value.
t := to
if subjectIsSubsetMatch(si.to, t) {
t = si.to
}
if err := si.acc.checkStreamImportsForCycles(to, visited); err != nil {
if err := si.acc.checkStreamImportsForCycles(t, visited); err != nil {
return err
}
a.mu.RLock()
@@ -3715,10 +3726,10 @@ func (s *Server) updateAccountClaimsWithRefresh(a *Account, ac *jwt.AccountClaim
// Now do limits if they are present.
a.mu.Lock()
a.msubs = int32(ac.Limits.Subs)
a.mpay = int32(ac.Limits.Payload)
a.mconns = int32(ac.Limits.Conn)
a.mleafs = int32(ac.Limits.LeafNodeConn)
a.msubs = clampInt64ToInt32(ac.Limits.Subs)
a.mpay = clampInt64ToInt32(ac.Limits.Payload)
a.mconns = clampInt64ToInt32(ac.Limits.Conn)
a.mleafs = clampInt64ToInt32(ac.Limits.LeafNodeConn)
a.disallowBearer = ac.Limits.DisallowBearer
// Check for any revocations
if len(ac.Revocations) > 0 {
+37 -8
View File
@@ -421,7 +421,9 @@ func (c *client) matchesPinnedCert(tlsPinnedCerts PinnedCertSet) bool {
}
var (
mustacheRE = regexp.MustCompile(`{{2}([^}]+)}{2}`)
mustacheRE = regexp.MustCompile(`{{2}([^}]+)}{2}`)
maxPermTemplateSubjectExpansions = 4096
errPermTemplateExpansionLimit error = fmt.Errorf("template expansion exceeds limit")
)
func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.UserClaims, acc *Account) (jwt.UserPermissionLimits, error) {
@@ -456,11 +458,11 @@ func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.User
return p
}
isTag := func(op string) []string {
if strings.EqualFold("tag(", op[:4]) && strings.HasSuffix(op, ")") {
if len(op) >= 4 && strings.EqualFold("tag(", op[:4]) && strings.HasSuffix(op, ")") {
v := strings.TrimPrefix(op, "tag(")
v = strings.TrimSuffix(v, ")")
return []string{"tag", v}
} else if strings.EqualFold("account-tag(", op[:12]) && strings.HasSuffix(op, ")") {
} else if len(op) >= 12 && strings.EqualFold("account-tag(", op[:12]) && strings.HasSuffix(op, ")") {
v := strings.TrimPrefix(op, "account-tag(")
v = strings.TrimSuffix(v, ")")
return []string{"account-tag", v}
@@ -529,7 +531,7 @@ func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.User
// generate an invalid subject?
values[tokenNum] = []string{" "}
}
} else if failOnBadSubject {
} else {
return nil, fmt.Errorf("template operation in %q: %q is not defined", list[i], op)
}
}
@@ -544,6 +546,20 @@ func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.User
return nil, fmt.Errorf("generated invalid subject")
}
} else {
expCount := 1
for _, v := range values {
if len(v) == 0 {
expCount = 0
break
}
if expCount > maxPermTemplateSubjectExpansions/len(v) {
return nil, fmt.Errorf("%w: %d", errPermTemplateExpansionLimit, maxPermTemplateSubjectExpansions)
}
expCount *= len(v)
}
if len(emittedList) > maxPermTemplateSubjectExpansions-expCount {
return nil, fmt.Errorf("%w: %d", errPermTemplateExpansionLimit, maxPermTemplateSubjectExpansions)
}
a := nArrayCartesianProduct(values...)
for _, aa := range a {
subj := list[i]
@@ -588,6 +604,7 @@ func processUserPermissionsTemplate(lim jwt.UserPermissionLimits, ujwt *jwt.User
func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (authorized bool) {
var (
nkey *NkeyUser
ujwt string
juc *jwt.UserClaims
acc *Account
user *User
@@ -782,16 +799,23 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
// Check if we have trustedKeys defined in the server. If so we require a user jwt.
if s.trustedKeys != nil {
if c.opts.JWT == _EMPTY_ && opts.DefaultSentinel != _EMPTY_ {
c.opts.JWT = opts.DefaultSentinel
ujwt = c.opts.JWT
if ujwt == _EMPTY_ && c.isMqtt() {
// For MQTT, we pass the password as the JWT too, but do so here so it's not
// publicly exposed in the client options if it isn't a JWT.
ujwt = c.opts.Password
}
if c.opts.JWT == _EMPTY_ {
if ujwt == _EMPTY_ && opts.DefaultSentinel != _EMPTY_ {
c.opts.JWT = opts.DefaultSentinel
ujwt = c.opts.JWT
}
if ujwt == _EMPTY_ {
s.mu.Unlock()
c.Debugf("Authentication requires a user JWT")
return false
}
// So we have a valid user jwt here.
juc, err = jwt.DecodeUserClaims(c.opts.JWT)
juc, err = jwt.DecodeUserClaims(ujwt)
if err != nil {
s.mu.Unlock()
c.Debugf("User JWT not valid: %v", err)
@@ -1061,6 +1085,11 @@ func (s *Server) processClientOrLeafAuthentication(c *client, opts *Options) (au
// Hold onto the user's public key.
c.mu.Lock()
c.pubKey = juc.Subject
// If this is a MQTT client, we purposefully didn't populate the JWT as it could contain
// a password or token. Now we know it's a valid JWT, we can populate it.
if c.isMqtt() {
c.opts.JWT = ujwt
}
c.tags = juc.Tags
c.nameTag = juc.Name
c.mu.Unlock()
+15 -8
View File
@@ -32,6 +32,14 @@ const (
AuthRequestXKeyHeader = "Nats-Server-Xkey"
)
func titleCase(m string) string {
r := []rune(m)
if len(r) == 0 {
return _EMPTY_
}
return string(append([]rune{unicode.ToUpper(r[0])}, r[1:]...))
}
// Process a callout on this client's behalf.
func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequired, trustedProxy bool) (authorized bool, errStr string) {
isOperatorMode := len(opts.TrustedKeys) > 0
@@ -50,6 +58,13 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequi
} else {
acc = c.acc
}
if acc == nil {
// FIX for https://github.com/nats-io/nats-server/issues/7841
// hand rolled creds on leafnode became crasher here
errStr = fmt.Sprintf("%s not mapped to a callout account", c.kindString())
s.Warnf(errStr)
return false, errStr
}
// Check if we have been requested to encrypt.
var xkp nkeys.KeyPair
@@ -66,9 +81,6 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequi
xkp, xkey = s.xkp, s.info.XKey
}
// FIXME: so things like the server ID that get assigned, are used as a sort of nonce - but
// reality is that the keypair here, is generated, so the response generated a JWT has to be
// this user - no replay possible
// Create a keypair for the user. We will expect this public user to be in the signed response.
// This prevents replay attacks.
ukp, _ := nkeys.CreateUser()
@@ -234,11 +246,6 @@ func (s *Server) processClientOrLeafCallout(c *client, opts *Options, proxyRequi
}
processReply := func(_ *subscription, rc *client, racc *Account, subject, reply string, rmsg []byte) {
titleCase := func(m string) string {
r := []rune(m)
return string(append([]rune{unicode.ToUpper(r[0])}, r[1:]...))
}
arc, err := decodeResponse(rc, rmsg, racc)
if err != nil {
c.authViolation()
+1 -1
View File
@@ -1,4 +1,4 @@
// Copyright 2023-2024 The NATS Authors
// Copyright 2023-2025 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
+254 -116
View File
@@ -1,4 +1,4 @@
// Copyright 2012-2025 The NATS Authors
// Copyright 2012-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -23,6 +23,7 @@ import (
"errors"
"fmt"
"io"
"math"
"math/rand"
"net"
"net/http"
@@ -35,8 +36,6 @@ import (
"sync/atomic"
"time"
"slices"
"github.com/klauspost/compress/s2"
"github.com/nats-io/jwt/v2"
"github.com/nats-io/nats-server/v2/internal/fastrand"
@@ -154,7 +153,6 @@ const (
compressionNegotiated // Marks if this connection has negotiated compression level with remote.
didTLSFirst // Marks if this connection requested and was accepted doing the TLS handshake first (prior to INFO).
isSlowConsumer // Marks connection as a slow consumer.
firstPong // Marks if this is the first PONG received
)
// set the flag (would be equivalent to set the boolean to true)
@@ -870,6 +868,11 @@ func (c *client) registerWithAccount(acc *Account) error {
}
c.mu.Lock()
// This check does not apply to SYSTEM or JETSTREAM or ACCOUNT clients (because they don't have a `nc`...)
if c.isClosed() && !isInternalClient(c.kind) {
c.mu.Unlock()
return ErrConnectionClosed
}
kind := c.kind
srv := c.srv
c.acc = acc
@@ -929,14 +932,14 @@ func (c *client) applyAccountLimits() {
c.msubs = jwt.NoLimit
if c.opts.JWT != _EMPTY_ { // user jwt implies account
if uc, _ := jwt.DecodeUserClaims(c.opts.JWT); uc != nil {
atomic.StoreInt32(&c.mpay, int32(uc.Limits.Payload))
c.msubs = int32(uc.Limits.Subs)
atomic.StoreInt32(&c.mpay, clampInt64ToInt32(uc.Limits.Payload))
c.msubs = clampInt64ToInt32(uc.Limits.Subs)
if uc.IssuerAccount != _EMPTY_ && uc.IssuerAccount != uc.Issuer {
if scope, ok := c.acc.signingKeys[uc.Issuer]; ok {
if userScope, ok := scope.(*jwt.UserScope); ok {
// if signing key disappeared or changed and we don't get here, the client will be disconnected
c.mpay = int32(userScope.Template.Limits.Payload)
c.msubs = int32(userScope.Template.Limits.Subs)
c.mpay = clampInt64ToInt32(userScope.Template.Limits.Payload)
c.msubs = clampInt64ToInt32(userScope.Template.Limits.Subs)
}
}
}
@@ -1354,6 +1357,13 @@ func (c *client) flushClients(budget time.Duration) time.Time {
return last
}
func (c *client) resetReadLoopStallTime() {
if c.in.tst >= stallClientMaxDuration {
c.rateLimitFormatWarnf("Producer was stalled for a total of %v", c.in.tst.Round(time.Millisecond))
}
c.in.tst = 0
}
// readLoop is the main socket read functionality.
// Runs in its own Go routine.
func (c *client) readLoop(pre []byte) {
@@ -1431,21 +1441,6 @@ func (c *client) readLoop(pre []byte) {
return
}
}
if ws {
bufs, err = c.wsRead(wsr, reader, b[:n])
if bufs == nil && err != nil {
if err != io.EOF {
c.Errorf("read error: %v", err)
}
c.closeConnection(closedStateForErr(err))
return
} else if bufs == nil {
continue
}
} else {
bufs[0] = b[:n]
}
// Check if the account has mappings and if so set the local readcache flag.
// We check here to make sure any changes such as config reload are reflected here.
if c.kind == CLIENT || c.kind == LEAF {
@@ -1463,17 +1458,32 @@ func (c *client) readLoop(pre []byte) {
c.in.bytes = 0
c.in.subs = 0
if ws {
err = c.wsReadAndParse(wsr, reader, b[:n])
if err != nil {
// Match the normal parse path: any already-buffered deliveries
// need their pending flush signals drained before we close.
c.flushClients(0)
if err != io.EOF {
c.Errorf("read error: %v", err)
}
c.closeConnection(closedStateForErr(err))
return
}
c.resetReadLoopStallTime()
goto postParse
} else {
bufs[0] = b[:n]
}
// Main call into parser for inbound data. This will generate callouts
// to process messages, etc.
for i := 0; i < len(bufs); i++ {
if err := c.parse(bufs[i]); err != nil {
if err == ErrMinimumVersionRequired {
// Special case here, currently only for leaf node connections.
// When process the CONNECT protocol, if the minimum version
// required was not met, an error was printed and sent back to
// the remote, and connection was closed after a certain delay
// (to avoid "rapid" reconnection from the remote).
// We don't need to do any of the things below, simply return.
// processLeafConnect() already sent the rejection and closed
// the connection, so there is nothing else to do here.
return
}
if dur := time.Since(c.in.start); dur >= readLoopReportThreshold {
@@ -1490,13 +1500,10 @@ func (c *client) readLoop(pre []byte) {
}
return
}
// Clear total stalled time here.
if c.in.tst >= stallClientMaxDuration {
c.rateLimitFormatWarnf("Producer was stalled for a total of %v", c.in.tst.Round(time.Millisecond))
}
c.in.tst = 0
c.resetReadLoopStallTime()
}
postParse:
// If we are a ROUTER/LEAF and have processed an INFO, it is possible that
// we are asked to switch to compression now.
if checkCompress && c.in.flags.isSet(switchToCompression) {
@@ -1687,9 +1694,11 @@ func (c *client) flushOutbound() bool {
cw.Reset(&bb)
for _, buf := range collapsed {
if _, err = cw.Write(buf); err != nil {
break
if err == nil {
_, err = cw.Write(buf)
}
// Return always after consumed or error.
nbPoolPut(buf)
}
if err == nil {
err = cw.Close()
@@ -2114,41 +2123,37 @@ func (c *client) processErr(errStr string) {
}
}
// Password pattern matcher.
var passPat = regexp.MustCompile(`"?\s*pass\S*?"?\s*[:=]\s*"?(([^",\r\n}])*)`)
var tokenPat = regexp.MustCompile(`"?\s*auth_token\S*?"?\s*[:=]\s*"?(([^",\r\n}])*)`)
// Matcher for pass/password and auth_token fields.
var prefixAuthPat = regexp.MustCompile(`"?\s*(?:auth_token\S*?|pass\S*?)"?\s*[:=]\s*"?([^",\r\n}]*)`)
// Exact matcher for fields sig, proxy_sig and nkey.
// Overlapping field "sig" does not match inside "proxy_sig".
var exactAuthPat = regexp.MustCompile(`(?:^|[^A-Za-z0-9_])"?\s*(?:proxy_sig|nkey|sig)"?\s*[:=]\s*"?([^",\r\n}]*)`)
// removeSecretsFromTrace removes any notion of passwords/tokens from trace
// messages for logging.
func removeSecretsFromTrace(arg []byte) []byte {
buf := redact("pass", passPat, arg)
return redact("auth_token", tokenPat, buf)
buf := redact(prefixAuthPat, arg)
return redact(exactAuthPat, buf)
}
func redact(name string, pat *regexp.Regexp, proto []byte) []byte {
if !bytes.Contains(proto, []byte(name)) {
func redact(pat *regexp.Regexp, proto []byte) []byte {
m := pat.FindAllSubmatchIndex(proto, -1)
if len(m) == 0 {
return proto
}
// Take a copy of the connect proto just for the trace message.
var _arg [4096]byte
buf := append(_arg[:0], proto...)
m := pat.FindAllSubmatchIndex(buf, -1)
if len(m) == 0 {
return proto
}
redactedPass := []byte("[REDACTED]")
for _, i := range m {
if len(i) < 4 {
for i := len(m) - 1; i >= 0; i-- {
match := m[i]
if len(match) < 4 {
continue
}
start := i[2]
end := i[3]
start, end := match[2], match[3]
// Replace value substring.
buf = append(buf[:start], append(redactedPass, buf[end:]...)...)
break
}
return buf
}
@@ -2684,11 +2689,9 @@ func (c *client) processPong() {
c.rtt = computeRTT(c.rttStart)
srv := c.srv
reorderGWs := c.kind == GATEWAY && c.gw.outbound
firstPong := c.flags.setIfNotSet(firstPong)
var ri *routeInfo
// When receiving the first PONG, for a route with pooling, we may be
// instructed to start a new route.
if firstPong && c.kind == ROUTER && c.route != nil {
// For a route with pooling, we may be instructed to start a new route.
if c.kind == ROUTER && c.route != nil && c.route.startNewRoute != nil {
ri = c.route.startNewRoute
c.route.startNewRoute = nil
}
@@ -2732,9 +2735,12 @@ func (c *client) updateS2AutoCompressionLevel(co *CompressionOpts, compression *
}
// Will return the parts from the raw wire msg.
// We return the `hdr` as a slice that is capped to the length of the headers
// so that if the caller later tries to append to the returned header slice it
// does not affect the message content.
func (c *client) msgParts(data []byte) (hdr []byte, msg []byte) {
if c != nil && c.pa.hdr > 0 {
return data[:c.pa.hdr], data[c.pa.hdr:]
return data[:c.pa.hdr:c.pa.hdr], data[c.pa.hdr:]
}
return nil, data
}
@@ -2803,9 +2809,10 @@ func (c *client) processHeaderPub(arg, remaining []byte) error {
// look for the tracing header and if found, we will generate a
// trace event with the max payload ingress error.
// Do this only for CLIENT connections.
if c.kind == CLIENT && len(remaining) > 0 {
if td := getHeader(MsgTraceDest, remaining); len(td) > 0 {
c.initAndSendIngressErrEvent(remaining, string(td), ErrMaxPayload)
if c.kind == CLIENT && c.pa.hdr > 0 && len(remaining) > 0 {
hdr := remaining[:min(len(remaining), c.pa.hdr)]
if td, ok := c.allowedMsgTraceDest(hdr, false); ok && td != _EMPTY_ {
c.initAndSendIngressErrEvent(hdr, td, ErrMaxPayload)
}
}
c.maxPayloadViolation(c.pa.size, maxPayload)
@@ -3022,7 +3029,7 @@ func (c *client) processSubEx(subject, queue, bsid []byte, cb msgHandler, noForw
return sub, nil
}
if err := c.addShadowSubscriptions(acc, sub, true); err != nil {
if err := c.addShadowSubscriptions(acc, sub); err != nil {
c.Errorf(err.Error())
}
@@ -3052,10 +3059,7 @@ type ime struct {
// If the client's account has stream imports and there are matches for this
// subscription's subject, then add shadow subscriptions in the other accounts
// that export this subject.
//
// enact=false allows MQTT clients to get the list of shadow subscriptions
// without enacting them, in order to first obtain matching "retained" messages.
func (c *client) addShadowSubscriptions(acc *Account, sub *subscription, enact bool) error {
func (c *client) addShadowSubscriptions(acc *Account, sub *subscription) error {
if acc == nil {
return ErrMissingAccount
}
@@ -3158,7 +3162,7 @@ func (c *client) addShadowSubscriptions(acc *Account, sub *subscription, enact b
for i := 0; i < len(ims); i++ {
ime := &ims[i]
// We will create a shadow subscription.
nsub, err := c.addShadowSub(sub, ime, enact)
nsub, err := c.addShadowSub(sub, ime)
if err != nil {
return err
}
@@ -3175,7 +3179,7 @@ func (c *client) addShadowSubscriptions(acc *Account, sub *subscription, enact b
}
// Add in the shadow subscription.
func (c *client) addShadowSub(sub *subscription, ime *ime, enact bool) (*subscription, error) {
func (c *client) addShadowSub(sub *subscription, ime *ime) (*subscription, error) {
c.mu.Lock()
nsub := *sub // copy
c.mu.Unlock()
@@ -3203,10 +3207,6 @@ func (c *client) addShadowSub(sub *subscription, ime *ime, enact bool) (*subscri
}
// Else use original subject
if !enact {
return &nsub, nil
}
c.Debugf("Creating import subscription on %q from account %q", nsub.subject, im.acc.Name)
if err := im.acc.sl.Insert(&nsub); err != nil {
@@ -3237,7 +3237,7 @@ func (c *client) canSubscribe(subject string, optQueue ...string) bool {
return true
}
allowed := true
allowed, checkAllow := true, true
// Optional queue group.
var queue string
@@ -3245,8 +3245,16 @@ func (c *client) canSubscribe(subject string, optQueue ...string) bool {
queue = optQueue[0]
}
// For CLIENT connections that are MQTT we will implicitly allow anything that starts with
// the "$MQTT.sub." or "$MQTT.deliver.pubrel." prefix. For other types of connections, we
// will implicitly allow anything that starts with the full "$MQTT." prefix. However,
// we don't just return here, we skip the check for "allow" but will check "deny".
if (c.isMqtt() && (strings.HasPrefix(subject, mqttSubPrefix) || strings.HasPrefix(subject, mqttPubRelDeliverySubjectPrefix))) ||
(c.kind != CLIENT && strings.HasPrefix(subject, mqttPrefix)) {
checkAllow = false
}
// Check allow list. If no allow list that means all are allowed. Deny can overrule.
if c.perms.sub.allow != nil {
if checkAllow && c.perms.sub.allow != nil {
r := c.perms.sub.allow.Match(subject)
allowed = len(r.psubs) > 0
if queue != _EMPTY_ && len(r.qsubs) > 0 {
@@ -3338,7 +3346,7 @@ func (c *client) unsubscribe(acc *Account, sub *subscription, force, remove bool
sub.shadow = nil
if len(shadowSubs) > 0 {
isSpokeLeaf = c.isSpokeLeafNode()
updateRoute = !isSpokeLeaf && (c.kind == CLIENT || c.kind == SYSTEM || c.kind == LEAF) && c.srv != nil
updateRoute = !isSpokeLeaf && (c.kind == CLIENT || c.kind == SYSTEM || c.kind == LEAF || c.kind == JETSTREAM) && c.srv != nil
}
sub.close()
c.mu.Unlock()
@@ -4052,6 +4060,41 @@ func (c *client) pubAllowed(subject string) bool {
return c.pubAllowedFullCheck(subject, true, false)
}
// allowedMsgTraceDest returns the trace destination if present and authorized.
// It only considers static publish permissions and does not consume dynamic
// reply permissions because the client is not publishing the trace event itself.
func (c *client) allowedMsgTraceDest(hdr []byte, hasLock bool) (string, bool) {
if len(hdr) == 0 {
return _EMPTY_, true
}
td := sliceHeader(MsgTraceDest, hdr)
if len(td) == 0 {
return _EMPTY_, true
}
dest := bytesToString(td)
if c.kind == CLIENT {
if hasGWRoutedReplyPrefix(td) {
return dest, false
}
var acc *Account
var srv *Server
if !hasLock {
c.mu.Lock()
}
acc, srv = c.acc, c.srv
if !hasLock {
c.mu.Unlock()
}
if bytes.HasPrefix(td, clientNRGPrefix) && srv != nil && acc != srv.SystemAccount() {
return dest, false
}
}
if c.perms != nil && (c.perms.pub.allow != nil || c.perms.pub.deny != nil) && !c.pubAllowedFullCheck(dest, false, hasLock) {
return dest, false
}
return dest, true
}
// pubAllowedFullCheck checks on all publish permissioning depending
// on the flag for dynamic reply permissions.
func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bool {
@@ -4063,9 +4106,15 @@ func (c *client) pubAllowedFullCheck(subject string, fullCheck, hasLock bool) bo
if ok {
return v.(bool)
}
allowed := true
allowed, checkAllow := true, true
// For any connections, other than CLIENT, we will implicitly allow anything that
// starts with the "$MQTT." prefix. However, we don't just return here,
// we skip the check for "allow" but will check "deny".
if c.kind != CLIENT && strings.HasPrefix(subject, mqttPrefix) {
checkAllow = false
}
// Cache miss, check allow then deny as needed.
if c.perms.pub.allow != nil {
if checkAllow && c.perms.pub.allow != nil {
np, _ := c.perms.pub.allow.NumInterest(subject)
allowed = np != 0
}
@@ -4181,10 +4230,19 @@ func (c *client) processInboundClientMsg(msg []byte) (bool, bool) {
genidAddr := &acc.sl.genid
// Check pub permissions
if c.perms != nil && (c.perms.pub.allow != nil || c.perms.pub.deny != nil) && !c.pubAllowedFullCheck(string(c.pa.subject), true, true) {
c.mu.Unlock()
c.pubPermissionViolation(c.pa.subject)
return false, true
if c.perms != nil && (c.perms.pub.allow != nil || c.perms.pub.deny != nil) {
if !c.pubAllowedFullCheck(string(c.pa.subject), true, true) {
c.mu.Unlock()
c.pubPermissionViolation(c.pa.subject)
return false, true
}
}
if c.pa.hdr > 0 {
if td, ok := c.allowedMsgTraceDest(msg[:c.pa.hdr], true); !ok {
c.mu.Unlock()
c.pubPermissionViolation(stringToBytes(td))
return false, true
}
}
c.mu.Unlock()
@@ -4384,28 +4442,43 @@ func (c *client) setupResponseServiceImport(acc *Account, si *serviceImport, tra
return rsi
}
// Will remove a header if present.
func removeHeaderIfPresent(hdr []byte, key string) []byte {
start := getHeaderKeyIndex(key, hdr)
// key can't be first and we want to check that it is preceded by a '\n'
if start < 1 || hdr[start-1] != '\n' {
// Will remove a status and description from the header if present.
func removeHeaderStatusIfPresent(hdr []byte) []byte {
k := []byte("NATS/1.0")
kl, i := len(k), bytes.IndexByte(hdr, '\r')
if !bytes.HasPrefix(hdr, k) || i <= kl {
return hdr
}
index := start + len(key)
if index >= len(hdr) || hdr[index] != ':' {
return hdr
}
end := bytes.Index(hdr[start:], []byte(_CRLF_))
if end < 0 {
return hdr
}
hdr = append(hdr[:start], hdr[start+end+len(_CRLF_):]...)
if len(hdr) <= len(emptyHdrLine) {
hdr = append(hdr[:kl], hdr[i:]...)
if len(hdr) == len(emptyHdrLine) {
return nil
}
return hdr
}
// Will remove a header if present.
func removeHeaderIfPresent(hdr []byte, key string) []byte {
for {
start := getHeaderKeyIndex(key, hdr)
// key can't be first and we want to check that it is preceded by a '\n'
if start < 1 || hdr[start-1] != '\n' {
return hdr
}
index := start + len(key)
if index >= len(hdr) || hdr[index] != ':' {
return hdr
}
end := bytes.Index(hdr[start:], []byte(_CRLF_))
if end < 0 {
return hdr
}
hdr = append(hdr[:start], hdr[start+end+len(_CRLF_):]...)
if len(hdr) <= len(emptyHdrLine) {
return nil
}
}
}
func removeHeaderIfPrefixPresent(hdr []byte, prefix string) []byte {
var index int
for {
@@ -4560,6 +4633,19 @@ func getHeaderKeyIndex(key string, hdr []byte) int {
}
}
// setHeader will replace the value of the first existing key `key`
// with the given value `val`, or add this new key at the end of
// the headers.
//
// Note: If the key does not exist, or if it exists but the new value
// would make the resulting byte slice larger than the original one,
// a new byte slice is returned and the original is left untouched.
// This is to prevent situations where caller may have a `hdr` and
// `msg` that are the parts of an underlying buffer. Extending the
// `hdr` would otherwise overwrite the `msg` part.
//
// If the new value is smaller, then the original `hdr` byte slice
// is modified.
func setHeader(key, val string, hdr []byte) []byte {
start := getHeaderKeyIndex(key, hdr)
if start >= 0 {
@@ -4574,15 +4660,45 @@ func setHeader(key, val string, hdr []byte) []byte {
return hdr // malformed headers
}
valEnd += valStart
suffix := slices.Clone(hdr[valEnd:])
newHdr := append(hdr[:valStart], val...)
return append(newHdr, suffix...)
// Length of the existing value (before the `\r`)
oldValLen := valEnd - valStart
// This is how many extra bytes we need for the new value.
// If <= 0, it means that we need less and so will reuse the `hdr` buffer.
if extra := len(val) - oldValLen; extra > 0 {
// Check that we don't overflow an "int".
if rem := math.MaxInt - hdrLen; rem < extra {
// We don't grow, and return the existing header.
return hdr
}
// The new size is the old size plus the extra bytes.
newHdrSize := hdrLen + extra
newHdr := make([]byte, newHdrSize)
// Copy the parts from `hdr` and `val` into the new buffer.
n := copy(newHdr, hdr[:valStart])
n += copy(newHdr[n:], val)
copy(newHdr[n:], hdr[valEnd:])
return newHdr
}
// We can write in place since it fits in the existing `hdr` buffer.
n := copy(hdr[valStart:], val)
n += copy(hdr[valStart+n:], hdr[valEnd:])
hdr = hdr[:valStart+n]
return hdr
}
if len(hdr) > 0 && bytes.HasSuffix(hdr, []byte("\r\n")) {
hdr = hdr[:len(hdr)-2]
val += "\r\n"
}
return fmt.Appendf(hdr, "%s: %s\r\n", key, val)
// Create the new buffer based on length of existing one and
// length of the new "<key>: <value>\r\n". Protect against "int" overflow.
newSize := uint64(len(hdr)) + uint64(len(key)) + 1 + 1 + uint64(len(val)) + 2
if newSize > uint64(math.MaxInt) {
// We don't grow, and return the existing header.
return hdr
}
newHdr := make([]byte, 0, int(newSize))
newHdr = append(newHdr, hdr...)
return fmt.Appendf(newHdr, "%s: %s\r\n", key, val)
}
// For bytes.HasPrefix below.
@@ -4697,16 +4813,33 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
if !isResponse {
isSysImport := siAcc == c.srv.SystemAccount()
var ci *ClientInfo
if hadPrevSi && c.pa.hdr >= 0 {
var cis ClientInfo
if err := json.Unmarshal(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr]), &cis); err == nil {
ci = &cis
var cis *ClientInfo
if c.pa.hdr >= 0 {
var hci ClientInfo
if err := json.Unmarshal(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr]), &hci); err == nil {
cis = &hci
}
}
if c.kind == LEAF && c.pa.hdr >= 0 && len(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])) > 0 {
// Leaf nodes may forward a Nats-Request-Info from a remote domain,
// but the local server must replace it with the identity of the
// authenticated leaf connection instead of trusting forwarded values.
ci = c.getClientInfo(share)
if hadPrevSi {
ci.Service = acc.Name
// Check if we are moving into a share details account from a non-shared
// and add in server and cluster details.
if !share && (si.share || isSysImport) {
c.addServerAndClusterInfo(ci)
}
} else if !share && isSysImport {
c.addServerAndClusterInfo(ci)
}
} else if hadPrevSi && cis != nil {
ci = cis
ci.Service = acc.Name
// Check if we are moving into a share details account from a non-shared
// and add in server and cluster details.
if !share && (si.share || isSysImport) {
c.addServerAndClusterInfo(ci)
}
} else if c.kind != LEAF || c.pa.hdr < 0 || len(sliceHeader(ClientInfoHdr, msg[:c.pa.hdr])) == 0 {
ci = c.getClientInfo(share)
@@ -4714,12 +4847,6 @@ func (c *client) processServiceImport(si *serviceImport, acc *Account, msg []byt
if !share && isSysImport {
c.addServerAndClusterInfo(ci)
}
} else if c.kind == LEAF && (si.share || isSysImport) {
// We have a leaf header here for ci, augment as above.
ci = c.getClientInfo(si.share)
if !si.share && isSysImport {
c.addServerAndClusterInfo(ci)
}
}
// Set clientInfo if present.
if ci != nil {
@@ -5341,8 +5468,10 @@ sendToRoutesOrLeafs:
// If we do have a deliver subject we need to do something with it.
// Again this is when JetStream (but possibly others) wants the system
// to rewrite the delivered subject. The way we will do that is place it
// at the end of the reply subject if it exists.
if len(deliver) > 0 && len(reply) > 0 {
// at the end of the reply subject if it exists. But only if this wasn't
// already performed, otherwise we'd end up with a duplicate '@' suffix
// resulting in a protocol error.
if len(deliver) > 0 && len(reply) > 0 && !remapped {
reply = append(reply, '@')
reply = append(reply, deliver...)
}
@@ -5501,6 +5630,9 @@ func (c *client) processPingTimer() {
if c.kind == ROUTER && opts.Cluster.PingInterval > 0 {
pingInterval = opts.Cluster.PingInterval
}
if c.isWebsocket() && opts.Websocket.PingInterval > 0 {
pingInterval = opts.Websocket.PingInterval
}
pingInterval = adjustPingInterval(c.kind, pingInterval)
now := time.Now()
needRTT := c.rtt == 0 || now.Sub(c.rttStart) > DEFAULT_RTT_MEASUREMENT_INTERVAL
@@ -5583,6 +5715,9 @@ func (c *client) setPingTimer() {
if c.kind == ROUTER && opts.Cluster.PingInterval > 0 {
d = opts.Cluster.PingInterval
}
if c.isWebsocket() && opts.Websocket.PingInterval > 0 {
d = opts.Websocket.PingInterval
}
d = adjustPingInterval(c.kind, d)
c.ping.tmr = time.AfterFunc(d, c.processPingTimer)
}
@@ -5788,7 +5923,7 @@ func (c *client) processSubsOnConfigReload(awcsti map[string]struct{}) {
oldShadows := sub.shadow
sub.shadow = nil
c.mu.Unlock()
c.addShadowSubscriptions(acc, sub, true)
c.addShadowSubscriptions(acc, sub)
for _, nsub := range oldShadows {
nsub.im.acc.sl.Remove(nsub)
}
@@ -6617,6 +6752,9 @@ func (c *client) setFirstPingTimer() {
if c.kind == ROUTER && opts.Cluster.PingInterval > 0 {
d = opts.Cluster.PingInterval
}
if c.isWebsocket() && opts.Websocket.PingInterval > 0 {
d = opts.Websocket.PingInterval
}
if !opts.DisableShortFirstPing {
if c.kind != CLIENT {
if d > firstPingInterval {
+29 -22
View File
@@ -131,19 +131,22 @@ func detectProxyProtoVersion(conn net.Conn) (version int, header []byte, err err
// readProxyProtoV1Header parses PROXY protocol v1 text format.
// Expects the "PROXY " prefix (6 bytes) to have already been consumed.
func readProxyProtoV1Header(conn net.Conn) (*proxyProtoAddr, error) {
// Returns any bytes that were read past the trailing CRLF so the caller can
// replay them into the next protocol layer.
func readProxyProtoV1Header(conn net.Conn) (*proxyProtoAddr, []byte, error) {
// Read rest of line (max 107 bytes total, already read 6)
maxRemaining := proxyProtoV1MaxLineLen - 6
// Read up to maxRemaining bytes at once (more efficient than byte-by-byte)
buf := make([]byte, maxRemaining)
var line []byte
var remaining []byte
for len(line) < maxRemaining {
// Read available data
n, err := conn.Read(buf[len(line):])
if err != nil {
return nil, fmt.Errorf("failed to read v1 line: %w", err)
return nil, nil, fmt.Errorf("failed to read v1 line: %w", err)
}
line = buf[:len(line)+n]
@@ -151,7 +154,8 @@ func readProxyProtoV1Header(conn net.Conn) (*proxyProtoAddr, error) {
// Look for CRLF in what we've read so far
for i := 0; i < len(line)-1; i++ {
if line[i] == '\r' && line[i+1] == '\n' {
// Found CRLF - extract just the line portion
// Found CRLF - keep any over-read bytes for the client parser.
remaining = append(remaining, line[i+2:]...)
line = line[:i]
goto foundCRLF
}
@@ -159,7 +163,7 @@ func readProxyProtoV1Header(conn net.Conn) (*proxyProtoAddr, error) {
}
// Exceeded max length without finding CRLF
return nil, fmt.Errorf("%w: v1 line too long", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: v1 line too long", errProxyProtoInvalid)
foundCRLF:
// Get parts from the protocol
@@ -167,17 +171,17 @@ foundCRLF:
// Validate format
if len(parts) < 1 {
return nil, fmt.Errorf("%w: invalid v1 format", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: invalid v1 format", errProxyProtoInvalid)
}
// Handle UNKNOWN (health check, like v2 LOCAL)
if parts[0] == proxyProtoV1Unknown {
return nil, nil
return nil, remaining, nil
}
// Must have exactly 5 parts: protocol, src-ip, dst-ip, src-port, dst-port
if len(parts) != 5 {
return nil, fmt.Errorf("%w: invalid v1 format", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: invalid v1 format", errProxyProtoInvalid)
}
protocol := parts[0]
@@ -185,29 +189,29 @@ foundCRLF:
dstIP := net.ParseIP(parts[2])
if srcIP == nil || dstIP == nil {
return nil, fmt.Errorf("%w: invalid address", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: invalid address", errProxyProtoInvalid)
}
// Parse ports
srcPort, err := strconv.ParseUint(parts[3], 10, 16)
if err != nil {
return nil, fmt.Errorf("invalid source port: %w", err)
return nil, nil, fmt.Errorf("invalid source port: %w", err)
}
dstPort, err := strconv.ParseUint(parts[4], 10, 16)
if err != nil {
return nil, fmt.Errorf("invalid dest port: %w", err)
return nil, nil, fmt.Errorf("invalid dest port: %w", err)
}
// Validate protocol matches IP version
if protocol == proxyProtoV1TCP4 && srcIP.To4() == nil {
return nil, fmt.Errorf("%w: TCP4 with IPv6 address", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: TCP4 with IPv6 address", errProxyProtoInvalid)
}
if protocol == proxyProtoV1TCP6 && srcIP.To4() != nil {
return nil, fmt.Errorf("%w: TCP6 with IPv4 address", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: TCP6 with IPv4 address", errProxyProtoInvalid)
}
if protocol != proxyProtoV1TCP4 && protocol != proxyProtoV1TCP6 {
return nil, fmt.Errorf("%w: invalid protocol %s", errProxyProtoInvalid, protocol)
return nil, nil, fmt.Errorf("%w: invalid protocol %s", errProxyProtoInvalid, protocol)
}
return &proxyProtoAddr{
@@ -215,25 +219,27 @@ foundCRLF:
srcPort: uint16(srcPort),
dstIP: dstIP,
dstPort: uint16(dstPort),
}, nil
}, remaining, nil
}
// readProxyProtoHeader reads and parses PROXY protocol (v1 or v2) from the connection.
// Automatically detects version and routes to appropriate parser.
// If the command is LOCAL/UNKNOWN (health check), it returns nil for addr and no error.
// If the command is PROXY, it returns the parsed address information.
// It also returns any bytes that were read past the v1 header terminator so the
// caller can replay them into the normal client parser.
// The connection must be fresh (no data read yet).
func readProxyProtoHeader(conn net.Conn) (*proxyProtoAddr, error) {
func readProxyProtoHeader(conn net.Conn) (*proxyProtoAddr, []byte, error) {
// Set read deadline to prevent hanging on slow/malicious clients
if err := conn.SetReadDeadline(time.Now().Add(proxyProtoReadTimeout)); err != nil {
return nil, err
return nil, nil, err
}
defer conn.SetReadDeadline(time.Time{})
// Detect version
version, firstBytes, err := detectProxyProtoVersion(conn)
if err != nil {
return nil, err
return nil, nil, err
}
switch version {
@@ -244,25 +250,26 @@ func readProxyProtoHeader(conn net.Conn) (*proxyProtoAddr, error) {
// Read rest of v2 signature (bytes 6-11, total 6 more bytes)
remaining := make([]byte, 6)
if _, err := io.ReadFull(conn, remaining); err != nil {
return nil, fmt.Errorf("failed to read v2 signature: %w", err)
return nil, nil, fmt.Errorf("failed to read v2 signature: %w", err)
}
// Verify full signature
fullSig := string(firstBytes) + string(remaining)
if fullSig != proxyProtoV2Sig {
return nil, fmt.Errorf("%w: invalid signature", errProxyProtoInvalid)
return nil, nil, fmt.Errorf("%w: invalid signature", errProxyProtoInvalid)
}
// Read rest of header: ver/cmd, fam/proto, addr-len (4 bytes)
header := make([]byte, 4)
if _, err := io.ReadFull(conn, header); err != nil {
return nil, fmt.Errorf("failed to read v2 header: %w", err)
return nil, nil, fmt.Errorf("failed to read v2 header: %w", err)
}
// Continue with parsing
return parseProxyProtoV2Header(conn, header)
addr, err := parseProxyProtoV2Header(conn, header)
return addr, nil, err
default:
return nil, fmt.Errorf("unsupported PROXY protocol version: %d", version)
return nil, nil, fmt.Errorf("unsupported PROXY protocol version: %d", version)
}
}
+1 -1
View File
@@ -66,7 +66,7 @@ func init() {
const (
// VERSION is the current version for the server.
VERSION = "2.12.2"
VERSION = "2.12.6"
// PROTO is the currently supported protocol.
// 0 was the original
+187 -123
View File
@@ -1,4 +1,4 @@
// Copyright 2019-2025 The NATS Authors
// Copyright 2019-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -42,7 +42,6 @@ import (
const (
JSPullRequestPendingMsgs = "Nats-Pending-Messages"
JSPullRequestPendingBytes = "Nats-Pending-Bytes"
JSPullRequestWrongPinID = "NATS/1.0 423 Nats-Wrong-Pin-Id\r\n\r\n"
JSPullRequestNatsPinId = "Nats-Pin-Id"
)
@@ -512,7 +511,7 @@ type consumer struct {
// Details described in ADR-42.
// currentPinId is the current nuid for the pinned consumer.
// If the Consumer is running in `PriorityPinnedClient` mode, server will
// If the Consumer is running in `PriorityPinnedClient` mode, server will
// pick up a new nuid and assign it to first pending pull request.
currentPinId string
/// pinnedTtl is the remaining time before the current PinId expires.
@@ -825,7 +824,7 @@ func checkConsumerCfg(
return NewJSStreamInvalidConfigError(ErrBadSubject)
}
for inner, ssubject := range subjectFilters {
if inner != outer && SubjectsCollide(subject, ssubject) {
if inner != outer && subjectIsSubsetMatch(subject, ssubject) {
return NewJSConsumerOverlappingSubjectFiltersError()
}
}
@@ -963,7 +962,7 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
}
mset.mu.RLock()
s, jsa, cfg, acc := mset.srv, mset.jsa, mset.cfg, mset.acc
s, js, jsa, cfg, acc := mset.srv, mset.js, mset.jsa, mset.cfg, mset.acc
mset.mu.RUnlock()
// If we do not have the consumer currently assigned to us in cluster mode we will proceed but warn.
@@ -1030,11 +1029,11 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
}
if cName != _EMPTY_ {
if eo, ok := mset.consumers[cName]; ok {
mset.mu.Unlock()
if action == ActionCreate {
ocfg := eo.config()
copyConsumerMetadata(config, &ocfg)
if !reflect.DeepEqual(config, &ocfg) {
mset.mu.Unlock()
return nil, NewJSConsumerAlreadyExistsError()
}
}
@@ -1042,9 +1041,11 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
if cfg.Retention == WorkQueuePolicy {
subjects := gatherSubjectFilters(config.FilterSubject, config.FilterSubjects)
if !mset.partitionUnique(cName, subjects) {
mset.mu.Unlock()
return nil, NewJSConsumerWQConsumerNotUniqueError()
}
}
mset.mu.Unlock()
err := eo.updateConfig(config)
if err == nil {
return eo, nil
@@ -1057,17 +1058,22 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
return nil, NewJSConsumerDoesNotExistError()
}
// Check for any limits, if the config for the consumer sets a limit we check against that
// but if not we use the value from account limits, if account limits is more restrictive
// than stream config we prefer the account limits to handle cases where account limits are
// updated during the lifecycle of the stream
maxc := cfg.MaxConsumers
if maxc <= 0 || (selectedLimits.MaxConsumers > 0 && selectedLimits.MaxConsumers < maxc) {
maxc = selectedLimits.MaxConsumers
}
if maxc > 0 && mset.numPublicConsumers() >= maxc {
mset.mu.Unlock()
return nil, NewJSMaximumConsumersLimitError()
// If we're clustered we've already done this check, only do this if we're a standalone server.
// But if we're standalone, only enforce if we're not recovering, since the MaxConsumers could've
// been updated while we already had more consumers on disk.
if !s.JetStreamIsClustered() && s.standAloneMode() && !isRecovering {
// Check for any limits, if the config for the consumer sets a limit we check against that
// but if not we use the value from account limits, if account limits is more restrictive
// than stream config we prefer the account limits to handle cases where account limits are
// updated during the lifecycle of the stream
maxc := cfg.MaxConsumers
if maxc <= 0 || (selectedLimits.MaxConsumers > 0 && selectedLimits.MaxConsumers < maxc) {
maxc = selectedLimits.MaxConsumers
}
if maxc > 0 && mset.numPublicConsumers() >= maxc {
mset.mu.Unlock()
return nil, NewJSMaximumConsumersLimitError()
}
}
// Check on stream type conflicts with WorkQueues.
@@ -1134,6 +1140,13 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
created: time.Now().UTC(),
}
// Add created timestamp used for the store, must match that of the consumer assignment if it exists.
if ca != nil {
js.mu.RLock()
o.created = ca.Created
js.mu.RUnlock()
}
// Bind internal client to the user account.
o.client.registerWithAccount(a)
// Bind to the system account.
@@ -1186,7 +1199,7 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
// Setup our storage if not a direct consumer.
if !config.Direct {
store, err := mset.store.ConsumerStore(o.name, config)
store, err := mset.store.ConsumerStore(o.name, o.created, config)
if err != nil {
mset.mu.Unlock()
o.deleteWithoutAdvisory()
@@ -1206,14 +1219,13 @@ func (mset *stream) addConsumerWithAssignment(config *ConsumerConfig, oname stri
// If we have multiple filter subjects, create a sublist which we will use
// in calling store.LoadNextMsgMulti.
if len(o.cfg.FilterSubjects) > 0 {
o.filters = gsl.NewSublist[struct{}]()
for _, filter := range o.cfg.FilterSubjects {
o.filters.Insert(filter, struct{}{})
}
} else {
// Make sure this is nil otherwise.
if len(o.subjf) <= 1 {
o.filters = nil
} else {
o.filters = gsl.NewSublist[struct{}]()
for _, filter := range o.subjf {
o.filters.Insert(filter.subject, struct{}{})
}
}
if o.store != nil && o.store.HasState() {
@@ -1393,8 +1405,12 @@ func (o *consumer) monitorQuitC() <-chan struct{} {
if o == nil {
return nil
}
o.mu.RLock()
defer o.mu.RUnlock()
o.mu.Lock()
defer o.mu.Unlock()
// Recreate if a prior monitor routine was stopped.
if o.mqch == nil {
o.mqch = make(chan struct{})
}
return o.mqch
}
@@ -1535,7 +1551,6 @@ func (o *consumer) setLeader(isLeader bool) {
if o.cfg.AckPolicy != AckNone {
if o.ackSub, err = o.subscribeInternal(o.ackSubj, o.pushAck); err != nil {
o.mu.Unlock()
o.deleteWithoutAdvisory()
return
}
}
@@ -1544,7 +1559,6 @@ func (o *consumer) setLeader(isLeader bool) {
// Will error if wrong mode to provide feedback to users.
if o.reqSub, err = o.subscribeInternal(o.nextMsgSubj, o.processNextMsgReq); err != nil {
o.mu.Unlock()
o.deleteWithoutAdvisory()
return
}
@@ -1554,7 +1568,6 @@ func (o *consumer) setLeader(isLeader bool) {
fcsubj := fmt.Sprintf(jsFlowControl, stream, o.name)
if o.fcSub, err = o.subscribeInternal(fcsubj, o.processFlowControl); err != nil {
o.mu.Unlock()
o.deleteWithoutAdvisory()
return
}
}
@@ -1680,6 +1693,7 @@ func (o *consumer) setLeader(isLeader bool) {
} else if o.srv.gateway.enabled {
stopAndClearTimer(&o.gwdtmr)
}
o.unassignPinId()
// If we were the leader make sure to drain queued up acks.
if wasLeader {
o.ackMsgs.drain()
@@ -2039,6 +2053,7 @@ func (o *consumer) deleteNotActive() {
if o.srv != nil {
qch = o.srv.quitCh
}
oqch := o.qch
o.mu.Unlock()
if js != nil {
cqch = js.clusterQuitC()
@@ -2087,6 +2102,9 @@ func (o *consumer) deleteNotActive() {
return
case <-cqch:
return
case <-oqch:
// The consumer has stopped already, likely by an earlier delete proposal being applied.
return
}
js.mu.RLock()
if js.shuttingDown {
@@ -2394,7 +2412,8 @@ func (o *consumer) updateConfig(cfg *ConsumerConfig) error {
// Check for Subject Filters update.
newSubjects := gatherSubjectFilters(cfg.FilterSubject, cfg.FilterSubjects)
if !subjectSliceEqual(newSubjects, o.subjf.subjects()) {
updatedFilters := !subjectSliceEqual(newSubjects, o.subjf.subjects())
if updatedFilters {
newSubjf := make(subjectFilters, 0, len(newSubjects))
for _, newFilter := range newSubjects {
fs := &subjectFilter{
@@ -2433,15 +2452,17 @@ func (o *consumer) updateConfig(cfg *ConsumerConfig) error {
// Allowed but considered no-op, [Description, SampleFrequency, MaxWaiting, HeadersOnly]
o.cfg = *cfg
// Cleanup messages that lost interest.
if o.retention == InterestPolicy {
o.mu.Unlock()
o.cleanupNoInterestMessages(o.mset, false)
o.mu.Lock()
}
if updatedFilters {
// Cleanup messages that lost interest.
if o.retention == InterestPolicy {
o.mu.Unlock()
o.cleanupNoInterestMessages(o.mset, false)
o.mu.Lock()
}
// Re-calculate num pending on update.
o.streamNumPending()
// Re-calculate num pending on update.
o.streamNumPending()
}
return nil
}
@@ -2831,14 +2852,10 @@ func (o *consumer) releaseAnyPendingRequests(isAssigned bool) {
if o.mset == nil || o.outq == nil || o.waiting.len() == 0 {
return
}
var hdr []byte
if !isAssigned {
hdr = []byte("NATS/1.0 409 Consumer Deleted\r\n\r\n")
}
wq := o.waiting
for wr := wq.head; wr != nil; {
if hdr != nil {
if !isAssigned {
hdr := []byte("NATS/1.0 409 Consumer Deleted\r\n\r\n")
o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))
}
next := wr.next
@@ -3042,6 +3059,11 @@ func (o *consumer) setStoreState(state *ConsumerState) error {
err := o.store.Update(state)
if err == nil {
o.applyState(state)
} else if err == ErrStoreOldUpdate {
// Our store already has a newer state, which is normal during recovery
// when the consumer was loaded from disk before the meta snapshot state
// was applied.
return nil
}
return err
}
@@ -3133,6 +3155,12 @@ func (o *consumer) infoWithSnapAndReply(snap bool, reply string) *ConsumerInfo {
})
}
np, err := o.checkNumPending()
if err != nil {
o.mu.Unlock()
return nil
}
cfg := o.cfg
info := &ConsumerInfo{
Stream: o.stream,
@@ -3149,7 +3177,7 @@ func (o *consumer) infoWithSnapAndReply(snap bool, reply string) *ConsumerInfo {
},
NumAckPending: len(o.pending),
NumRedelivered: len(o.rdc),
NumPending: o.checkNumPending(),
NumPending: np,
PushBound: o.isPushMode() && o.active,
TimeStamp: time.Now().UTC(),
PriorityGroups: priorityGroups,
@@ -3900,7 +3928,12 @@ func (o *consumer) setPinnedTimer(priorityGroup string) {
} else {
o.pinnedTtl = time.AfterFunc(o.cfg.PinnedTTL, func() {
o.mu.Lock()
o.currentPinId = _EMPTY_
// Skip if already unset.
if o.currentPinId == _EMPTY_ {
o.mu.Unlock()
return
}
o.unassignPinId()
o.sendUnpinnedAdvisoryLocked(priorityGroup, "timeout")
o.mu.Unlock()
o.signalNewMessages()
@@ -3908,6 +3941,28 @@ func (o *consumer) setPinnedTimer(priorityGroup string) {
}
}
// Lock should be held.
func (o *consumer) assignNewPinId(wr *waitingRequest) {
if wr.priorityGroup == nil || wr.priorityGroup.Group == _EMPTY_ {
return
}
o.currentPinId = nuid.Next()
o.pinnedTS = time.Now().UTC()
wr.priorityGroup.Id = o.currentPinId
o.setPinnedTimer(wr.priorityGroup.Group)
o.sendPinnedAdvisoryLocked(wr.priorityGroup.Group)
}
// Lock should be held.
func (o *consumer) unassignPinId() {
o.currentPinId = _EMPTY_
o.pinnedTS = time.Time{}
if o.pinnedTtl != nil {
o.pinnedTtl.Stop()
o.pinnedTtl = nil
}
}
// Return next waiting request. This will check for expirations but not noWait or interest.
// That will be handled by processWaiting.
// Lock should be held.
@@ -3918,11 +3973,6 @@ func (o *consumer) nextWaiting(sz int) *waitingRequest {
// Check if server needs to assign a new pin id.
needNewPin := o.currentPinId == _EMPTY_ && o.cfg.PriorityPolicy == PriorityPinnedClient
// As long as we support only one priority group, we can capture that group here and reuse it.
var priorityGroup string
if len(o.cfg.PriorityGroups) > 0 {
priorityGroup = o.cfg.PriorityGroups[0]
}
numCycled := 0
for wr := o.waiting.peek(); !o.waiting.isEmpty(); wr = o.waiting.peek() {
@@ -3956,15 +4006,12 @@ func (o *consumer) nextWaiting(sz int) *waitingRequest {
if wr.expires.IsZero() || time.Now().Before(wr.expires) {
if needNewPin {
if wr.priorityGroup.Id == _EMPTY_ {
o.currentPinId = nuid.Next()
o.pinnedTS = time.Now().UTC()
wr.priorityGroup.Id = o.currentPinId
o.setPinnedTimer(priorityGroup)
o.assignNewPinId(wr)
} else {
// There is pin id set, but not a matching one. Send a notification to the client and remove the request.
// Probably this is the old pin id.
o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, []byte(JSPullRequestWrongPinID), nil, nil, 0))
hdr := fmt.Appendf(nil, "NATS/1.0 423 Nats-Wrong-Pin-Id\r\n%s: %d\r\n%s: %d\r\n\r\n", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)
o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))
o.waiting.removeCurrent()
if o.node != nil {
o.removeClusterPendingRequest(wr.reply)
@@ -3985,7 +4032,8 @@ func (o *consumer) nextWaiting(sz int) *waitingRequest {
continue
} else {
// There is pin id set, but not a matching one. Send a notification to the client and remove the request.
o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, []byte(JSPullRequestWrongPinID), nil, nil, 0))
hdr := fmt.Appendf(nil, "NATS/1.0 423 Nats-Wrong-Pin-Id\r\n%s: %d\r\n%s: %d\r\n\r\n", JSPullRequestPendingMsgs, wr.n, JSPullRequestPendingBytes, wr.b)
o.outq.send(newJSPubMsg(wr.reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))
o.waiting.removeCurrent()
if o.node != nil {
o.removeClusterPendingRequest(wr.reply)
@@ -3997,9 +4045,13 @@ func (o *consumer) nextWaiting(sz int) *waitingRequest {
if o.cfg.PriorityPolicy == PriorityOverflow {
if wr.priorityGroup != nil &&
// If both limits are zero we don't cycle and the request will be fulfilled.
(wr.priorityGroup.MinPending > 0 || wr.priorityGroup.MinAckPending > 0) &&
// We need to check o.npc+1, because before calling nextWaiting, we do o.npc--
(wr.priorityGroup.MinPending > 0 && wr.priorityGroup.MinPending > o.npc+1 ||
wr.priorityGroup.MinAckPending > 0 && wr.priorityGroup.MinAckPending > int64(len(o.pending))) {
// If one OR the other limit is exceeded, we want to fulfill the request.
// This is an inverted check. For clarity, we check the positive condition and negate.
!((wr.priorityGroup.MinPending > 0 && wr.priorityGroup.MinPending <= o.npc+1) ||
(wr.priorityGroup.MinAckPending > 0 && wr.priorityGroup.MinAckPending <= int64(len(o.pending)))) {
o.waiting.cycle()
numCycled++
// We're done cycling through the requests.
@@ -4010,19 +4062,10 @@ func (o *consumer) nextWaiting(sz int) *waitingRequest {
}
}
if wr.acc.sl.HasInterest(wr.interest) {
if needNewPin {
o.sendPinnedAdvisoryLocked(priorityGroup)
}
return o.waiting.popOrPopAndRequeue(o.cfg.PriorityPolicy)
} else if time.Since(wr.received) < defaultGatewayRecentSubExpiration && (o.srv.leafNodeEnabled || o.srv.gateway.enabled) {
if needNewPin {
o.sendPinnedAdvisoryLocked(priorityGroup)
}
return o.waiting.popOrPopAndRequeue(o.cfg.PriorityPolicy)
} else if o.srv.gateway.enabled && o.srv.hasGatewayInterest(wr.acc.Name, wr.interest) {
if needNewPin {
o.sendPinnedAdvisoryLocked(priorityGroup)
}
return o.waiting.popOrPopAndRequeue(o.cfg.PriorityPolicy)
}
} else {
@@ -4180,15 +4223,7 @@ func (o *consumer) processNextMsgRequest(reply string, msg []byte) {
sendErr(400, "Bad Request - Priority Group missing")
return
}
found := false
for _, group := range o.cfg.PriorityGroups {
if group == priorityGroup.Group {
found = true
break
}
}
if !found {
if !slices.Contains(o.cfg.PriorityGroups, priorityGroup.Group) {
sendErr(400, "Bad Request - Invalid Priority Group")
return
}
@@ -4426,6 +4461,8 @@ func (o *consumer) getNextMsg() (*jsPubMsg, uint64, error) {
// scheduled for redelivery, but it has been removed from the stream.
// o.processTerm is called in a goroutine so could run after we get here.
// That will correct the pending state and delivery/ack floors, so just skip here.
pmsg.returnToPool()
pmsg = nil
continue
}
return pmsg, dc, err
@@ -4453,6 +4490,7 @@ func (o *consumer) getNextMsg() (*jsPubMsg, uint64, error) {
sm, err := o.mset.store.LoadMsg(seq, &pmsg.StoreMsg)
if sm == nil || err != nil {
pmsg.returnToPool()
pmsg = nil
}
o.sseq++
return pmsg, 1, err
@@ -4964,6 +5002,7 @@ func (o *consumer) loopAndGatherMsgs(qch chan struct{}) {
o.addToRedeliverQueue(pmsg.seq)
}
pmsg.returnToPool()
pmsg = nil
goto waitForMsgs
}
@@ -4974,6 +5013,7 @@ func (o *consumer) loopAndGatherMsgs(qch chan struct{}) {
select {
case <-qch:
pmsg.returnToPool()
pmsg = nil
return
case <-time.After(delay):
}
@@ -4994,6 +5034,7 @@ func (o *consumer) loopAndGatherMsgs(qch chan struct{}) {
select {
case <-qch:
pmsg.returnToPool()
pmsg = nil
return
case <-time.After(delay):
}
@@ -5097,17 +5138,22 @@ func (o *consumer) setMaxPendingBytes(limit int) {
// The race is a getNextMsg skips a deleted msg, and then the decStreamPending call fires.
// This does some quick sanity checks to see if we should re-calculate num pending.
// Lock should be held.
func (o *consumer) checkNumPending() uint64 {
func (o *consumer) checkNumPending() (uint64, error) {
if o.mset != nil && o.mset.store != nil {
var state StreamState
o.mset.store.FastState(&state)
npc := o.numPending()
if o.sseq > state.LastSeq && npc > 0 || npc > state.Msgs {
// Re-calculate.
o.streamNumPending()
// Make sure we can't report more messages than there are.
// TODO(nat): It's not great that this means consumer info has side effects,
// since we can't know whether anyone will call it or not. The previous num
// pending calculation that this replaces had the same problem though.
if o.sseq > state.LastSeq {
o.npc = 0
} else if npc > 0 {
o.npc = int64(min(npc, state.Msgs, state.LastSeq-o.sseq+1))
}
}
return o.numPending()
return o.numPending(), nil
}
// Lock should be held.
@@ -5134,7 +5180,7 @@ func (o *consumer) checkNumPendingOnEOF() {
}
// Call into streamNumPending after acquiring the consumer lock.
func (o *consumer) streamNumPendingLocked() uint64 {
func (o *consumer) streamNumPendingLocked() (uint64, error) {
o.mu.Lock()
defer o.mu.Unlock()
return o.streamNumPending()
@@ -5143,22 +5189,25 @@ func (o *consumer) streamNumPendingLocked() uint64 {
// Will force a set from the stream store of num pending.
// Depends on delivery policy, for last per subject we calculate differently.
// Lock should be held.
func (o *consumer) streamNumPending() uint64 {
func (o *consumer) streamNumPending() (uint64, error) {
if o.mset == nil || o.mset.store == nil {
o.npc, o.npf = 0, 0
return 0
return 0, nil
}
npc, npf, err := o.calculateNumPending()
if err != nil {
return 0, err
}
npc, npf := o.calculateNumPending()
o.npc, o.npf = int64(npc), npf
return o.numPending()
return o.numPending(), nil
}
// Will calculate num pending but only requires a read lock.
// Depends on delivery policy, for last per subject we calculate differently.
// At least RLock should be held.
func (o *consumer) calculateNumPending() (npc, npf uint64) {
func (o *consumer) calculateNumPending() (npc, npf uint64, err error) {
if o.mset == nil || o.mset.store == nil {
return 0, 0
return 0, 0, nil
}
isLastPerSubject := o.cfg.DeliverPolicy == DeliverLastPerSubject
@@ -5349,6 +5398,15 @@ func (o *consumer) trackPending(sseq, dseq uint64) {
o.pending = make(map[uint64]*Pending)
}
now := time.Now()
if p, ok := o.pending[sseq]; ok {
// Update timestamp but keep original consumer delivery sequence.
// So do not update p.Sequence.
p.Timestamp = now.UnixNano()
} else {
o.pending[sseq] = &Pending{dseq, now.UnixNano()}
}
// We could have a backoff that set a timer higher than what we need for this message.
// In that case, reset to lowest backoff required for a message redelivery.
minDelay := o.ackWait(0)
@@ -5361,18 +5419,10 @@ func (o *consumer) trackPending(sseq, dseq uint64) {
}
minDelay = o.ackWait(o.cfg.BackOff[bi])
}
minDeadline := time.Now().Add(minDelay)
minDeadline := now.Add(minDelay)
if o.ptmr == nil || o.ptmrEnd.After(minDeadline) {
o.resetPtmr(minDelay)
}
if p, ok := o.pending[sseq]; ok {
// Update timestamp but keep original consumer delivery sequence.
// So do not update p.Sequence.
p.Timestamp = time.Now().UnixNano()
} else {
o.pending[sseq] = &Pending{dseq, time.Now().UnixNano()}
}
}
// Credit back a failed delivery.
@@ -6445,6 +6495,10 @@ func (o *consumer) checkStateForInterestStream(ss *StreamState) error {
if asflr&(1<<63) != 0 {
return errAckFloorInvalid
}
dflr := asflr
if len(state.Pending) > 0 && state.Delivered.Stream > dflr {
dflr = state.Delivered.Stream
}
// Check if the underlying stream's last sequence is less than our floor.
// This can happen if the stream has been reset and has not caught up yet.
@@ -6463,7 +6517,7 @@ func (o *consumer) checkStateForInterestStream(ss *StreamState) error {
}
var retryAsflr uint64
for seq = fseq; asflr > 0 && seq <= asflr; seq++ {
for seq = fseq; dflr > 0 && seq <= dflr; seq++ {
if filters != nil {
_, nseq, err = store.LoadNextMsgMulti(filters, seq, &smv)
} else {
@@ -6473,19 +6527,43 @@ func (o *consumer) checkStateForInterestStream(ss *StreamState) error {
if nseq > seq {
seq = nseq
}
// Only ack though if no error and seq <= ack floor.
if err == nil && seq <= asflr {
didRemove := mset.ackMsg(o, seq)
// Removing the message could fail. For example if clustered since we need to propose it.
// Overwrite retry floor (only the first time) to allow us to check next time if the removal was successful.
if didRemove && retryAsflr == 0 {
retryAsflr = seq
if err == nil {
// Only ack though if no error and seq <= ack floor.
if seq <= asflr {
didRemove := mset.ackMsg(o, seq)
// Removing the message could fail. For example if clustered since we need to propose it.
// Overwrite retry floor (only the first time) to allow us to check next time if the removal was successful.
if didRemove && retryAsflr == 0 {
retryAsflr = seq
}
} else if seq <= dflr {
// Store the first entry above our ack floor, so we don't need to look it up again on retryAsflr=0.
if retryAsflr == 0 {
retryAsflr = seq
}
// If we have pending, we will need to walk through to delivered in case we missed any of those acks as well.
if _, ok := state.Pending[seq]; !ok {
// The filters are already taken into account,
mset.ackMsg(o, seq)
}
}
} else if err == ErrStoreEOF {
break
}
}
// If retry floor was not overwritten, set to ack floor+1, we don't need to account for any retries below it.
// However, our ack floor may be lower than the next message we can receive, so we correct it upward if needed.
if retryAsflr == 0 {
retryAsflr = asflr + 1
if filters != nil {
_, nseq, err = store.LoadNextMsgMulti(filters, asflr+1, &smv)
} else {
_, nseq, err = store.LoadNextMsg(filter, wc, asflr+1, &smv)
}
if err == nil {
retryAsflr = max(asflr+1, nseq)
} else if err == ErrStoreEOF {
retryAsflr = ss.LastSeq + 1
}
}
o.mu.Lock()
@@ -6494,21 +6572,7 @@ func (o *consumer) checkStateForInterestStream(ss *StreamState) error {
if retryAsflr > o.chkflr {
o.chkflr = retryAsflr
}
// See if we need to process this update if our parent stream is not a limits policy stream.
state, _ = o.store.State()
o.mu.Unlock()
// If we have pending, we will need to walk through to delivered in case we missed any of those acks as well.
if state != nil && len(state.Pending) > 0 && state.AckFloor.Stream > 0 {
for seq := state.AckFloor.Stream + 1; seq <= state.Delivered.Stream; seq++ {
if _, ok := state.Pending[seq]; !ok {
// Want to call needAck since it is filter aware.
if o.needAck(seq, _EMPTY_) {
mset.ackMsg(o, seq)
}
}
}
}
return nil
}
+6
View File
@@ -215,6 +215,9 @@ var (
// ErrMinimumVersionRequired is returned when a connection is not at the minimum version required.
ErrMinimumVersionRequired = errors.New("minimum version required")
// ErrLeafNodeMinVersionRejected is the leafnode protocol error prefix used
// when rejecting a remote due to leafnodes.min_version.
ErrLeafNodeMinVersionRejected = errors.New("connection rejected since minimum version required is")
// ErrInvalidMappingDestination is used for all subject mapping destination errors
ErrInvalidMappingDestination = errors.New("invalid mapping destination")
@@ -251,6 +254,9 @@ type mappingDestinationErr struct {
}
func (e *mappingDestinationErr) Error() string {
if e.token == _EMPTY_ {
return e.err.Error()
}
return fmt.Sprintf("%s in %s", e.err, e.token)
}
+10
View File
@@ -1998,5 +1998,15 @@
"help": "",
"url": "",
"deprecates": ""
},
{
"constant": "JSClusterServerMemberChangeInflightErr",
"code": 400,
"error_code": 10202,
"description": "cluster member change is in progress",
"comment": "",
"help": "",
"url": "",
"deprecates": ""
}
]
+15
View File
@@ -1500,6 +1500,8 @@ func (s *Server) initEventTracking() {
type UserInfo struct {
UserID string `json:"user"`
Account string `json:"account"`
AccountName string `json:"account_name,omitempty"`
UserName string `json:"user_name,omitempty"`
Permissions *Permissions `json:"permissions,omitempty"`
Expires time.Duration `json:"expires,omitempty"`
}
@@ -1519,9 +1521,22 @@ func (s *Server) userInfoReq(sub *subscription, c *client, _ *Account, subject,
return
}
// Look up the requester's account directly from ci.Account rather than
// using the acc returned by getRequestInfo, which may resolve to the
// service account (ci.Service) when the request arrives via a chained
// service import.
var accountName string
if ci.Account != _EMPTY_ {
if reqAcc, _ := s.LookupAccount(ci.Account); reqAcc != nil {
accountName = reqAcc.getNameTag()
}
}
response.Data = &UserInfo{
UserID: ci.User,
Account: ci.Account,
AccountName: accountName,
UserName: ci.NameTag,
Permissions: c.publicPermissions(),
Expires: c.claimExpiration(),
}
File diff suppressed because it is too large Load Diff
+47 -84
View File
@@ -17,9 +17,6 @@ import (
"errors"
"strings"
"sync"
"unsafe"
"github.com/nats-io/nats-server/v2/server/stree"
)
// Sublist is a routing mechanism to handle subject distribution and
@@ -51,6 +48,11 @@ var (
// unnecessary allocations.
type SimpleSublist = GenericSublist[struct{}]
// NewSimpleSublist will create a simple sublist.
func NewSimpleSublist() *SimpleSublist {
return &GenericSublist[struct{}]{root: newLevel[struct{}]()}
}
// A GenericSublist stores and efficiently retrieves subscriptions.
type GenericSublist[T comparable] struct {
sync.RWMutex
@@ -246,7 +248,9 @@ func matchLevelForAny[T comparable](l *level[T], toks []string, np *int) bool {
if np != nil {
*np += len(n.subs)
}
return len(n.subs) > 0
if len(n.subs) > 0 {
return true
}
}
if pwc != nil {
if np != nil {
@@ -365,6 +369,36 @@ func (s *GenericSublist[T]) Remove(subject string, value T) error {
return s.remove(subject, value, true)
}
// HasInterestStartingIn is a helper for subject tree intersection.
func (s *GenericSublist[T]) HasInterestStartingIn(subj string) bool {
s.RLock()
defer s.RUnlock()
var _tokens [64]string
tokens := tokenizeSubjectIntoSlice(_tokens[:0], subj)
return hasInterestStartingIn(s.root, tokens)
}
func hasInterestStartingIn[T comparable](l *level[T], tokens []string) bool {
if l == nil {
return false
}
if len(tokens) == 0 {
return true
}
token := tokens[0]
if l.fwc != nil {
return true
}
found := false
if pwc := l.pwc; pwc != nil {
found = found || hasInterestStartingIn(pwc.next, tokens[1:])
}
if n := l.nodes[token]; n != nil {
found = found || hasInterestStartingIn(n.next, tokens[1:])
}
return found
}
// pruneNode is used to prune an empty node from the tree.
func (l *level[T]) pruneNode(n *node[T], t string) {
if n == nil {
@@ -458,86 +492,15 @@ func visitLevel[T comparable](l *level[T], depth int) int {
return maxDepth
}
// IntersectStree will match all items in the given subject tree that
// have interest expressed in the given sublist. The callback will only be called
// once for each subject, regardless of overlapping subscriptions in the sublist.
func IntersectStree[T1 any, T2 comparable](st *stree.SubjectTree[T1], sl *GenericSublist[T2], cb func(subj []byte, entry *T1)) {
var _subj [255]byte
intersectStree(st, sl.root, _subj[:0], cb)
}
func intersectStree[T1 any, T2 comparable](st *stree.SubjectTree[T1], r *level[T2], subj []byte, cb func(subj []byte, entry *T1)) {
nsubj := subj
if len(nsubj) > 0 {
nsubj = append(subj, '.')
}
if r.fwc != nil {
// We've reached a full wildcard, do a FWC match on the stree at this point
// and don't keep iterating downward.
nsubj := append(nsubj, '>')
st.Match(nsubj, cb)
return
}
if r.pwc != nil {
// We've found a partial wildcard. We'll keep iterating downwards, but first
// check whether there's interest at this level (without triggering dupes) and
// match if so.
var done bool
nsubj := append(nsubj, '*')
if len(r.pwc.subs) > 0 {
st.Match(nsubj, cb)
done = true
}
if r.pwc.next.numNodes() > 0 {
intersectStree(st, r.pwc.next, nsubj, cb)
}
if done {
return
}
}
// Normal node with subject literals, keep iterating.
for t, n := range r.nodes {
if r.pwc != nil && r.pwc.next.numNodes() > 0 && n.next.numNodes() > 0 {
// A wildcard at the next level will already visit these descendents
// so skip so we don't callback the same subject more than once.
continue
// use similar to append. meaning, the updated slice will be returned
func tokenizeSubjectIntoSlice(tts []string, subject string) []string {
start := 0
for i := 0; i < len(subject); i++ {
if subject[i] == btsep {
tts = append(tts, subject[start:i])
start = i + 1
}
nsubj := append(nsubj, t...)
if len(n.subs) > 0 {
if subjectHasWildcard(bytesToString(nsubj)) {
st.Match(nsubj, cb)
} else {
if e, ok := st.Find(nsubj); ok {
cb(nsubj, e)
}
}
}
if n.next.numNodes() > 0 {
intersectStree(st, n.next, nsubj, cb)
}
}
}
// Determine if a subject has any wildcard tokens.
func subjectHasWildcard(subject string) bool {
// This one exits earlier then !subjectIsLiteral(subject)
for i, c := range subject {
if c == pwc || c == fwc {
if (i == 0 || subject[i-1] == btsep) &&
(i+1 == len(subject) || subject[i+1] == btsep) {
return true
}
}
}
return false
}
// Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer.
// So this should be used sparingly when we know the encompassing byte slice's lifetime is the same.
func bytesToString(b []byte) string {
if len(b) == 0 {
return _EMPTY_
}
p := unsafe.SliceData(b)
return unsafe.String(p, len(b))
tts = append(tts, subject[start:])
return tts
}
+17 -18
View File
@@ -1,4 +1,4 @@
// Copyright 2019-2025 The NATS Authors
// Copyright 2019-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -1142,6 +1142,12 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
}
js.mu.Lock()
// Accounts get reset to nil on shutdown, since we re-acquire the locks here, we need to check again.
if js.accounts == nil {
js.mu.Unlock()
return NewJSNotEnabledError()
}
if jsa, ok := js.accounts[a.Name]; ok {
a.mu.Lock()
a.js = jsa
@@ -1370,7 +1376,7 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
}
obs, err := mset.addConsumerWithAssignment(&cfg.ConsumerConfig, _EMPTY_, nil, true, ActionCreateOrUpdate, false)
if err != nil {
s.Warnf(" Error adding consumer %q: %v", cfg.Name, err)
s.Warnf(" Error adding consumer '%s > %s > %s': %v", a.Name, mset.name(), cfg.Name, err)
continue
}
if isEphemeral {
@@ -1379,9 +1385,6 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
if !cfg.Created.IsZero() {
obs.setCreatedTime(cfg.Created)
}
if err != nil {
s.Warnf(" Error restoring consumer %q state: %v", cfg.Name, err)
}
}
}
@@ -1560,7 +1563,7 @@ func (a *Account) EnableJetStream(limits map[string]JetStreamAccountLimits, tq c
}
// Add in the stream.
mset, err := a.addStream(&cfg.StreamConfig)
mset, err := a.recoverStream(&cfg.StreamConfig)
if err != nil {
s.Warnf(" Error recreating stream %q: %v", cfg.Name, err)
// If we removed a keyfile from above make sure to put it back.
@@ -2362,8 +2365,10 @@ func tierName(replicas int) string {
}
func isSameTier(cfgA, cfgB *StreamConfig) bool {
a := max(1, cfgA.Replicas)
b := max(1, cfgB.Replicas)
// TODO (mh) this is where we could select based off a placement tag as well "qos:tier"
return cfgA.Replicas == cfgB.Replicas
return a == b
}
func (jsa *jsAccount) jetStreamAndClustered() (*jetStream, bool) {
@@ -2438,17 +2443,11 @@ func (jsa *jsAccount) wouldExceedLimits(storeType StorageType, tierName string,
// Since tiers are flat we need to scale limit up by replicas when checking.
if storeType == MemoryStorage {
totalMem := inUse.total.mem + (int64(memStoreMsgSize(subj, hdr, msg)) * r)
if selectedLimits.MemoryMaxStreamBytes > 0 && totalMem > selectedLimits.MemoryMaxStreamBytes*lr {
return true, nil
}
if selectedLimits.MaxMemory >= 0 && totalMem > selectedLimits.MaxMemory*lr {
return true, nil
}
} else {
totalStore := inUse.total.store + (int64(fileStoreMsgSize(subj, hdr, msg)) * r)
if selectedLimits.StoreMaxStreamBytes > 0 && totalStore > selectedLimits.StoreMaxStreamBytes*lr {
return true, nil
}
if selectedLimits.MaxStore >= 0 && totalStore > selectedLimits.MaxStore*lr {
return true, nil
}
@@ -2487,25 +2486,25 @@ func (js *jetStream) checkBytesLimits(selectedLimits *JetStreamAccountLimits, ad
if addBytes < 0 {
addBytes = 1
}
totalBytes := addBytes + maxBytesOffset
totalBytes := addSaturate(addBytes, maxBytesOffset)
switch storage {
case MemoryStorage:
// Account limits defined.
if selectedLimits.MaxMemory >= 0 && currentRes+totalBytes > selectedLimits.MaxMemory {
if selectedLimits.MaxMemory >= 0 && (currentRes > selectedLimits.MaxMemory || totalBytes > selectedLimits.MaxMemory-currentRes) {
return NewJSMemoryResourcesExceededError()
}
// Check if this server can handle request.
if checkServer && js.memReserved+totalBytes > js.config.MaxMemory {
if checkServer && (js.memReserved > js.config.MaxMemory || totalBytes > js.config.MaxMemory-js.memReserved) {
return NewJSMemoryResourcesExceededError()
}
case FileStorage:
// Account limits defined.
if selectedLimits.MaxStore >= 0 && currentRes+totalBytes > selectedLimits.MaxStore {
if selectedLimits.MaxStore >= 0 && (currentRes > selectedLimits.MaxStore || totalBytes > selectedLimits.MaxStore-currentRes) {
return NewJSStorageResourcesExceededError()
}
// Check if this server can handle request.
if checkServer && js.storeReserved+totalBytes > js.config.MaxStore {
if checkServer && (js.storeReserved > js.config.MaxStore || totalBytes > js.config.MaxStore-js.storeReserved) {
return NewJSStorageResourcesExceededError()
}
}
File diff suppressed because it is too large Load Diff
+35 -30
View File
@@ -611,41 +611,46 @@ func checkMsgHeadersPreClusteredProposal(
// We need to deny here otherwise we'd need to bump CLFS, and it could succeed on some
// peers and not others depending on consumer ack state (if interest policy).
// So we deny here, if we allow that means we know it would succeed on every peer.
if discard == DiscardNew && (maxMsgs > 0 || maxBytes > 0) {
// Error if over DiscardNew per subject threshold.
if discardNewPer {
totalMsgsForSubject := i.ops
if i, ok = mset.inflight[subject]; ok {
totalMsgsForSubject += i.ops
if discard == DiscardNew {
if maxMsgs > 0 || maxBytes > 0 {
// Track usual max msgs/bytes thresholds for DiscardNew.
var state StreamState
mset.store.FastState(&state)
totalMsgs := state.Msgs
totalBytes := state.Bytes
for _, i = range mset.inflight {
totalMsgs += i.ops
totalBytes += i.bytes
}
if maxMsgsPer > 0 && totalMsgsForSubject > uint64(maxMsgsPer) {
err = ErrMaxMsgsPerSubject
for _, i = range diff.inflight {
totalMsgs += i.ops
totalBytes += i.bytes
}
if maxMsgs > 0 && totalMsgs > uint64(maxMsgs) {
err = ErrMaxMsgs
} else if maxBytes > 0 && totalBytes > uint64(maxBytes) {
err = ErrMaxBytes
}
if err != nil {
return hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err
}
}
// Track usual max msgs/bytes thresholds for DiscardNew.
var state StreamState
mset.store.FastState(&state)
totalMsgs := state.Msgs
totalBytes := state.Bytes
for _, i = range mset.inflight {
totalMsgs += i.ops
totalBytes += i.bytes
}
for _, i = range diff.inflight {
totalMsgs += i.ops
totalBytes += i.bytes
}
if maxMsgs > 0 && totalMsgs > uint64(maxMsgs) {
err = ErrMaxMsgs
} else if maxBytes > 0 && totalBytes > uint64(maxBytes) {
err = ErrMaxBytes
}
if err != nil {
return hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err
// Similarly, check DiscardNew per-subject threshold to not need to bump CLFS.
if discardNewPer && maxMsgsPer > 0 {
// Get the current total for this subject.
totalMsgsForSubject := mset.store.SubjectsTotals(subject)[subject]
// Add inflight count in this batch and for this stream.
totalMsgsForSubject += i.ops
if i, ok = mset.inflight[subject]; ok {
totalMsgsForSubject += i.ops
}
if totalMsgsForSubject > uint64(maxMsgsPer) {
err = ErrMaxMsgsPerSubject
return hdr, msg, 0, NewJSStreamStoreFailedError(err, Unless(err)), err
}
}
}
File diff suppressed because it is too large Load Diff
@@ -59,6 +59,9 @@ const (
// JSClusterRequiredErr JetStream clustering support required
JSClusterRequiredErr ErrorIdentifier = 10010
// JSClusterServerMemberChangeInflightErr cluster member change is in progress
JSClusterServerMemberChangeInflightErr ErrorIdentifier = 10202
// JSClusterServerNotMemberErr server is not a member of the cluster
JSClusterServerNotMemberErr ErrorIdentifier = 10044
@@ -626,6 +629,7 @@ var (
JSClusterNotLeaderErr: {Code: 500, ErrCode: 10009, Description: "JetStream cluster can not handle request"},
JSClusterPeerNotMemberErr: {Code: 400, ErrCode: 10040, Description: "peer not a member"},
JSClusterRequiredErr: {Code: 503, ErrCode: 10010, Description: "JetStream clustering support required"},
JSClusterServerMemberChangeInflightErr: {Code: 400, ErrCode: 10202, Description: "cluster member change is in progress"},
JSClusterServerNotMemberErr: {Code: 400, ErrCode: 10044, Description: "server is not a member of the cluster"},
JSClusterTagsErr: {Code: 400, ErrCode: 10011, Description: "tags placement not supported for operation"},
JSClusterUnSupportFeatureErr: {Code: 503, ErrCode: 10036, Description: "not currently supported in clustered mode"},
@@ -1031,6 +1035,16 @@ func NewJSClusterRequiredError(opts ...ErrorOption) *ApiError {
return ApiErrors[JSClusterRequiredErr]
}
// NewJSClusterServerMemberChangeInflightError creates a new JSClusterServerMemberChangeInflightErr error: "cluster member change is in progress"
func NewJSClusterServerMemberChangeInflightError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
if ae, ok := eopts.err.(*ApiError); ok {
return ae
}
return ApiErrors[JSClusterServerMemberChangeInflightErr]
}
// NewJSClusterServerNotMemberError creates a new JSClusterServerNotMemberErr error: "server is not a member of the cluster"
func NewJSClusterServerNotMemberError(opts ...ErrorOption) *ApiError {
eopts := parseOpts(opts)
+1 -1
View File
@@ -17,7 +17,7 @@ import "strconv"
const (
// JSApiLevel is the maximum supported JetStream API level for this server.
JSApiLevel int = 2
JSApiLevel int = 3
JSRequiredLevelMetadataKey = "_nats.req.level"
JSServerVersionMetadataKey = "_nats.ver"
+38 -23
View File
@@ -63,9 +63,9 @@ const (
// LEAF connection as opposed to a CLIENT.
leafNodeWSPath = "/leafnode"
// This is the time the server will wait, when receiving a CONNECT,
// before closing the connection if the required minimum version is not met.
leafNodeWaitBeforeClose = 5 * time.Second
// When a soliciting leafnode is rejected because it does not meet the
// configured minimum version, delay the next reconnect attempt by this long.
leafNodeMinVersionReconnectDelay = 5 * time.Second
)
type leaf struct {
@@ -691,9 +691,8 @@ func (s *Server) connectToRemoteLeafNode(remote *leafNodeCfg, firstConnect bool)
} else {
s.Debugf("Trying to connect as leafnode to remote server on %q%s", rURL.Host, ipStr)
// Check if proxy is configured first, then check if URL supports it
if proxyURL != _EMPTY_ && isWSURL(rURL) {
// Use proxy for WebSocket connections - use original hostname, resolved IP for connection
// Check if proxy is configured
if proxyURL != _EMPTY_ {
targetHost := rURL.Host
// If URL doesn't include port, add the default port for the scheme
if rURL.Port() == _EMPTY_ {
@@ -2082,17 +2081,11 @@ func (c *client) processLeafNodeConnect(s *Server, arg []byte, lang string) erro
if mv := s.getOpts().LeafNode.MinVersion; mv != _EMPTY_ {
major, minor, update, _ := versionComponents(mv)
if !versionAtLeast(proto.Version, major, minor, update) {
// We are going to send back an INFO because otherwise recent
// versions of the remote server would simply break the connection
// after 2 seconds if not receiving it. Instead, we want the
// other side to just "stall" until we finish waiting for the holding
// period and close the connection below.
// Send back an INFO so recent remote servers process the rejection
// cleanly, then close immediately. The soliciting side applies the
// reconnect delay when it processes the error.
s.sendPermsAndAccountInfo(c)
c.sendErrAndErr(fmt.Sprintf("connection rejected since minimum version required is %q", mv))
select {
case <-c.srv.quitCh:
case <-time.After(leafNodeWaitBeforeClose):
}
c.sendErrAndErr(fmt.Sprintf("%s %q", ErrLeafNodeMinVersionRejected, mv))
c.closeConnection(MinimumVersionRequired)
return ErrMinimumVersionRequired
}
@@ -2760,6 +2753,14 @@ func (c *client) processLeafSub(argo []byte) (err error) {
}
acc := c.acc
// Guard against LS+ arriving before CONNECT has been processed, which
// can happen when compression is enabled.
if acc == nil {
c.mu.Unlock()
c.sendErr("Authorization Violation")
c.closeConnection(ProtocolViolation)
return nil
}
// Check if we have a loop.
ldsPrefix := bytes.HasPrefix(sub.subject, []byte(leafNodeLoopDetectionSubjectPrefix))
@@ -2832,7 +2833,7 @@ func (c *client) processLeafSub(argo []byte) (err error) {
// Only add in shadow subs if a new sub or qsub.
if osub == nil {
if err := c.addShadowSubscriptions(acc, sub, true); err != nil {
if err := c.addShadowSubscriptions(acc, sub); err != nil {
c.Errorf(err.Error())
}
}
@@ -2876,7 +2877,6 @@ func (c *client) processLeafUnsub(arg []byte) error {
// Indicate any activity, so pub and sub or unsubs.
c.in.subs++
acc := c.acc
srv := c.srv
c.mu.Lock()
@@ -2885,6 +2885,15 @@ func (c *client) processLeafUnsub(arg []byte) error {
return nil
}
acc := c.acc
// Guard against LS- arriving before CONNECT has been processed.
if acc == nil {
c.mu.Unlock()
c.sendErr("Authorization Violation")
c.closeConnection(ProtocolViolation)
return nil
}
spoke := c.isSpokeLeafNode()
// We store local subs by account and subject and optionally queue name.
// LS- will have the arg exactly as the key.
@@ -2916,8 +2925,7 @@ func (c *client) processLeafUnsub(arg []byte) error {
func (c *client) processLeafHeaderMsgArgs(arg []byte) error {
// Unroll splitArgs to avoid runtime/heap issues
a := [MAX_MSG_ARGS][]byte{}
args := a[:0]
args := c.argsa[:0]
start := -1
for i, b := range arg {
switch b {
@@ -3000,8 +3008,7 @@ func (c *client) processLeafHeaderMsgArgs(arg []byte) error {
func (c *client) processLeafMsgArgs(arg []byte) error {
// Unroll splitArgs to avoid runtime/heap issues
a := [MAX_MSG_ARGS][]byte{}
args := a[:0]
args := c.argsa[:0]
start := -1
for i, b := range arg {
switch b {
@@ -3181,6 +3188,11 @@ func (c *client) leafProcessErr(errStr string) {
c.Errorf("Leafnode connection dropped with same cluster name error. Delaying attempt to reconnect for %v", delay)
return
}
if strings.Contains(errStr, ErrLeafNodeMinVersionRejected.Error()) {
_, delay := c.setLeafConnectDelayIfSoliciting(leafNodeMinVersionReconnectDelay)
c.Errorf("Leafnode connection dropped due to minimum version requirement. Delaying attempt to reconnect for %v", delay)
return
}
// We will look for Loop detected error coming from the other side.
// If we solicit, set the connect delay.
@@ -3203,7 +3215,10 @@ func (c *client) setLeafConnectDelayIfSoliciting(delay time.Duration) (string, t
}
c.leaf.remote.setConnectDelay(delay)
}
accName := c.acc.Name
var accName string
if c.acc != nil {
accName = c.acc.Name
}
c.mu.Unlock()
return accName, delay
}
+194 -54
View File
@@ -19,7 +19,6 @@ import (
"fmt"
"math"
"slices"
"sort"
"sync"
"time"
@@ -445,7 +444,6 @@ func (ms *memStore) RegisterProcessJetStreamMsg(cb ProcessJetStreamMsgHandler) {
// GetSeqFromTime looks for the first sequence number that has the message
// with >= timestamp.
// FIXME(dlc) - inefficient.
func (ms *memStore) GetSeqFromTime(t time.Time) uint64 {
ts := t.UnixNano()
ms.mu.RLock()
@@ -469,18 +467,57 @@ func (ms *memStore) GetSeqFromTime(t time.Time) uint64 {
last := lmsg.ts
if ts == last {
return ms.state.LastSeq
return lmsg.seq
}
if ts > last {
return ms.state.LastSeq + 1
}
index := sort.Search(len(ms.msgs), func(i int) bool {
if msg := ms.msgs[ms.state.FirstSeq+uint64(i)]; msg != nil {
return msg.ts >= ts
var (
cts int64
cseq uint64
off uint64
)
// Using a binary search, but need to be aware of interior deletes.
fseq := ms.state.FirstSeq
lseq := ms.state.LastSeq
seq := lseq + 1
loop:
for fseq <= lseq {
mid := fseq + (lseq-fseq)/2
off = 0
// Potentially skip over gaps. We keep the original middle but keep track of a
// potential delete range with an offset.
for {
msg := ms.msgs[mid+off]
if msg == nil {
off++
if mid+off <= lseq {
continue
} else {
// Continue search to the left. Purposely ignore the skipped deletes here.
lseq = mid - 1
continue loop
}
}
cts = msg.ts
cseq = msg.seq
break
}
return false
})
return uint64(index) + ms.state.FirstSeq
if cts >= ts {
seq = cseq
if mid == fseq {
break
}
// Continue search to the left.
lseq = mid - 1
} else {
// Continue search to the right (potentially skipping over interior deletes).
fseq = mid + off + 1
}
}
return seq
}
// FilteredState will return the SimpleState associated with the filtered subject and a proposed starting sequence.
@@ -749,6 +786,10 @@ func (ms *memStore) allLastSeqsLocked() ([]uint64, error) {
seqs := make([]uint64, 0, ms.fss.Size())
ms.fss.IterFast(func(subj []byte, ss *SimpleState) bool {
// Check if we need to recalculate. We only care about the last sequence.
if ss.lastNeedsUpdate {
ms.recalculateForSubj(bytesToString(subj), ss)
}
seqs = append(seqs, ss.Last)
return true
})
@@ -766,6 +807,7 @@ func (ms *memStore) filterIsAll(filters []string) bool {
}
// Sort so we can compare.
slices.Sort(filters)
slices.Sort(ms.cfg.Subjects)
for i, subj := range filters {
if !subjectIsSubsetMatch(ms.cfg.Subjects[i], subj) {
return false
@@ -777,8 +819,8 @@ func (ms *memStore) filterIsAll(filters []string) bool {
// MultiLastSeqs will return a sorted list of sequences that match all subjects presented in filters.
// We will not exceed the maxSeq, which if 0 becomes the store's last sequence.
func (ms *memStore) MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error) {
ms.mu.RLock()
defer ms.mu.RUnlock()
ms.mu.Lock()
defer ms.mu.Unlock()
if len(ms.msgs) == 0 {
return nil, nil
@@ -806,6 +848,9 @@ func (ms *memStore) MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed in
for _, filter := range filters {
ms.fss.Match(stringToBytes(filter), func(subj []byte, ss *SimpleState) {
if ss.lastNeedsUpdate {
ms.recalculateForSubj(bytesToString(subj), ss)
}
if ss.Last <= maxSeq {
addIfNotDupe(ss.Last)
} else if ss.Msgs > 1 {
@@ -861,17 +906,17 @@ func (ms *memStore) subjectsTotalsLocked(filterSubject string) map[string]uint64
}
// NumPending will return the number of pending messages matching the filter subject starting at sequence.
func (ms *memStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64) {
func (ms *memStore) NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error) {
// This needs to be a write lock, as filteredStateLocked can mutate the per-subject state.
ms.mu.Lock()
defer ms.mu.Unlock()
ss := ms.filteredStateLocked(sseq, filter, lastPerSubject)
return ss.Msgs, ms.state.LastSeq
return ss.Msgs, ms.state.LastSeq, nil
}
// NumPending will return the number of pending messages matching any subject in the sublist starting at sequence.
func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64) {
func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error) {
if sl == nil {
return ms.NumPending(sseq, fwcs, lastPerSubject)
}
@@ -886,7 +931,7 @@ func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerS
}
// If past the end no results.
if sseq > ms.state.LastSeq {
return 0, ms.state.LastSeq
return 0, ms.state.LastSeq, nil
}
update := func(fss *SimpleState) {
@@ -906,7 +951,7 @@ func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerS
var havePartial bool
var totalSkipped uint64
// We will track start and end sequences as we go.
gsl.IntersectStree[SimpleState](ms.fss, sl, func(subj []byte, fss *SimpleState) {
stree.IntersectGSL[SimpleState](ms.fss, sl, func(subj []byte, fss *SimpleState) {
if fss.firstNeedsUpdate || fss.lastNeedsUpdate {
ms.recalculateForSubj(bytesToString(subj), fss)
}
@@ -924,7 +969,7 @@ func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerS
// If we did not encounter any partials we can return here.
if !havePartial {
return ss.Msgs, ms.state.LastSeq
return ss.Msgs, ms.state.LastSeq, nil
}
// If we are here we need to scan the msgs.
@@ -1015,7 +1060,7 @@ func (ms *memStore) NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerS
ss.Msgs -= adjust
}
return ss.Msgs, ms.state.LastSeq
return ss.Msgs, ms.state.LastSeq, nil
}
// Will check the msg limit for this tracked subject.
@@ -1463,6 +1508,12 @@ func (ms *memStore) compact(seq uint64) (uint64, error) {
var purged, bytes uint64
ms.mu.Lock()
// Short-circuit if the store was already compacted past this point.
if ms.state.FirstSeq > seq {
ms.mu.Unlock()
return purged, nil
}
cb := ms.scb
if seq <= ms.state.LastSeq {
fseq := ms.state.FirstSeq
@@ -1621,7 +1672,8 @@ func (ms *memStore) SubjectForSeq(seq uint64) (string, error) {
return _EMPTY_, ErrStoreMsgNotFound
}
if sm, ok := ms.msgs[seq]; ok {
return sm.subj, nil
// Copy the subject, as it's used elsewhere, and we've released the lock in the meantime.
return copyString(sm.subj), nil
}
return _EMPTY_, ErrStoreMsgNotFound
}
@@ -1673,6 +1725,10 @@ func (ms *memStore) LoadLastMsg(subject string, smp *StoreMsg) (*StoreMsg, error
} else if subjectIsLiteral(subject) {
var ss *SimpleState
if ss, ok = ms.fss.Find(stringToBytes(subject)); ok && ss.Msgs > 0 {
// Check if we need to recalculate. We only care about the last sequence.
if ss.lastNeedsUpdate {
ms.recalculateForSubj(subject, ss)
}
sm, ok = ms.msgs[ss.Last]
}
} else if ss := ms.filteredStateLocked(1, subject, true); ss.Msgs > 0 {
@@ -1731,6 +1787,78 @@ func (ms *memStore) LoadNextMsg(filter string, wc bool, start uint64, smp *Store
return ms.loadNextMsgLocked(filter, wc, start, smp)
}
// Find sequence bounds matching a wildcard filter from ms.fss.
// Returns (first, last, true) if there is at least one matching
// subject at or after start (start <= first <= last).
// Returns (0, 0, false) if the subject does not exist or has no
// messages at or after start.
// Lock should be held.
func (ms *memStore) nextWildcardMatchLocked(filter string, start uint64) (uint64, uint64, bool) {
found := false
first, last := ms.state.LastSeq, uint64(0)
ms.fss.MatchUntil(stringToBytes(filter), func(subj []byte, ss *SimpleState) bool {
ms.recalculateForSubj(string(subj), ss)
// Skip matches that are below our starting sequence
if start > ss.Last {
return true
}
// A match was found, adjust the bounds accordingly
found = true
if ss.First < first {
first = ss.First
}
if ss.Last > last {
last = ss.Last
}
// If first > start, there may be more matches between
// start and first, in which case we keep searching.
// If not, we have a match between start and last, we
// can break out of the search.
// This could be further optimized: if first and start
// are "close", we could just extend the linear search,
// especially if we know that the remaining ms.fss to
// explore is large.
return first > start
})
if !found {
return 0, 0, false
}
return max(first, start), last, found
}
// Find sequence bounds matching a literal filter from ms.fss.
// Returns (first, last, true) if there is a matching literal
// subject at or after start (start <= first <= last).
// Returns (0, 0, false) if the subject does not exist or has no
// messages at or after start.
// Lock should be held.
func (ms *memStore) nextLiteralMatchLocked(filter string, start uint64) (uint64, uint64, bool) {
ss, ok := ms.fss.Find(stringToBytes(filter))
if !ok {
return 0, 0, false
}
ms.recalculateForSubj(filter, ss)
if start > ss.Last {
return 0, 0, false
}
return max(start, ss.First), ss.Last, true
}
// Returns true if LoadNextMsg should perform a linear scan,
// false if it should use the subject tree to try to reduce
// the search space.
// Lock should be held.
func (ms *memStore) shouldLinearScan(filter string, wc bool, start uint64) bool {
// Skip scan of ms.fss if number of messages in the block are less than
// 1/2 the number of subjects in ms.fss. Or we have a wc and lots of fss entries.
const linearScanMaxFSS = 256
isAll := filter == fwcs
return isAll || 2*int(ms.state.LastSeq-start) < ms.fss.Size() || (wc && ms.fss.Size() > linearScanMaxFSS)
}
// Lock should be held.
func (ms *memStore) loadNextMsgLocked(filter string, wc bool, start uint64, smp *StoreMsg) (*StoreMsg, uint64, error) {
if start < ms.state.FirstSeq {
@@ -1747,46 +1875,24 @@ func (ms *memStore) loadNextMsgLocked(filter string, wc bool, start uint64, smp
}
isAll := filter == fwcs
// Skip scan of ms.fss if number of messages in the block are less than
// 1/2 the number of subjects in ms.fss. Or we have a wc and lots of fss entries.
const linearScanMaxFSS = 256
doLinearScan := isAll || 2*int(ms.state.LastSeq-start) < ms.fss.Size() || (wc && ms.fss.Size() > linearScanMaxFSS)
// Initial setup.
fseq, lseq := start, ms.state.LastSeq
if !doLinearScan {
subs := []string{filter}
if wc || isAll {
subs = subs[:0]
ms.fss.Match(stringToBytes(filter), func(subj []byte, val *SimpleState) {
subs = append(subs, string(subj))
})
if !ms.shouldLinearScan(filter, wc, start) {
var found bool
if wc {
fseq, lseq, found = ms.nextWildcardMatchLocked(filter, start)
} else {
fseq, lseq, found = ms.nextLiteralMatchLocked(filter, start)
}
fseq, lseq = ms.state.LastSeq, uint64(0)
for _, subj := range subs {
ss, ok := ms.fss.Find(stringToBytes(subj))
if !ok {
continue
}
if ss.firstNeedsUpdate || ss.lastNeedsUpdate {
ms.recalculateForSubj(subj, ss)
}
if ss.First < fseq {
fseq = ss.First
}
if ss.Last > lseq {
lseq = ss.Last
}
}
if fseq < start {
fseq = start
if !found {
return nil, ms.state.LastSeq, ErrStoreEOF
}
}
eq := subjectsEqual
if wc {
eq = subjectIsSubsetMatch
eq = matchLiteral
}
for nseq := fseq; nseq <= lseq; nseq++ {
@@ -1828,6 +1934,40 @@ func (ms *memStore) LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err
return nil, ErrStoreEOF
}
// LoadPrevMsgMulti will find the previous message matching any entry in the sublist.
func (ms *memStore) LoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error) {
// TODO(dlc) - for now simple linear walk to get started.
ms.mu.RLock()
defer ms.mu.RUnlock()
if start > ms.state.LastSeq {
start = ms.state.LastSeq
}
// If past the start no results.
if start < ms.state.FirstSeq || ms.state.Msgs == 0 {
return nil, ms.state.FirstSeq, ErrStoreEOF
}
// Initial setup.
fseq, lseq := start, ms.state.FirstSeq
for nseq := fseq; nseq >= lseq; nseq-- {
sm, ok := ms.msgs[nseq]
if !ok {
continue
}
if sl.HasInterest(sm.subj) {
if smp == nil {
smp = new(StoreMsg)
}
sm.copy(smp)
return smp, nseq, nil
}
}
return nil, ms.state.LastSeq, ErrStoreEOF
}
// RemoveMsg will remove the message from this store.
// Will return the number of bytes removed.
func (ms *memStore) RemoveMsg(seq uint64) (bool, error) {
@@ -2036,8 +2176,8 @@ func (ms *memStore) FastState(state *StreamState) {
}
func (ms *memStore) State() StreamState {
ms.mu.RLock()
defer ms.mu.RUnlock()
ms.mu.Lock()
defer ms.mu.Unlock()
state := ms.state
state.Consumers = ms.consumers
@@ -2129,7 +2269,7 @@ type consumerMemStore struct {
closed bool
}
func (ms *memStore) ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerStore, error) {
func (ms *memStore) ConsumerStore(name string, _ time.Time, cfg *ConsumerConfig) (ConsumerStore, error) {
if ms == nil {
return nil, fmt.Errorf("memstore is nil")
}
@@ -2259,7 +2399,7 @@ func (o *consumerMemStore) Update(state *ConsumerState) error {
// Check to see if this is an outdated update.
if state.Delivered.Consumer < o.state.Delivered.Consumer || state.AckFloor.Stream < o.state.AckFloor.Stream {
return fmt.Errorf("old update ignored")
return ErrStoreOldUpdate
}
o.state.Delivered = state.Delivered
+72 -38
View File
@@ -1,4 +1,4 @@
// Copyright 2013-2025 The NATS Authors
// Copyright 2013-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -500,31 +500,31 @@ func (s *Server) Connz(opts *ConnzOptions) (*Connz, error) {
switch sortOpt {
case ByCid, ByStart:
sort.Sort(byCid{pconns})
sort.Sort(SortByCid{pconns})
case BySubs:
sort.Sort(sort.Reverse(bySubs{pconns}))
sort.Sort(sort.Reverse(SortBySubs{pconns}))
case ByPending:
sort.Sort(sort.Reverse(byPending{pconns}))
sort.Sort(sort.Reverse(SortByPending{pconns}))
case ByOutMsgs:
sort.Sort(sort.Reverse(byOutMsgs{pconns}))
sort.Sort(sort.Reverse(SortByOutMsgs{pconns}))
case ByInMsgs:
sort.Sort(sort.Reverse(byInMsgs{pconns}))
sort.Sort(sort.Reverse(SortByInMsgs{pconns}))
case ByOutBytes:
sort.Sort(sort.Reverse(byOutBytes{pconns}))
sort.Sort(sort.Reverse(SortByOutBytes{pconns}))
case ByInBytes:
sort.Sort(sort.Reverse(byInBytes{pconns}))
sort.Sort(sort.Reverse(SortByInBytes{pconns}))
case ByLast:
sort.Sort(sort.Reverse(byLast{pconns}))
sort.Sort(sort.Reverse(SortByLast{pconns}))
case ByIdle:
sort.Sort(sort.Reverse(byIdle{pconns, c.Now}))
sort.Sort(sort.Reverse(SortByIdle{pconns, c.Now}))
case ByUptime:
sort.Sort(byUptime{pconns, time.Now()})
sort.Sort(SortByUptime{pconns, time.Now()})
case ByStop:
sort.Sort(sort.Reverse(byStop{pconns}))
sort.Sort(sort.Reverse(SortByStop{pconns}))
case ByReason:
sort.Sort(byReason{pconns})
sort.Sort(SortByReason{pconns})
case ByRTT:
sort.Sort(sort.Reverse(byRTT{pconns}))
sort.Sort(sort.Reverse(SortByRTT{pconns}))
}
minoff := c.Offset
@@ -1279,6 +1279,7 @@ type Varz struct {
SlowConsumersStats *SlowConsumersStats `json:"slow_consumer_stats"` // SlowConsumersStats are statistics about all detected Slow Consumer
StaleConnectionStats *StaleConnectionStats `json:"stale_connection_stats,omitempty"` // StaleConnectionStats are statistics about all detected Stale Connections
Proxies *ProxiesOptsVarz `json:"proxies,omitempty"` // Proxies hold information about network proxy devices
TLSCertNotAfter time.Time `json:"tls_cert_not_after,omitzero"` // TLSCertNotAfter is the expiration date of the TLS certificate of this server
}
// JetStreamVarz contains basic runtime information about jetstream
@@ -1291,34 +1292,36 @@ type JetStreamVarz struct {
// ClusterOptsVarz contains monitoring cluster information
type ClusterOptsVarz struct {
Name string `json:"name,omitempty"` // Name is the configured cluster name
Host string `json:"addr,omitempty"` // Host is the host the cluster listens on for connections
Port int `json:"cluster_port,omitempty"` // Port is the port the cluster listens on for connections
AuthTimeout float64 `json:"auth_timeout,omitempty"` // AuthTimeout is the time cluster connections have to complete authentication
URLs []string `json:"urls,omitempty"` // URLs is the list of cluster URLs
TLSTimeout float64 `json:"tls_timeout,omitempty"` // TLSTimeout is how long TLS operations have to complete
TLSRequired bool `json:"tls_required,omitempty"` // TLSRequired indicates if TLS is required for connections
TLSVerify bool `json:"tls_verify,omitempty"` // TLSVerify indicates if full verification of TLS connections is performed
PoolSize int `json:"pool_size,omitempty"` // PoolSize is the configured route connection pool size
WriteDeadline time.Duration `json:"write_deadline,omitempty"` // WriteDeadline is the maximum time writes to sockets have to complete
WriteTimeout string `json:"write_timeout,omitempty"` // WriteTimeout is the closure policy for write deadline errors
Name string `json:"name,omitempty"` // Name is the configured cluster name
Host string `json:"addr,omitempty"` // Host is the host the cluster listens on for connections
Port int `json:"cluster_port,omitempty"` // Port is the port the cluster listens on for connections
AuthTimeout float64 `json:"auth_timeout,omitempty"` // AuthTimeout is the time cluster connections have to complete authentication
URLs []string `json:"urls,omitempty"` // URLs is the list of cluster URLs
TLSTimeout float64 `json:"tls_timeout,omitempty"` // TLSTimeout is how long TLS operations have to complete
TLSRequired bool `json:"tls_required,omitempty"` // TLSRequired indicates if TLS is required for connections
TLSVerify bool `json:"tls_verify,omitempty"` // TLSVerify indicates if full verification of TLS connections is performed
PoolSize int `json:"pool_size,omitempty"` // PoolSize is the configured route connection pool size
WriteDeadline time.Duration `json:"write_deadline,omitempty"` // WriteDeadline is the maximum time writes to sockets have to complete
WriteTimeout string `json:"write_timeout,omitempty"` // WriteTimeout is the closure policy for write deadline errors
TLSCertNotAfter time.Time `json:"tls_cert_not_after,omitzero"` // TLSCertNotAfter is the expiration date of the TLS certificate
}
// GatewayOptsVarz contains monitoring gateway information
type GatewayOptsVarz struct {
Name string `json:"name,omitempty"` // Name is the configured cluster name
Host string `json:"host,omitempty"` // Host is the host the gateway listens on for connections
Port int `json:"port,omitempty"` // Port is the post gateway connections listens on
AuthTimeout float64 `json:"auth_timeout,omitempty"` // AuthTimeout is the time cluster connections have to complete authentication
TLSTimeout float64 `json:"tls_timeout,omitempty"` // TLSTimeout is how long TLS operations have to complete
TLSRequired bool `json:"tls_required,omitempty"` // TLSRequired indicates if TLS is required for connections
TLSVerify bool `json:"tls_verify,omitempty"` // TLSVerify indicates if full verification of TLS connections is performed
Advertise string `json:"advertise,omitempty"` // Advertise is the URL advertised to remote gateway clients
ConnectRetries int `json:"connect_retries,omitempty"` // ConnectRetries is how many connection attempts the route will make
Gateways []RemoteGatewayOptsVarz `json:"gateways,omitempty"` // Gateways is state of configured gateway remotes
RejectUnknown bool `json:"reject_unknown,omitempty"` // RejectUnknown indicates if unknown cluster connections will be rejected
WriteDeadline time.Duration `json:"write_deadline,omitempty"` // WriteDeadline is the maximum time writes to sockets have to complete
WriteTimeout string `json:"write_timeout,omitempty"` // WriteTimeout is the closure policy for write deadline errors
Name string `json:"name,omitempty"` // Name is the configured cluster name
Host string `json:"host,omitempty"` // Host is the host the gateway listens on for connections
Port int `json:"port,omitempty"` // Port is the post gateway connections listens on
AuthTimeout float64 `json:"auth_timeout,omitempty"` // AuthTimeout is the time cluster connections have to complete authentication
TLSTimeout float64 `json:"tls_timeout,omitempty"` // TLSTimeout is how long TLS operations have to complete
TLSRequired bool `json:"tls_required,omitempty"` // TLSRequired indicates if TLS is required for connections
TLSVerify bool `json:"tls_verify,omitempty"` // TLSVerify indicates if full verification of TLS connections is performed
Advertise string `json:"advertise,omitempty"` // Advertise is the URL advertised to remote gateway clients
ConnectRetries int `json:"connect_retries,omitempty"` // ConnectRetries is how many connection attempts the route will make
Gateways []RemoteGatewayOptsVarz `json:"gateways,omitempty"` // Gateways is state of configured gateway remotes
RejectUnknown bool `json:"reject_unknown,omitempty"` // RejectUnknown indicates if unknown cluster connections will be rejected
WriteDeadline time.Duration `json:"write_deadline,omitempty"` // WriteDeadline is the maximum time writes to sockets have to complete
WriteTimeout string `json:"write_timeout,omitempty"` // WriteTimeout is the closure policy for write deadline errors
TLSCertNotAfter time.Time `json:"tls_cert_not_after,omitzero"` // TLSCertNotAfter is the expiration date of the TLS certificaet
}
// RemoteGatewayOptsVarz contains monitoring remote gateway information
@@ -1340,6 +1343,7 @@ type LeafNodeOptsVarz struct {
TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` // TLSOCSPPeerVerify indicates if OCSP verification will be performed
WriteDeadline time.Duration `json:"write_deadline,omitempty"` // WriteDeadline is the maximum time writes to sockets have to complete
WriteTimeout string `json:"write_timeout,omitempty"` // WriteTimeout is the closure policy for write deadline errors
TLSCertNotAfter time.Time `json:"tls_cert_not_after,omitzero"` // TLSCertNotAfter is the expiration date of the TLS certificate
}
// DenyRules Contains lists of subjects not allowed to be imported/exported
@@ -1370,6 +1374,7 @@ type MQTTOptsVarz struct {
AckWait time.Duration `json:"ack_wait,omitempty"` // AckWait is how long the internal JetStream state store will allow acks to complete
MaxAckPending uint16 `json:"max_ack_pending,omitempty"` // MaxAckPending is how many outstanding acks the internal JetStream state store will allow
TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` // TLSOCSPPeerVerify indicates if OCSP verification will be done
TLSCertNotAfter time.Time `json:"tls_cert_not_after,omitzero"` // TLSCertNotAfter is the expiration date of the TLS certificate
}
// WebsocketOptsVarz contains monitoring websocket information
@@ -1388,6 +1393,7 @@ type WebsocketOptsVarz struct {
AllowedOrigins []string `json:"allowed_origins,omitempty"` // AllowedOrigins list of configured trusted origins
Compression bool `json:"compression,omitempty"` // Compression indicates if compression is supported
TLSOCSPPeerVerify bool `json:"tls_ocsp_peer_verify,omitempty"` // TLSOCSPPeerVerify indicates if OCSP verification will be done
TLSCertNotAfter time.Time `json:"tls_cert_not_after,omitzero"` // TLSCertNotAfter is the expiration date of the TLS certificate
}
// OCSPResponseCacheVarz contains OCSP response cache information
@@ -1454,6 +1460,22 @@ func myUptime(d time.Duration) string {
return fmt.Sprintf("%ds", tsecs)
}
func tlsCertNotAfter(config *tls.Config) time.Time {
if config == nil || len(config.Certificates) == 0 {
return time.Time{}
}
cert := config.Certificates[0]
leaf := cert.Leaf
if leaf == nil {
var err error
leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return time.Time{}
}
}
return leaf.NotAfter
}
// HandleRoot will show basic info and links to others handlers.
func (s *Server) HandleRoot(w http.ResponseWriter, r *http.Request) {
// This feels dumb to me, but is required: https://code.google.com/p/go/issues/detail?id=4799
@@ -1779,6 +1801,13 @@ func (s *Server) updateVarzConfigReloadableFields(v *Varz) {
v.TLSOCSPPeerVerify = s.ocspPeerVerify && v.TLSRequired && s.opts.tlsConfigOpts != nil && s.opts.tlsConfigOpts.OCSPPeerConfig != nil && s.opts.tlsConfigOpts.OCSPPeerConfig.Verify
v.TLSCertNotAfter = tlsCertNotAfter(opts.TLSConfig)
v.Cluster.TLSCertNotAfter = tlsCertNotAfter(opts.Cluster.TLSConfig)
v.Gateway.TLSCertNotAfter = tlsCertNotAfter(opts.Gateway.TLSConfig)
v.LeafNode.TLSCertNotAfter = tlsCertNotAfter(opts.LeafNode.TLSConfig)
v.MQTT.TLSCertNotAfter = tlsCertNotAfter(opts.MQTT.TLSConfig)
v.Websocket.TLSCertNotAfter = tlsCertNotAfter(opts.Websocket.TLSConfig)
if opts.Proxies != nil {
if v.Proxies == nil {
v.Proxies = &ProxiesOptsVarz{}
@@ -3982,6 +4011,11 @@ func (s *Server) healthz(opts *HealthzOptions) *HealthStatus {
return health
}
// Healthz returns the health status of the server.
func (s *Server) Healthz(opts *HealthzOptions) *HealthStatus {
return s.healthz(opts)
}
type ExpvarzStatus struct {
Memstats json.RawMessage `json:"memstats"`
Cmdline json.RawMessage `json:"cmdline"`
+26 -26
View File
@@ -50,64 +50,64 @@ const (
// Individual sort options provide the Less for sort.Interface. Len and Swap are on cList.
// CID
type byCid struct{ ConnInfos }
type SortByCid struct{ ConnInfos }
func (l byCid) Less(i, j int) bool { return l.ConnInfos[i].Cid < l.ConnInfos[j].Cid }
func (l SortByCid) Less(i, j int) bool { return l.ConnInfos[i].Cid < l.ConnInfos[j].Cid }
// Number of Subscriptions
type bySubs struct{ ConnInfos }
type SortBySubs struct{ ConnInfos }
func (l bySubs) Less(i, j int) bool { return l.ConnInfos[i].NumSubs < l.ConnInfos[j].NumSubs }
func (l SortBySubs) Less(i, j int) bool { return l.ConnInfos[i].NumSubs < l.ConnInfos[j].NumSubs }
// Pending Bytes
type byPending struct{ ConnInfos }
type SortByPending struct{ ConnInfos }
func (l byPending) Less(i, j int) bool { return l.ConnInfos[i].Pending < l.ConnInfos[j].Pending }
func (l SortByPending) Less(i, j int) bool { return l.ConnInfos[i].Pending < l.ConnInfos[j].Pending }
// Outbound Msgs
type byOutMsgs struct{ ConnInfos }
type SortByOutMsgs struct{ ConnInfos }
func (l byOutMsgs) Less(i, j int) bool { return l.ConnInfos[i].OutMsgs < l.ConnInfos[j].OutMsgs }
func (l SortByOutMsgs) Less(i, j int) bool { return l.ConnInfos[i].OutMsgs < l.ConnInfos[j].OutMsgs }
// Inbound Msgs
type byInMsgs struct{ ConnInfos }
type SortByInMsgs struct{ ConnInfos }
func (l byInMsgs) Less(i, j int) bool { return l.ConnInfos[i].InMsgs < l.ConnInfos[j].InMsgs }
func (l SortByInMsgs) Less(i, j int) bool { return l.ConnInfos[i].InMsgs < l.ConnInfos[j].InMsgs }
// Outbound Bytes
type byOutBytes struct{ ConnInfos }
type SortByOutBytes struct{ ConnInfos }
func (l byOutBytes) Less(i, j int) bool { return l.ConnInfos[i].OutBytes < l.ConnInfos[j].OutBytes }
func (l SortByOutBytes) Less(i, j int) bool { return l.ConnInfos[i].OutBytes < l.ConnInfos[j].OutBytes }
// Inbound Bytes
type byInBytes struct{ ConnInfos }
type SortByInBytes struct{ ConnInfos }
func (l byInBytes) Less(i, j int) bool { return l.ConnInfos[i].InBytes < l.ConnInfos[j].InBytes }
func (l SortByInBytes) Less(i, j int) bool { return l.ConnInfos[i].InBytes < l.ConnInfos[j].InBytes }
// Last Activity
type byLast struct{ ConnInfos }
type SortByLast struct{ ConnInfos }
func (l byLast) Less(i, j int) bool {
func (l SortByLast) Less(i, j int) bool {
return l.ConnInfos[i].LastActivity.UnixNano() < l.ConnInfos[j].LastActivity.UnixNano()
}
// Idle time
type byIdle struct {
type SortByIdle struct {
ConnInfos
now time.Time
}
func (l byIdle) Less(i, j int) bool {
func (l SortByIdle) Less(i, j int) bool {
return l.now.Sub(l.ConnInfos[i].LastActivity) < l.now.Sub(l.ConnInfos[j].LastActivity)
}
// Uptime
type byUptime struct {
type SortByUptime struct {
ConnInfos
now time.Time
}
func (l byUptime) Less(i, j int) bool {
func (l SortByUptime) Less(i, j int) bool {
ci := l.ConnInfos[i]
cj := l.ConnInfos[j]
var upi, upj time.Duration
@@ -125,25 +125,25 @@ func (l byUptime) Less(i, j int) bool {
}
// Stop
type byStop struct{ ConnInfos }
type SortByStop struct{ ConnInfos }
func (l byStop) Less(i, j int) bool {
func (l SortByStop) Less(i, j int) bool {
ciStop := l.ConnInfos[i].Stop
cjStop := l.ConnInfos[j].Stop
return ciStop.Before(*cjStop)
}
// Reason
type byReason struct{ ConnInfos }
type SortByReason struct{ ConnInfos }
func (l byReason) Less(i, j int) bool {
func (l SortByReason) Less(i, j int) bool {
return l.ConnInfos[i].Reason < l.ConnInfos[j].Reason
}
// RTT - Default is descending
type byRTT struct{ ConnInfos }
type SortByRTT struct{ ConnInfos }
func (l byRTT) Less(i, j int) bool { return l.ConnInfos[i].rtt < l.ConnInfos[j].rtt }
func (l SortByRTT) Less(i, j int) bool { return l.ConnInfos[i].rtt < l.ConnInfos[j].rtt }
// IsValid determines if a sort option is valid
func (s SortOpt) IsValid() bool {
+497 -341
View File
File diff suppressed because it is too large Load Diff
+7
View File
@@ -367,6 +367,13 @@ func (c *client) initMsgTrace() *msgTrace {
}
}
dest = getHdrVal(MsgTraceDest)
if c.kind == CLIENT {
if td, ok := c.allowedMsgTraceDest(hdr, false); !ok {
return nil
} else if td != _EMPTY_ {
dest = td
}
}
// Check the destination to see if this is a valid public subject.
if !IsValidPublishSubject(dest) {
// We still have to return a msgTrace object (if traceOnly is set)
+31 -4
View File
@@ -389,6 +389,7 @@ type Options struct {
JetStreamRequestQueueLimit int64
JetStreamMetaCompact uint64
JetStreamMetaCompactSize uint64
JetStreamMetaCompactSync bool
StreamMaxBufferedMsgs int `json:"-"`
StreamMaxBufferedSize int64 `json:"-"`
StoreDir string `json:"-"`
@@ -595,6 +596,11 @@ type WebsocketOpts struct {
// time needed for the TLS Handshake.
HandshakeTimeout time.Duration
// How often to send pings to WebSocket clients. When set to a non-zero
// duration, this overrides the default PingInterval for WebSocket connections.
// If not set or zero, the server's default PingInterval will be used.
PingInterval time.Duration
// Headers to be added to the upgrade response.
// Useful for adding custom headers like Strict-Transport-Security.
Headers map[string]string
@@ -1263,7 +1269,9 @@ func (o *Options) processConfigFileLine(k string, v any, errors *[]error, warnin
case "proxy_protocol":
o.ProxyProtocol = v.(bool)
case "max_connections", "max_conn":
o.MaxConn = int(v.(int64))
if o.MaxConn = int(v.(int64)); o.MaxConn == 0 {
o.MaxConn = -1
}
case "max_traced_msg_len":
o.MaxTracedMsgLen = int(v.(int64))
case "max_subscriptions", "max_subs":
@@ -1685,7 +1693,7 @@ func (o *Options) processConfigFileLine(k string, v any, errors *[]error, warnin
case "reconnect_error_reports":
o.ReconnectErrorReports = int(v.(int64))
case "websocket", "ws":
if err := parseWebsocket(tk, o, errors); err != nil {
if err := parseWebsocket(tk, o, errors, warnings); err != nil {
*errors = append(*errors, err)
return
}
@@ -2321,7 +2329,7 @@ func parseJetStreamForAccount(v any, acc *Account, errors *[]error) error {
case "cluster_traffic":
vv, ok := mv.(string)
if !ok {
return &configErr{tk, fmt.Sprintf("Expected either 'system' or 'account' string value for %q, got %v", mk, mv)}
return &configErr{tk, fmt.Sprintf("Expected either 'system' or 'owner' string value for %q, got %v", mk, mv)}
}
switch vv {
case "system", _EMPTY_:
@@ -2648,6 +2656,8 @@ func parseJetStream(v any, opts *Options, errors *[]error, warnings *[]error) er
return &configErr{tk, fmt.Sprintf("Expected an absolute size for %q, got %v", mk, mv)}
}
opts.JetStreamMetaCompactSize = uint64(s)
case "meta_compact_sync":
opts.JetStreamMetaCompactSync = mv.(bool)
default:
if !tk.IsUsedVariable() {
err := &unknownConfigFieldErr{
@@ -5313,7 +5323,7 @@ func parseStringArray(fieldName string, tk token, lt *token, mv any, errors *[]e
}
}
func parseWebsocket(v any, o *Options, errors *[]error) error {
func parseWebsocket(v any, o *Options, errors *[]error, warnings *[]error) error {
var lt token
defer convertPanicToErrorList(&lt, errors)
@@ -5414,6 +5424,8 @@ func parseWebsocket(v any, o *Options, errors *[]error) error {
o.Websocket.Headers[key] = headerValue
}
}
case "ping_interval":
o.Websocket.PingInterval = parseDuration("ping_interval", tk, mv, errors, warnings)
default:
if !tk.IsUsedVariable() {
err := &unknownConfigFieldErr{
@@ -6423,3 +6435,18 @@ func expandPath(p string) (string, error) {
return filepath.Join(home, p[1:]), nil
}
// RedactArgs redacts sensitive arguments from the command line.
// For example, turns '--pass=secret' into '--pass=[REDACTED]'.
func RedactArgs(args []string) {
secret := regexp.MustCompile("^-{1,2}(user|pass|auth)(=.*)?$")
for i, arg := range args {
if secret.MatchString(arg) {
if idx := strings.Index(arg, "="); idx != -1 {
args[i] = arg[:idx] + "=[REDACTED]"
} else if i+1 < len(args) {
args[i+1] = "[REDACTED]"
}
}
}
}
+1
View File
@@ -32,6 +32,7 @@ type parseState struct {
msgBuf []byte
header http.Header // access via getHeader
scratch [MAX_CONTROL_LINE_SIZE]byte
argsa [MAX_HMSG_ARGS + 1][]byte // pre-allocated args array to avoid per-call heap escape
}
type pubArg struct {
+6
View File
@@ -63,8 +63,14 @@ func protoScanFieldValue(typ int, b []byte) (size int, err error) {
case 0:
_, size, err = protoScanVarint(b)
case 5: // fixed32
if len(b) < 4 {
return 0, errProtoInsufficient
}
size = 4
case 1: // fixed64
if len(b) < 8 {
return 0, errProtoInsufficient
}
size = 8
case 2: // length-delimited
size, err = protoScanBytes(b)
+614 -205
View File
File diff suppressed because it is too large Load Diff
+12 -12
View File
@@ -564,21 +564,20 @@ type maxConnOption struct {
// below the limit if necessary.
func (m *maxConnOption) Apply(server *Server) {
server.mu.Lock()
var (
clients = make([]*client, len(server.clients))
i = 0
)
clients := make([]*client, 0, len(server.clients))
// Map iteration is random, which allows us to close random connections.
for _, client := range server.clients {
clients[i] = client
i++
if isInternalClient(client.kind) {
continue
}
clients = append(clients, client)
}
server.mu.Unlock()
if m.newValue > 0 && len(clients) > m.newValue {
if newc := max(0, m.newValue); len(clients) > newc {
// Close connections til we are within the limit.
var (
numClose = len(clients) - m.newValue
numClose = len(clients) - newc
closed = 0
)
for _, client := range clients {
@@ -1659,7 +1658,7 @@ func (s *Server) diffOptions(newOpts *Options) ([]option, error) {
return nil, fmt.Errorf("config reload not supported for jetstream max memory and store")
}
}
case "jetstreammetacompact", "jetstreammetacompactsize":
case "jetstreammetacompact", "jetstreammetacompactsize", "jetstreammetacompactsync":
// Allowed at runtime but monitorCluster looks at s.opts directly, so no further work needed here.
case "websocket":
// Similar to gateways
@@ -2126,9 +2125,6 @@ func (s *Server) reloadAuthorization() {
resetCh <- struct{}{}
}
// Check that publish retained messages sources are still allowed to publish.
s.mqttCheckPubRetainedPerms()
// Close clients that have moved accounts
for _, client := range cclients {
client.closeConnection(ClientClosed)
@@ -2168,6 +2164,10 @@ func (s *Server) reloadAuthorization() {
s.Errorf(err.Error())
}
}
// Check that publish retained messages sources are still allowed to publish.
// Do this after dealing with JetStream.
s.mqttCheckPubRetainedPerms()
}
// Returns true if given client current account has changed (or user
+4 -7
View File
@@ -88,7 +88,7 @@ type route struct {
// an implicit route and sending to the remote.
gossipMode byte
// This will be set in case of pooling so that a route can trigger
// the creation of the next after receiving the first PONG, ensuring
// the creation of the next after receiving a PONG, ensuring
// that authentication did not fail.
startNewRoute *routeInfo
}
@@ -181,8 +181,7 @@ func (c *client) processAccountUnsub(arg []byte) {
// we have an origin cluster and we force header semantics.
func (c *client) processRoutedOriginClusterMsgArgs(arg []byte) error {
// Unroll splitArgs to avoid runtime/heap issues
a := [MAX_HMSG_ARGS + 1][]byte{}
args := a[:0]
args := c.argsa[:0]
start := -1
for i, b := range arg {
switch b {
@@ -280,8 +279,7 @@ func (c *client) processRoutedOriginClusterMsgArgs(arg []byte) error {
// Process an inbound HMSG specification from the remote route.
func (c *client) processRoutedHeaderMsgArgs(arg []byte) error {
// Unroll splitArgs to avoid runtime/heap issues
a := [MAX_HMSG_ARGS][]byte{}
args := a[:0]
args := c.argsa[:0]
var an []byte
if c.kind == ROUTER {
if an = c.route.accName; len(an) > 0 {
@@ -377,8 +375,7 @@ func (c *client) processRoutedHeaderMsgArgs(arg []byte) error {
// Process an inbound RMSG or LMSG specification from the remote route.
func (c *client) processRoutedMsgArgs(arg []byte) error {
// Unroll splitArgs to avoid runtime/heap issues
a := [MAX_RMSG_ARGS][]byte{}
args := a[:0]
args := c.argsa[:0]
var an []byte
if c.kind == ROUTER {
if an = c.route.accName; len(an) > 0 {
+3 -3
View File
@@ -3377,7 +3377,7 @@ func (s *Server) createClientEx(conn net.Conn, inProcess bool) *client {
// If there is a max connections specified, check that adding
// this new client would not push us over the max
if opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn {
if opts.MaxConn < 0 || (opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn) {
s.mu.Unlock()
c.maxConnExceeded()
return nil
@@ -3452,7 +3452,7 @@ func (s *Server) createClientEx(conn net.Conn, inProcess bool) *client {
pre = pre[:n]
}
conn = &tlsMixConn{conn, bytes.NewBuffer(pre)}
addr, err := readProxyProtoHeader(conn)
addr, proxyPre, err := readProxyProtoHeader(conn)
if err != nil && err != errProxyProtoUnrecognized {
// err != errProxyProtoUnrecognized implies that we detected a proxy
// protocol header but we failed to parse it, so don't continue.
@@ -3480,7 +3480,7 @@ func (s *Server) createClientEx(conn net.Conn, inProcess bool) *client {
// that it's a non-proxied connection and we want the pre-read to remain
// for the next step.
if err == nil {
pre = nil
pre = proxyPre
}
// Because we have ProxyProtocol enabled, our earlier INFO message didn't
// include the client_ip. If we need to send it again then we will include
+26 -18
View File
@@ -42,7 +42,6 @@ type winServiceWrapper struct {
}
var dockerized = false
var startupDelay = 10 * time.Second
func init() {
if v, exists := os.LookupEnv("NATS_DOCKERIZED"); exists && v == "1" {
@@ -67,6 +66,7 @@ func (w *winServiceWrapper) Execute(args []string, changes <-chan svc.ChangeRequ
status <- svc.Status{State: svc.StartPending}
go w.server.Start()
var startupDelay = 10 * time.Second
if v, exists := os.LookupEnv("NATS_STARTUP_DELAY"); exists {
if delay, err := time.ParseDuration(v); err == nil {
startupDelay = delay
@@ -86,24 +86,32 @@ func (w *winServiceWrapper) Execute(args []string, changes <-chan svc.ChangeRequ
}
loop:
for change := range changes {
switch change.Cmd {
case svc.Interrogate:
status <- change.CurrentStatus
case svc.Stop, svc.Shutdown:
w.server.Shutdown()
break loop
case reopenLogCmd:
// File log re-open for rotating file logs.
w.server.ReOpenLogFile()
case ldmCmd:
go w.server.lameDuckMode()
case svc.ParamChange:
if err := w.server.Reload(); err != nil {
w.server.Errorf("Failed to reload server configuration: %s", err)
for {
select {
case change, ok := <-changes:
if !ok {
break loop
}
default:
w.server.Debugf("Unexpected control request: %v", change.Cmd)
switch change.Cmd {
case svc.Interrogate:
status <- change.CurrentStatus
case svc.Stop, svc.Shutdown:
w.server.Shutdown()
break loop
case reopenLogCmd:
// File log re-open for rotating file logs.
w.server.ReOpenLogFile()
case ldmCmd:
go w.server.lameDuckMode()
case svc.ParamChange:
if err := w.server.Reload(); err != nil {
w.server.Errorf("Failed to reload server configuration: %s", err)
}
default:
w.server.Debugf("Unexpected control request: %v", change.Cmd)
}
case <-w.server.quitCh:
break loop
}
}
+6 -3
View File
@@ -65,6 +65,8 @@ var (
ErrCorruptStreamState = errors.New("stream state snapshot is corrupt")
// ErrTooManyResults
ErrTooManyResults = errors.New("too many matching results for request")
// ErrStoreOldUpdate is returned when a consumer update is older than the current state.
ErrStoreOldUpdate = errors.New("old update ignored")
)
// StoreMsg is the stored message format for messages that are retained by the Store layer.
@@ -99,6 +101,7 @@ type StreamStore interface {
LoadNextMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error)
LoadLastMsg(subject string, sm *StoreMsg) (*StoreMsg, error)
LoadPrevMsg(start uint64, smp *StoreMsg) (sm *StoreMsg, err error)
LoadPrevMsgMulti(sl *gsl.SimpleSublist, start uint64, smp *StoreMsg) (sm *StoreMsg, skip uint64, err error)
RemoveMsg(seq uint64) (bool, error)
EraseMsg(seq uint64) (bool, error)
Purge() (uint64, error)
@@ -112,8 +115,8 @@ type StreamStore interface {
AllLastSeqs() ([]uint64, error)
MultiLastSeqs(filters []string, maxSeq uint64, maxAllowed int) ([]uint64, error)
SubjectForSeq(seq uint64) (string, error)
NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64)
NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64)
NumPending(sseq uint64, filter string, lastPerSubject bool) (total, validThrough uint64, err error)
NumPendingMulti(sseq uint64, sl *gsl.SimpleSublist, lastPerSubject bool) (total, validThrough uint64, err error)
State() StreamState
FastState(*StreamState)
EncodedStreamState(failed uint64) (enc []byte, err error)
@@ -125,7 +128,7 @@ type StreamStore interface {
UpdateConfig(cfg *StreamConfig) error
Delete(inline bool) error
Stop() error
ConsumerStore(name string, cfg *ConsumerConfig) (ConsumerStore, error)
ConsumerStore(name string, created time.Time, cfg *ConsumerConfig) (ConsumerStore, error)
AddConsumer(o ConsumerStore) error
RemoveConsumer(o ConsumerStore) error
Snapshot(deadline time.Duration, includeConsumers, checkMsgs bool) (*SnapshotResult, error)
+274 -172
View File
@@ -1,4 +1,4 @@
// Copyright 2019-2025 The NATS Authors
// Copyright 2019-2026 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -425,9 +425,11 @@ type stream struct {
active bool // Indicates that there are active internal subscriptions (for the subject filters)
// and/or mirror/sources consumers are scheduled to be established or already started.
closed atomic.Bool // Set to true when stop() is called on the stream.
cisrun atomic.Bool // Indicates one checkInterestState is already running.
// Mirror
mirror *sourceInfo
mirror *sourceInfo
mirrorConsumerSetup *time.Timer
// Sources
sources map[string]*sourceInfo
@@ -618,19 +620,24 @@ const StreamMaxReplicas = 5
// AddStream adds a stream for the given account.
func (a *Account) addStream(config *StreamConfig) (*stream, error) {
return a.addStreamWithAssignment(config, nil, nil, false)
return a.addStreamWithAssignment(config, nil, nil, false, false)
}
// recoverStream recovers a stream from disk for the given account.
func (a *Account) recoverStream(config *StreamConfig) (*stream, error) {
return a.addStreamWithAssignment(config, nil, nil, false, true)
}
// AddStreamWithStore adds a stream for the given account with custome store config options.
func (a *Account) addStreamWithStore(config *StreamConfig, fsConfig *FileStoreConfig) (*stream, error) {
return a.addStreamWithAssignment(config, fsConfig, nil, false)
return a.addStreamWithAssignment(config, fsConfig, nil, false, false)
}
func (a *Account) addStreamPedantic(config *StreamConfig, pedantic bool) (*stream, error) {
return a.addStreamWithAssignment(config, nil, nil, pedantic)
return a.addStreamWithAssignment(config, nil, nil, pedantic, false)
}
func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileStoreConfig, sa *streamAssignment, pedantic bool) (*stream, error) {
func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileStoreConfig, sa *streamAssignment, pedantic, recovering bool) (*stream, error) {
s, jsa, err := a.checkForJetStream()
if err != nil {
return nil, err
@@ -677,6 +684,7 @@ func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileSt
}()
}
// Note that isClustered will be false during recovery, even if we're part of a cluster. It shouldn't be used then.
js, isClustered := jsa.jetStreamAndClustered()
jsa.mu.Lock()
if mset, ok := jsa.streams[cfg.Name]; ok {
@@ -706,25 +714,30 @@ func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileSt
jsa.usageMu.RLock()
selected, tier, hasTier := jsa.selectLimits(cfg.Replicas)
jsa.usageMu.RUnlock()
reserved := int64(0)
if !isClustered {
reserved = jsa.tieredReservation(tier, cfg)
}
jsa.mu.Unlock()
if !hasTier {
jsa.mu.Unlock()
return nil, NewJSNoLimitsError()
}
js.mu.RLock()
if isClustered {
_, reserved = tieredStreamAndReservationCount(js.cluster.streams[a.Name], tier, cfg)
}
if err := js.checkAllLimits(&selected, cfg, reserved, 0); err != nil {
// Skip if we're recovering.
if !recovering {
reserved := int64(0)
if !isClustered {
reserved = jsa.tieredReservation(tier, cfg)
}
jsa.mu.Unlock()
js.mu.RLock()
if isClustered {
_, reserved = js.tieredStreamAndReservationCount(a.Name, tier, cfg)
}
if err := js.checkAllLimits(&selected, cfg, reserved, 0); err != nil {
js.mu.RUnlock()
return nil, err
}
js.mu.RUnlock()
return nil, err
jsa.mu.Lock()
}
js.mu.RUnlock()
jsa.mu.Lock()
// Check for template ownership if present.
if cfg.Template != _EMPTY_ && jsa.account != nil {
if !jsa.checkTemplateOwnership(cfg.Template, cfg.Name) {
@@ -789,11 +802,6 @@ func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileSt
return nil, NewJSStreamSubjectOverlapError()
}
if !hasTier {
jsa.mu.Unlock()
return nil, fmt.Errorf("no applicable tier found")
}
// Setup the internal clients.
c := s.createInternalJetStreamClient()
ic := s.createInternalJetStreamClient()
@@ -827,11 +835,19 @@ func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileSt
ipqLimitByLen[*inMsg](mlen),
ipqLimitBySize[*inMsg](msz),
),
gets: newIPQueue[*directGetReq](s, qpfx+"direct gets"),
qch: make(chan struct{}),
mqch: make(chan struct{}),
uch: make(chan struct{}, 4),
sch: make(chan struct{}, 1),
gets: newIPQueue[*directGetReq](s, qpfx+"direct gets"),
qch: make(chan struct{}),
mqch: make(chan struct{}),
uch: make(chan struct{}, 4),
sch: make(chan struct{}, 1),
created: time.Now().UTC(),
}
// Add created timestamp used for the store, must match that of the stream assignment if it exists.
if sa != nil {
// The following assignment does not require mutex
// protection: sa.Created is immutable.
mset.created = sa.Created
}
// Start our signaling routine to process consumers.
@@ -895,7 +911,6 @@ func (a *Account) addStreamWithAssignment(config *StreamConfig, fsConfig *FileSt
fsCfg.SyncAlways = false
fsCfg.AsyncFlush = true
}
if err := mset.setupStore(fsCfg); err != nil {
mset.stop(true, false)
return nil, NewJSStreamStoreFailedError(err)
@@ -1080,8 +1095,12 @@ func (mset *stream) monitorQuitC() <-chan struct{} {
if mset == nil {
return nil
}
mset.mu.RLock()
defer mset.mu.RUnlock()
mset.mu.Lock()
defer mset.mu.Unlock()
// Recreate if a prior monitor routine was stopped.
if mset.mqch == nil {
mset.mqch = make(chan struct{})
}
return mset.mqch
}
@@ -1811,10 +1830,10 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic boo
}
}
// check for duplicates
// check sources for duplicates
var iNames = make(map[string]struct{})
for _, src := range cfg.Sources {
if !isValidName(src.Name) {
if src == nil || !isValidName(src.Name) {
return StreamConfig{}, NewJSSourceInvalidStreamNameError()
}
if _, ok := iNames[src.composeIName()]; !ok {
@@ -1822,6 +1841,30 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic boo
} else {
return StreamConfig{}, NewJSSourceDuplicateDetectedError()
}
if src.FilterSubject != _EMPTY_ && len(src.SubjectTransforms) != 0 {
return StreamConfig{}, NewJSSourceMultipleFiltersNotAllowedError()
}
for _, tr := range src.SubjectTransforms {
if tr.Source != _EMPTY_ && !IsValidSubject(tr.Source) {
return StreamConfig{}, NewJSSourceInvalidSubjectFilterError(fmt.Errorf("%w %s", ErrBadSubject, tr.Source))
}
err := ValidateMapping(tr.Source, tr.Destination)
if err != nil {
return StreamConfig{}, NewJSSourceInvalidTransformDestinationError(err)
}
}
// Check subject filters overlap.
for outer, tr := range src.SubjectTransforms {
for inner, innertr := range src.SubjectTransforms {
if inner != outer && subjectIsSubsetMatch(tr.Source, innertr.Source) {
return StreamConfig{}, NewJSSourceOverlappingSubjectFiltersError()
}
}
}
// Do not perform checks if External is provided, as it could lead to
// checking against itself (if sourced stream name is the same on different JetStream)
if src.External == nil {
@@ -1834,30 +1877,6 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic boo
return StreamConfig{}, NewJSSourceMaxMessageSizeTooBigError()
}
}
if src.FilterSubject != _EMPTY_ && len(src.SubjectTransforms) != 0 {
return StreamConfig{}, NewJSSourceMultipleFiltersNotAllowedError()
}
for _, tr := range src.SubjectTransforms {
if tr.Source != _EMPTY_ && !IsValidSubject(tr.Source) {
return StreamConfig{}, NewJSSourceInvalidSubjectFilterError(fmt.Errorf("%w %s", ErrBadSubject, tr.Source))
}
err := ValidateMapping(tr.Source, tr.Destination)
if err != nil {
return StreamConfig{}, NewJSSourceInvalidTransformDestinationError(err)
}
}
// Check subject filters overlap.
for outer, tr := range src.SubjectTransforms {
for inner, innertr := range src.SubjectTransforms {
if inner != outer && subjectIsSubsetMatch(tr.Source, innertr.Source) {
return StreamConfig{}, NewJSSourceOverlappingSubjectFiltersError()
}
}
}
continue
} else {
if src.External.DeliverPrefix != _EMPTY_ {
@@ -1949,7 +1968,7 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic boo
// Check for literal duplication of subject interest in config
// and no overlap with any JS or SYS API subject space.
dset := make(map[string]struct{}, len(cfg.Subjects))
for _, subj := range cfg.Subjects {
for i, subj := range cfg.Subjects {
// Make sure the subject is valid. Check this first.
if !IsValidSubject(subj) {
return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("invalid subject"))
@@ -1983,6 +2002,13 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic boo
}
}
}
// Now check if we have multiple subjects that we do not overlap ourselves
// which would cause duplicate entries (assuming no MsgID).
for _, tsubj := range cfg.Subjects[i+1:] {
if SubjectsCollide(tsubj, subj) {
return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("subject %q overlaps with %q", subj, tsubj))
}
}
// Mark for duplicate check.
dset[subj] = struct{}{}
}
@@ -2000,18 +2026,6 @@ func (s *Server) checkStreamCfg(config *StreamConfig, acc *Account, pedantic boo
return StreamConfig{}, NewJSStreamMaxStreamBytesExceededError()
}
// Now check if we have multiple subjects they we do not overlap ourselves
// which would cause duplicate entries (assuming no MsgID).
if len(cfg.Subjects) > 1 {
for _, subj := range cfg.Subjects {
for _, tsubj := range cfg.Subjects {
if tsubj != subj && SubjectsCollide(tsubj, subj) {
return StreamConfig{}, NewJSStreamInvalidConfigError(fmt.Errorf("subject %q overlaps with %q", subj, tsubj))
}
}
}
}
// Check the subject transform if any
if cfg.SubjectTransform != nil {
if cfg.SubjectTransform.Source != _EMPTY_ && !IsValidSubject(cfg.SubjectTransform.Source) {
@@ -2102,10 +2116,6 @@ func (jsa *jsAccount) configUpdateCheck(old, new *StreamConfig, s *Server, pedan
if cfg.Name != old.Name {
return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration name must match original"))
}
// Can't change MaxConsumers for now.
if cfg.MaxConsumers != old.MaxConsumers {
return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not change MaxConsumers"))
}
// Can't change storage types.
if cfg.Storage != old.Storage {
return nil, NewJSStreamInvalidConfigError(fmt.Errorf("stream configuration update can not change storage type"))
@@ -2222,13 +2232,15 @@ func (jsa *jsAccount) configUpdateCheck(old, new *StreamConfig, s *Server, pedan
js.mu.RLock()
defer js.mu.RUnlock()
if isClustered {
_, reserved = tieredStreamAndReservationCount(js.cluster.streams[acc.Name], tier, &cfg)
_, reserved = js.tieredStreamAndReservationCount(acc.Name, tier, &cfg)
}
// reservation does not account for this stream, hence add the old value
if tier == _EMPTY_ && old.Replicas > 1 {
reserved += old.MaxBytes * int64(old.Replicas)
} else {
reserved += old.MaxBytes
if old.MaxBytes > 0 {
if tier == _EMPTY_ && old.Replicas > 1 {
reserved = addSaturate(reserved, mulSaturate(int64(old.Replicas), old.MaxBytes))
} else {
reserved = addSaturate(reserved, old.MaxBytes)
}
}
if err := js.checkAllLimits(&selected, &cfg, reserved, maxBytesOffset); err != nil {
return nil, err
@@ -2797,6 +2809,12 @@ func (mset *stream) processMirrorMsgs(mirror *sourceInfo, ready *sync.WaitGroup)
// Grab stream quit channel.
mset.mu.Lock()
msgs, qch, siqch := mirror.msgs, mset.qch, mirror.qch
// If the mirror was already canceled before we got here, exit early.
if siqch == nil {
mset.mu.Unlock()
ready.Done()
return
}
// Set the last seen as now so that we don't fail at the first check.
mirror.last.Store(time.Now().UnixNano())
mset.mu.Unlock()
@@ -3111,7 +3129,8 @@ func (mset *stream) scheduleSetupMirrorConsumerRetry() {
// Add some jitter.
next += time.Duration(rand.Intn(int(100*time.Millisecond))) + 100*time.Millisecond
time.AfterFunc(next, func() {
stopAndClearTimer(&mset.mirrorConsumerSetup)
mset.mirrorConsumerSetup = time.AfterFunc(next, func() {
mset.mu.Lock()
mset.setupMirrorConsumer()
mset.mu.Unlock()
@@ -3155,7 +3174,6 @@ func (mset *stream) setupMirrorConsumer() error {
}
mirror := mset.mirror
mirrorWg := &mirror.wg
// We want to throttle here in terms of how fast we request new consumers,
// or if the previous is still in progress.
@@ -3314,7 +3332,16 @@ func (mset *stream) setupMirrorConsumer() error {
// Wait for previous processMirrorMsgs go routine to be completely done.
// If none is running, this will not block.
mirrorWg.Wait()
mset.mu.Lock()
if mset.mirror == nil {
// Mirror config has been removed.
mset.mu.Unlock()
return
} else {
wg := &mset.mirror.wg
mset.mu.Unlock()
wg.Wait()
}
select {
case ccr := <-respCh:
@@ -3393,6 +3420,7 @@ func (mset *stream) setupMirrorConsumer() error {
"consumer": mirror.cname,
},
) {
mirror.wg.Done()
ready.Done()
}
}
@@ -3955,7 +3983,6 @@ func (mset *stream) processInboundSourceMsg(si *sourceInfo, m *inMsg) bool {
} else {
err = mset.processJetStreamMsg(m.subj, _EMPTY_, hdr, msg, 0, 0, nil, true, true)
}
if err != nil {
s := mset.srv
if strings.Contains(err.Error(), "no space left") {
@@ -3965,31 +3992,35 @@ func (mset *stream) processInboundSourceMsg(si *sourceInfo, m *inMsg) bool {
mset.mu.RLock()
accName, sname, iName := mset.acc.Name, mset.cfg.Name, si.iname
mset.mu.RUnlock()
// Can happen temporarily all the time during normal operations when the sourcing stream
// is working queue/interest with a limit and discard new.
// TODO - Improve sourcing to WQ with limit and new to use flow control rather than re-creating the consumer.
if errors.Is(err, ErrMaxMsgs) || errors.Is(err, ErrMaxBytes) {
// Can happen temporarily all the time during normal operations when the sourcing stream is discard new
// (example use case is for sourcing into a work queue)
// TODO - Maybe improve sourcing to WQ with limit and new to use flow control rather than re-creating the consumer.
if errors.Is(err, ErrMaxMsgs) || errors.Is(err, ErrMaxBytes) || errors.Is(err, ErrMaxMsgsPerSubject) {
// Do not need to do a full retry that includes finding the last sequence in the stream
// for that source. Just re-create starting with the seq we couldn't store instead.
mset.mu.Lock()
mset.retrySourceConsumerAtSeq(iName, si.sseq)
mset.mu.Unlock()
} else {
// Log some warning for errors other than errLastSeqMismatch or errMaxMsgs.
if !errors.Is(err, errLastSeqMismatch) {
// Log some warning for errors other than errLastSeqMismatch.
if !errors.Is(err, errLastSeqMismatch) && !errors.Is(err, errMsgIdDuplicate) {
s.RateLimitWarnf("Error processing inbound source %q for '%s' > '%s': %v",
iName, accName, sname, err)
}
// Retry in all type of errors if we are still leader.
// Retry in all type of errors we do not want to skip if we are still leader.
if mset.isLeader() {
// This will make sure the source is still in mset.sources map,
// find the last sequence and then call setupSourceConsumer.
iNameMap := map[string]struct{}{iName: {}}
mset.setStartingSequenceForSources(iNameMap)
mset.mu.Lock()
mset.retrySourceConsumerAtSeq(iName, si.sseq+1)
mset.mu.Unlock()
if !errors.Is(err, errMsgIdDuplicate) {
// This will make sure the source is still in mset.sources map,
// find the last sequence and then call setupSourceConsumer.
iNameMap := map[string]struct{}{iName: {}}
mset.setStartingSequenceForSources(iNameMap)
mset.mu.Lock()
mset.retrySourceConsumerAtSeq(iName, si.sseq+1)
mset.mu.Unlock()
} else {
// skipping the message but keep processing the rest of the batch
return true
}
}
}
}
@@ -4086,28 +4117,61 @@ func (mset *stream) setStartingSequenceForSources(iNames map[string]struct{}) {
return
}
// From the provided list of sources, we build a sublist that contains
// the interested filters (including transforms). As we figure out the
// starting sequence for each source, we will eliminate the source from
// the map and then refresh the sublist, which in turn makes the sublist
// ideally more specific. This allows LoadPrevMsgsMulti to work most
// effectively.
// Because this is a SimpleSublist we can't just remove the entries per
// source so we have no other option but to rebuild it from scratch, but
// this is cheap enough to do so not the end of the world.
var sl *gsl.SimpleSublist
refreshSublist := func() {
sl = gsl.NewSimpleSublist()
for iName := range iNames {
si := mset.sources[iName]
if si == nil {
continue
}
if si.sf == _EMPTY_ {
sl.Insert(fwcs, struct{}{})
} else {
sl.Insert(si.sf, struct{}{})
}
for _, sf := range si.sfs {
if sf == _EMPTY_ {
sl.Insert(fwcs, struct{}{})
} else {
sl.Insert(sf, struct{}{})
}
}
}
}
refreshSublist()
var smv StoreMsg
for seq := state.LastSeq; seq >= state.FirstSeq; {
sm, err := mset.store.LoadPrevMsg(seq, &smv)
for last := state.LastSeq; ; {
sm, seq, err := mset.store.LoadPrevMsgMulti(sl, last, &smv)
if err == ErrStoreEOF || err != nil {
break
}
seq = sm.seq - 1
last = seq - 1
if len(sm.hdr) == 0 {
continue
}
ss := getHeader(JSStreamSource, sm.hdr)
ss := sliceHeader(JSStreamSource, sm.hdr)
if len(ss) == 0 {
continue
}
streamName, indexName, sseq := streamAndSeq(bytesToString(ss))
streamName, indexName, sseq := streamAndSeq(bytesToString(ss))
if _, ok := iNames[indexName]; ok {
si := mset.sources[indexName]
si.sseq = sseq
si.dseq = 0
delete(iNames, indexName)
refreshSublist()
} else if indexName == _EMPTY_ && streamName != _EMPTY_ {
for iName := range iNames {
// TODO streamSource is a linear walk, to optimize later
@@ -4116,6 +4180,7 @@ func (mset *stream) setStartingSequenceForSources(iNames map[string]struct{}) {
si.sseq = sseq
si.dseq = 0
delete(iNames, iName)
refreshSublist()
break
}
}
@@ -4197,26 +4262,61 @@ func (mset *stream) startingSequenceForSources() {
}
}()
// Generate a list of sources and, from that, a sublist that contains
// the interested filters (including transforms). As we figure out the
// starting sequence for each source, we will eliminate the source from
// the map and then refresh the sublist, which in turn makes the sublist
// ideally more specific. This allows LoadPrevMsgsMulti to work most
// effectively.
// Because this is a SimpleSublist we can't just remove the entries per
// source so we have no other option but to rebuild it from scratch, but
// this is cheap enough to do so not the end of the world.
sources := map[string]*StreamSource{}
for _, src := range mset.cfg.Sources {
sources[src.composeIName()] = src
}
var sl *gsl.SimpleSublist
refreshSublist := func() {
sl = gsl.NewSimpleSublist()
for _, src := range sources {
if src.FilterSubject == _EMPTY_ {
sl.Insert(fwcs, struct{}{})
} else {
sl.Insert(src.FilterSubject, struct{}{})
}
for _, tr := range src.SubjectTransforms {
if tr.Destination == _EMPTY_ {
sl.Insert(fwcs, struct{}{})
} else {
sl.Insert(tr.Destination, struct{}{})
}
}
}
}
refreshSublist()
update := func(iName string, seq uint64) {
// Only update active in case we have older ones in here that got configured out.
if si := mset.sources[iName]; si != nil {
if _, ok := seqs[iName]; !ok {
seqs[iName] = seq
delete(sources, iName)
refreshSublist()
}
}
}
var smv StoreMsg
for seq := state.LastSeq; ; {
sm, err := mset.store.LoadPrevMsg(seq, &smv)
for last := state.LastSeq; ; {
sm, seq, err := mset.store.LoadPrevMsgMulti(sl, last, &smv)
if err == ErrStoreEOF || err != nil {
break
}
seq = sm.seq - 1
last = seq - 1
if len(sm.hdr) == 0 {
continue
}
ss := getHeader(JSStreamSource, sm.hdr)
ss := sliceHeader(JSStreamSource, sm.hdr)
if len(ss) == 0 {
continue
}
@@ -4546,8 +4646,6 @@ func (mset *stream) unsubscribe(sub *subscription) {
func (mset *stream) setupStore(fsCfg *FileStoreConfig) error {
mset.mu.Lock()
mset.created = time.Now().UTC()
switch mset.cfg.Storage {
case MemoryStorage:
ms, err := newMemStore(&mset.cfg)
@@ -5208,7 +5306,10 @@ func (mset *stream) getDirectRequest(req *JSApiMsgGetRequest, reply string) {
} else {
// This is a batch request, capture initial numPending.
isBatchRequest = true
np, validThrough = store.NumPending(seq, req.NextFor, false)
var err error
if np, validThrough, err = store.NumPending(seq, req.NextFor, false); err != nil {
return
}
}
// Grab MaxBytes
@@ -5301,7 +5402,10 @@ func (mset *stream) getDirectRequest(req *JSApiMsgGetRequest, reply string) {
if isBatchRequest {
// Update if the stream's last sequence has moved past our validThrough.
if mset.lseq > validThrough {
np, _ = store.NumPending(seq, req.NextFor, false)
var err error
if np, _, err = store.NumPending(seq, req.NextFor, false); err != nil {
return
}
}
hdr := fmt.Appendf(nil, eob, np, lseq)
mset.outq.send(newJSPubMsg(reply, _EMPTY_, _EMPTY_, hdr, nil, nil, 0))
@@ -5311,6 +5415,7 @@ func (mset *stream) getDirectRequest(req *JSApiMsgGetRequest, reply string) {
// processInboundJetStreamMsg handles processing messages bound for a stream.
func (mset *stream) processInboundJetStreamMsg(_ *subscription, c *client, _ *Account, subject, reply string, rmsg []byte) {
hdr, msg := c.msgParts(copyBytes(rmsg)) // Need to copy.
hdr = removeHeaderStatusIfPresent(hdr)
if mt, traceOnly := c.isMsgTraceEnabled(); mt != nil {
// If message is delivered, we need to disable the message trace headers
// to prevent a trace event to be generated when a stored message
@@ -5706,7 +5811,7 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte,
mset.ddMu.Unlock()
if seq > 0 {
if canRespond {
response := append(pubAck, strconv.FormatUint(dde.seq, 10)...)
response := append(pubAck, strconv.FormatUint(seq, 10)...)
response = append(response, ",\"duplicate\": true}"...)
outq.sendMsg(reply, response)
}
@@ -5965,13 +6070,6 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte,
return nil
}
// If here we will attempt to store the message.
// Assume this will succeed.
olmsgId := mset.lmsgId
mset.lmsgId = msgId
mset.lseq++
tierName := mset.tier
// Republish state if needed.
var tsubj string
var tlseq uint64
@@ -5995,7 +6093,7 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte,
// If clustered this was already checked and we do not want to check here and possibly introduce skew.
// Don't error and log if we're tracing when clustered.
if !isClustered {
if exceeded, err := jsa.wouldExceedLimits(stype, tierName, mset.cfg.Replicas, subject, hdr, msg); exceeded {
if exceeded, err := jsa.wouldExceedLimits(stype, mset.tier, mset.cfg.Replicas, subject, hdr, msg); exceeded {
if err == nil {
err = NewJSAccountResourcesExceededError()
}
@@ -6054,11 +6152,7 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte,
mset.srv.Warnf("Filesystem permission denied while writing msg, disabling JetStream: %v", err)
return err
}
// If we did not succeed put those values back and increment clfs in case we are clustered.
var state StreamState
mset.store.FastState(&state)
mset.lseq = state.LastSeq
mset.lmsgId = olmsgId
// If we did not succeed increment clfs in case we are clustered.
bumpCLFS()
switch err {
@@ -6079,6 +6173,8 @@ func (mset *stream) processJetStreamMsg(subject, reply string, hdr, msg []byte,
}
// If here we succeeded in storing the message.
mset.lmsgId = msgId
mset.lseq = seq
// If we have a msgId make sure to save.
// This will replace our estimate from the cluster layer if we are clustered.
@@ -6673,36 +6769,31 @@ type jsPubMsg struct {
o *consumer
}
var jsPubMsgPool sync.Pool
var jsPubMsgPool = sync.Pool{
New: func() any {
return &jsPubMsg{}
},
}
func newJSPubMsg(dsubj, subj, reply string, hdr, msg []byte, o *consumer, seq uint64) *jsPubMsg {
var m *jsPubMsg
var buf []byte
pm := jsPubMsgPool.Get()
if pm != nil {
m = pm.(*jsPubMsg)
buf = m.buf[:0]
if hdr != nil {
hdr = append(m.hdr[:0], hdr...)
}
} else {
m = new(jsPubMsg)
m := getJSPubMsgFromPool()
if m.buf == nil {
m.buf = make([]byte, 0, len(hdr)+len(msg))
}
buf := append(m.buf[:0], hdr...)
buf = append(buf, msg...)
hdr = buf[:len(hdr):len(hdr)]
msg = buf[len(hdr):]
// When getting something from a pool it is critical that all fields are
// initialized. Doing this way guarantees that if someone adds a field to
// the structure, the compiler will fail the build if this line is not updated.
(*m) = jsPubMsg{dsubj, reply, StoreMsg{subj, hdr, msg, buf, seq, 0}, o}
return m
}
// Gets a jsPubMsg from the pool.
func getJSPubMsgFromPool() *jsPubMsg {
pm := jsPubMsgPool.Get()
if pm != nil {
return pm.(*jsPubMsg)
}
return new(jsPubMsg)
return jsPubMsgPool.Get().(*jsPubMsg)
}
func (pm *jsPubMsg) returnToPool() {
@@ -6713,9 +6804,6 @@ func (pm *jsPubMsg) returnToPool() {
if len(pm.buf) > 0 {
pm.buf = pm.buf[:0]
}
if len(pm.hdr) > 0 {
pm.hdr = pm.hdr[:0]
}
jsPubMsgPool.Put(pm)
}
@@ -7218,11 +7306,30 @@ func (mset *stream) checkInterestState() {
return
}
// Ensure only one of these runs at the same time.
if !mset.cisrun.CompareAndSwap(false, true) {
return
}
defer mset.cisrun.Store(false)
var ss StreamState
mset.store.FastState(&ss)
asflr := uint64(math.MaxUint64)
for _, o := range mset.getConsumers() {
o.checkStateForInterestStream(&ss)
o.mu.RLock()
chkflr := o.chkflr
o.mu.RUnlock()
asflr = min(asflr, chkflr)
}
mset.cfgMu.RLock()
rp := mset.cfg.Retention
mset.cfgMu.RUnlock()
// Remove as many messages from the "head" of the stream if there's no interest anymore.
if rp == InterestPolicy && asflr != math.MaxUint64 {
mset.store.Compact(asflr)
}
}
@@ -7309,20 +7416,18 @@ func (mset *stream) swapSigSubs(o *consumer, newFilters []string) {
o.sigSubs = nil
}
if o.isLeader() {
if mset.csl == nil {
mset.csl = gsl.NewSublist[*consumer]()
}
// If no filters are preset, add fwcs to sublist for that consumer.
if newFilters == nil {
mset.csl.Insert(fwcs, o)
o.sigSubs = append(o.sigSubs, fwcs)
// If there are filters, add their subjects to sublist.
} else {
for _, filter := range newFilters {
mset.csl.Insert(filter, o)
o.sigSubs = append(o.sigSubs, filter)
}
if mset.csl == nil {
mset.csl = gsl.NewSublist[*consumer]()
}
// If no filters are present, add fwcs to sublist for that consumer.
if newFilters == nil {
mset.csl.Insert(fwcs, o)
o.sigSubs = append(o.sigSubs, fwcs)
} else {
// If there are filters, add their subjects to sublist.
for _, filter := range newFilters {
mset.csl.Insert(filter, o)
o.sigSubs = append(o.sigSubs, filter)
}
}
o.mu.Unlock()
@@ -7399,14 +7504,18 @@ func (mset *stream) partitionUnique(name string, partitions []string) bool {
if n == name {
continue
}
o.mu.RLock()
if o.subjf == nil {
o.mu.RUnlock()
return false
}
for _, filter := range o.subjf {
if SubjectsCollide(partition, filter.subject) {
o.mu.RUnlock()
return false
}
}
o.mu.RUnlock()
}
}
return true
@@ -7590,15 +7699,8 @@ func (mset *stream) ackMsg(o *consumer, seq uint64) bool {
return false
}
var shouldRemove bool
switch mset.cfg.Retention {
case WorkQueuePolicy:
// Normally we just remove a message when its ack'd here but if we have direct consumers
// from sources and/or mirrors we need to make sure they have delivered the msg.
shouldRemove = mset.directs <= 0 || mset.noInterest(seq, o)
case InterestPolicy:
shouldRemove = mset.noInterest(seq, o)
}
// If there's no interest left on this message for all consumers, we can remove it.
shouldRemove := mset.noInterest(seq, nil)
// If nothing else to do.
if !shouldRemove {
@@ -7709,7 +7811,7 @@ func (a *Account) RestoreStream(ncfg *StreamConfig, r io.Reader) (*stream, error
if hasTier {
if isClustered {
js.mu.RLock()
_, reserved = tieredStreamAndReservationCount(js.cluster.streams[a.Name], tier, &cfg)
_, reserved = js.tieredStreamAndReservationCount(a.Name, tier, &cfg)
js.mu.RUnlock()
} else {
reserved = jsa.tieredReservation(tier, &cfg)
+4 -1
View File
@@ -36,7 +36,6 @@ func genParts(filter []byte, parts [][]byte) [][]byte {
}
start = i + 1
} else if i < e && filter[i+1] == fwc && i+1 == e {
// We have a fwc
if i > start {
parts = append(parts, filter[start:i+1])
}
@@ -53,6 +52,10 @@ func genParts(filter []byte, parts [][]byte) [][]byte {
if next := i + 1; next == e || next < e && filter[next] != tsep {
continue
}
// Full wildcard must be terminal.
if filter[i] == fwc && i < e {
break
}
// We start with a pwc or fwc.
parts = append(parts, filter[i:i+1])
if i+1 <= e {
+102 -12
View File
@@ -16,6 +16,9 @@ package stree
import (
"bytes"
"slices"
"unsafe"
"github.com/nats-io/nats-server/v2/server/gsl"
)
// SubjectTree is an adaptive radix trie (ART) for storing subject information on literal subjects.
@@ -121,7 +124,25 @@ func (t *SubjectTree[T]) Match(filter []byte, cb func(subject []byte, val *T)) {
var raw [16][]byte
parts := genParts(filter, raw[:0])
var _pre [256]byte
t.match(t.root, parts, _pre[:0], cb)
t.match(t.root, parts, _pre[:0], func(subject []byte, val *T) bool {
cb(subject, val)
return true
})
}
// MatchUntil will match against a subject that can have wildcards and invoke
// the callback func for each matched value.
// Returning false from the callback will stop matching immediately.
// Returns true if matching ran to completion, false if callback stopped it early.
func (t *SubjectTree[T]) MatchUntil(filter []byte, cb func(subject []byte, val *T) bool) bool {
if t == nil || t.root == nil || len(filter) == 0 || cb == nil {
return true
}
// We need to break this up into chunks based on wildcards, either pwc '*' or fwc '>'.
var raw [16][]byte
parts := genParts(filter, raw[:0])
var _pre [256]byte
return t.match(t.root, parts, _pre[:0], cb)
}
// IterOrdered will walk all entries in the SubjectTree lexicographically. The callback can return false to terminate the walk.
@@ -293,7 +314,8 @@ func (t *SubjectTree[T]) delete(np *node, subject []byte, si int) (*T, bool) {
// Internal function which can be called recursively to match all leaf nodes to a given filter subject which
// once here has been decomposed to parts. These parts only care about wildcards, both pwc and fwc.
func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subject []byte, val *T)) {
// Returns false if the callback requested to stop matching.
func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subject []byte, val *T) bool) bool {
// Capture if we are sitting on a terminal fwc.
var hasFWC bool
if lp := len(parts); lp > 0 && len(parts[lp-1]) > 0 && parts[lp-1][0] == fwc {
@@ -304,15 +326,17 @@ func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subje
nparts, matched := n.matchParts(parts)
// Check if we did not match.
if !matched {
return
return true
}
// We have matched here. If we are a leaf and have exhausted all parts or he have a FWC fire callback.
if n.isLeaf() {
if len(nparts) == 0 || (hasFWC && len(nparts) == 1) {
ln := n.(*leaf[T])
cb(append(pre, ln.suffix...), &ln.value)
if !cb(append(pre, ln.suffix...), &ln.value) {
return false
}
}
return
return true
}
// We have normal nodes here.
// We need to append our prefix
@@ -340,17 +364,23 @@ func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subje
if cn.isLeaf() {
ln := cn.(*leaf[T])
if len(ln.suffix) == 0 {
cb(append(pre, ln.suffix...), &ln.value)
if !cb(append(pre, ln.suffix...), &ln.value) {
return false
}
} else if hasTermPWC && bytes.IndexByte(ln.suffix, tsep) < 0 {
cb(append(pre, ln.suffix...), &ln.value)
if !cb(append(pre, ln.suffix...), &ln.value) {
return false
}
}
} else if hasTermPWC {
// We have terminal pwc so call into match again with the child node.
t.match(cn, nparts, pre, cb)
if !t.match(cn, nparts, pre, cb) {
return false
}
}
}
// Return regardless.
return
return true
}
// If we are sitting on a terminal fwc, put back and continue.
if hasFWC && len(nparts) == 0 {
@@ -367,18 +397,21 @@ func (t *SubjectTree[T]) match(n node, parts [][]byte, pre []byte, cb func(subje
// to see if we match further down.
for _, cn := range n.children() {
if cn != nil {
t.match(cn, nparts, pre, cb)
if !t.match(cn, nparts, pre, cb) {
return false
}
}
}
return
return true
}
// Here we have normal traversal, so find the next child.
nn := n.findChild(p)
if nn == nil {
return
return true
}
n, parts = *nn, nparts
}
return true
}
// Internal iter function to walk nodes in lexicographical order.
@@ -448,3 +481,60 @@ func LazyIntersect[TL, TR any](tl *SubjectTree[TL], tr *SubjectTree[TR], cb func
})
}
}
// IntersectGSL will match all items in the given subject tree that
// have interest expressed in the given sublist. The callback will only be called
// once for each subject, regardless of overlapping subscriptions in the sublist.
func IntersectGSL[T any, SL comparable](t *SubjectTree[T], sl *gsl.GenericSublist[SL], cb func(subject []byte, val *T)) {
if t == nil || t.root == nil || sl == nil {
return
}
var _pre [256]byte
_intersectGSL(t.root, _pre[:0], sl, cb)
}
func _intersectGSL[T any, SL comparable](n node, pre []byte, sl *gsl.GenericSublist[SL], cb func(subject []byte, val *T)) {
if n.isLeaf() {
ln := n.(*leaf[T])
subj := append(pre, ln.suffix...)
if sl.HasInterest(bytesToString(subj)) {
cb(subj, &ln.value)
}
return
}
bn := n.base()
pre = append(pre, bn.prefix...)
for _, cn := range n.children() {
if cn == nil {
continue
}
subj := append(pre, cn.path()...)
if !hasInterestForTokens(sl, subj, len(pre)) {
continue
}
_intersectGSL(cn, pre, sl, cb)
}
}
// The subject tree can return partial tokens so we need to check starting interest
// only from whole tokens when we encounter a tsep.
func hasInterestForTokens[SL comparable](sl *gsl.GenericSublist[SL], subj []byte, since int) bool {
for i := since; i < len(subj); i++ {
if subj[i] == tsep {
if !sl.HasInterestStartingIn(bytesToString(subj[:i])) {
return false
}
}
}
return true
}
// Note this will avoid a copy of the data used for the string, but it will also reference the existing slice's data pointer.
// So this should be used sparingly when we know the encompassing byte slice's lifetime is the same.
func bytesToString(b []byte) string {
if len(b) == 0 {
return ""
}
p := unsafe.SliceData(b)
return unsafe.String(p, len(b))
}
+8 -67
View File
@@ -21,8 +21,6 @@ import (
"sync"
"sync/atomic"
"unicode/utf8"
"github.com/nats-io/nats-server/v2/server/stree"
)
// Sublist is a routing mechanism to handle subject distribution and
@@ -818,7 +816,9 @@ func matchLevelForAny(l *level, toks []string, np, nq *int) bool {
*nq += len(qsub)
}
}
return len(n.plist) > 0 || len(n.psubs) > 0 || len(n.qsubs) > 0
if len(n.plist) > 0 || len(n.psubs) > 0 || len(n.qsubs) > 0 {
return true
}
}
if pwc != nil {
if np != nil && nq != nil {
@@ -1327,8 +1327,9 @@ func SubjectsCollide(subj1, subj2 string) bool {
if subj1 == subj2 {
return true
}
toks1 := strings.Split(subj1, tsep)
toks2 := strings.Split(subj2, tsep)
tsa, tsb := [32]string{}, [32]string{}
toks1 := tokenizeSubjectIntoSlice(tsa[:0], subj1)
toks2 := tokenizeSubjectIntoSlice(tsb[:0], subj2)
pwc1, fwc1 := analyzeTokens(toks1)
pwc2, fwc2 := analyzeTokens(toks2)
// if both literal just string compare.
@@ -1338,9 +1339,9 @@ func SubjectsCollide(subj1, subj2 string) bool {
}
// So one or both have wildcards. If one is literal than we can do subset matching.
if l1 && !l2 {
return isSubsetMatch(toks1, subj2)
return isSubsetMatchTokenized(toks1, toks2)
} else if l2 && !l1 {
return isSubsetMatch(toks2, subj1)
return isSubsetMatchTokenized(toks2, toks1)
}
// Both have wildcards.
// If they only have partials then the lengths must match.
@@ -1726,63 +1727,3 @@ func getAllNodes(l *level, results *SublistResult) {
getAllNodes(n.next, results)
}
}
// IntersectStree will match all items in the given subject tree that
// have interest expressed in the given sublist. The callback will only be called
// once for each subject, regardless of overlapping subscriptions in the sublist.
func IntersectStree[T any](st *stree.SubjectTree[T], sl *Sublist, cb func(subj []byte, entry *T)) {
var _subj [255]byte
intersectStree(st, sl.root, _subj[:0], cb)
}
func intersectStree[T any](st *stree.SubjectTree[T], r *level, subj []byte, cb func(subj []byte, entry *T)) {
nsubj := subj
if len(nsubj) > 0 {
nsubj = append(subj, '.')
}
if r.fwc != nil {
// We've reached a full wildcard, do a FWC match on the stree at this point
// and don't keep iterating downward.
nsubj := append(nsubj, '>')
st.Match(nsubj, cb)
return
}
if r.pwc != nil {
// We've found a partial wildcard. We'll keep iterating downwards, but first
// check whether there's interest at this level (without triggering dupes) and
// match if so.
var done bool
nsubj := append(nsubj, '*')
if len(r.pwc.psubs)+len(r.pwc.qsubs) > 0 {
st.Match(nsubj, cb)
done = true
}
if r.pwc.next.numNodes() > 0 {
intersectStree(st, r.pwc.next, nsubj, cb)
}
if done {
return
}
}
// Normal node with subject literals, keep iterating.
for t, n := range r.nodes {
if r.pwc != nil && r.pwc.next.numNodes() > 0 && n.next.numNodes() > 0 {
// A wildcard at the next level will already visit these descendents
// so skip so we don't callback the same subject more than once.
continue
}
nsubj := append(nsubj, t...)
if len(n.psubs)+len(n.qsubs) > 0 {
if subjectHasWildcard(bytesToString(nsubj)) {
st.Match(nsubj, cb)
} else {
if e, ok := st.Find(nsubj); ok {
cb(nsubj, e)
}
}
}
if n.next.numNodes() > 0 {
intersectStree(st, n.next, nsubj, cb)
}
}
}
+23 -2
View File
@@ -1,4 +1,4 @@
// Copyright 2012-2024 The NATS Authors
// Copyright 2012-2025 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
@@ -20,6 +20,7 @@ import (
"errors"
"fmt"
"math"
"math/bits"
"net"
"net/url"
"reflect"
@@ -165,7 +166,7 @@ func urlsAreEqual(u1, u2 *url.URL) bool {
// e.g. comma(834142) -> 834,142
//
// This function was copied from the github.com/dustin/go-humanize
// package and is Copyright Dustin Sallings <dustin@spy.net>
// package (MIT License) and is Copyright Dustin Sallings <dustin@spy.net>
func comma(v int64) string {
sign := ""
@@ -363,3 +364,23 @@ func parallelTaskQueue(mp int) chan<- func() {
}
return tq
}
// addSaturate returns a + b, saturating at math.MaxInt64.
// Both a and b must be non-negative.
func addSaturate(a, b int64) int64 {
sum, carry := bits.Add64(uint64(a), uint64(b), 0)
if carry != 0 || sum > uint64(math.MaxInt64) {
return math.MaxInt64
}
return int64(sum)
}
// mulSaturate returns a * b, saturating at math.MaxInt64.
// Both a and b must be non-negative.
func mulSaturate(a, b int64) int64 {
hi, lo := bits.Mul64(uint64(a), uint64(b))
if hi != 0 || lo > uint64(math.MaxInt64) {
return math.MaxInt64
}
return int64(lo)
}
+222 -83
View File
@@ -31,6 +31,7 @@ import (
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
@@ -59,6 +60,8 @@ const (
wsMaxControlPayloadSize = 125
wsFrameSizeForBrowsers = 4096 // From experiment, webrowsers behave better with limited frame size
wsCompressThreshold = 64 // Don't compress for small buffer(s)
wsMaxMsgPayloadMultiple = 8
wsMaxMsgPayloadLimit = 64 * 1024 * 1024
wsCloseSatusSize = 2
// From https://tools.ietf.org/html/rfc6455#section-11.7
@@ -127,7 +130,7 @@ type srvWebsocket struct {
server *http.Server
listener net.Listener
listenerErr error
allowedOrigins map[string]*allowedOrigin // host will be the key
allowedOrigins map[string][]*allowedOrigin // host will be the key
sameOrigin bool
connectURLs []string
connectURLsMap refCountedUrlSet
@@ -153,7 +156,7 @@ type wsUpgradeResult struct {
}
type wsReadInfo struct {
rem int
rem uint64
fs bool
ff bool
fc bool
@@ -162,31 +165,57 @@ type wsReadInfo struct {
mkey [4]byte
cbufs [][]byte
coff int
csz uint64
}
func (r *wsReadInfo) init() {
r.fs, r.ff = true, true
}
func (r *wsReadInfo) resetCompressedState() {
r.fs = true
r.ff = true
r.fc = false
r.rem = 0
r.cbufs = nil
r.coff = 0
r.csz = 0
}
// Compressed WebSocket messages have to be accumulated before they can be
// decompressed and handed to the parser, so this transport limit needs to
// allow batching several max_payload-sized NATS operations while still
// capping resource usage on the buffered compressed path.
func wsMaxMessageSize(mpay int) uint64 {
if mpay <= 0 {
mpay = MAX_PAYLOAD_SIZE
}
limit := uint64(mpay) * wsMaxMsgPayloadMultiple
if limit > wsMaxMsgPayloadLimit {
limit = wsMaxMsgPayloadLimit
}
return limit
}
// Returns a slice containing `needed` bytes from the given buffer `buf`
// starting at position `pos`, and possibly read from the given reader `r`.
// When bytes are present in `buf`, the `pos` is incremented by the number
// of bytes found up to `needed` and the new position is returned. If not
// enough bytes are found, the bytes found in `buf` are copied to the returned
// slice and the remaning bytes are read from `r`.
func wsGet(r io.Reader, buf []byte, pos, needed int) ([]byte, int, error) {
avail := len(buf) - pos
func wsGet(r io.Reader, buf []byte, pos, needed uint64) ([]byte, uint64, error) {
avail := uint64(len(buf)) - pos
if avail >= needed {
return buf[pos : pos+needed], pos + needed, nil
}
b := make([]byte, needed)
start := copy(b, buf[pos:])
start := uint64(copy(b, buf[pos:]))
for start != needed {
n, err := r.Read(b[start:cap(b)])
if err != nil {
return nil, 0, err
}
start += n
start += uint64(n)
}
return b, pos + avail, nil
}
@@ -205,12 +234,43 @@ func (c *client) isWebsocket() bool {
//
// Client lock MUST NOT be held on entry.
func (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, error) {
var bufs [][]byte
err := c.wsReadLoop(r, ior, buf, func(b []byte, compressed, final bool) error {
if compressed {
return errors.New("compressed websocket frames require wsReadAndParse")
}
bufs = append(bufs, b)
return nil
})
return bufs, err
}
func (c *client) wsReadAndParse(r *wsReadInfo, ior io.Reader, buf []byte) error {
mpay := int(atomic.LoadInt32(&c.mpay))
if mpay <= 0 {
mpay = MAX_PAYLOAD_SIZE
}
return c.wsReadLoop(r, ior, buf, func(b []byte, compressed, final bool) error {
if compressed {
if err := c.wsDecompressAndParse(r, b, final, mpay); err != nil {
r.resetCompressedState()
return err
}
if final {
r.fc = false
}
return nil
}
return c.parse(b)
})
}
func (c *client) wsReadLoop(r *wsReadInfo, ior io.Reader, buf []byte, handle func([]byte, bool, bool) error) error {
var (
bufs [][]byte
tmpBuf []byte
err error
pos int
max = len(buf)
pos uint64
max = uint64(len(buf))
)
for pos != max {
if r.fs {
@@ -218,69 +278,80 @@ func (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, err
frameType := wsOpCode(b0 & 0xF)
final := b0&wsFinalBit != 0
compressed := b0&wsRsv1Bit != 0
if b0&(wsRsv2Bit|wsRsv3Bit) != 0 {
return c.wsHandleProtocolError("RSV2 and RSV3 must be clear")
}
if compressed && !c.ws.compress {
return c.wsHandleProtocolError("compressed frame received without negotiated permessage-deflate")
}
pos++
tmpBuf, pos, err = wsGet(ior, buf, pos, 1)
if err != nil {
return bufs, err
return err
}
b1 := tmpBuf[0]
// Clients MUST set the mask bit. If not set, reject.
// However, LEAF by default will not have masking, unless they are forced to, by configuration.
if r.mask && b1&wsMaskBit == 0 {
return bufs, c.wsHandleProtocolError("mask bit missing")
return c.wsHandleProtocolError("mask bit missing")
}
// Store size in case it is < 125
r.rem = int(b1 & 0x7F)
r.rem = uint64(b1 & 0x7F)
switch frameType {
case wsPingMessage, wsPongMessage, wsCloseMessage:
if r.rem > wsMaxControlPayloadSize {
return bufs, c.wsHandleProtocolError(
return c.wsHandleProtocolError(
fmt.Sprintf("control frame length bigger than maximum allowed of %v bytes",
wsMaxControlPayloadSize))
}
if !final {
return bufs, c.wsHandleProtocolError("control frame does not have final bit set")
return c.wsHandleProtocolError("control frame does not have final bit set")
}
if compressed {
return c.wsHandleProtocolError("control frame must not be compressed")
}
case wsTextMessage, wsBinaryMessage:
if !r.ff {
return bufs, c.wsHandleProtocolError("new message started before final frame for previous message was received")
return c.wsHandleProtocolError("new message started before final frame for previous message was received")
}
r.ff = final
r.fc = compressed
case wsContinuationFrame:
// Compressed bit must be only set in the first frame
if r.ff || compressed {
return bufs, c.wsHandleProtocolError("invalid continuation frame")
return c.wsHandleProtocolError("invalid continuation frame")
}
r.ff = final
default:
return bufs, c.wsHandleProtocolError(fmt.Sprintf("unknown opcode %v", frameType))
return c.wsHandleProtocolError(fmt.Sprintf("unknown opcode %v", frameType))
}
switch r.rem {
case 126:
tmpBuf, pos, err = wsGet(ior, buf, pos, 2)
if err != nil {
return bufs, err
return err
}
r.rem = int(binary.BigEndian.Uint16(tmpBuf))
r.rem = uint64(binary.BigEndian.Uint16(tmpBuf))
case 127:
tmpBuf, pos, err = wsGet(ior, buf, pos, 8)
if err != nil {
return bufs, err
return err
}
if r.rem = binary.BigEndian.Uint64(tmpBuf); r.rem&(uint64(1)<<63) != 0 {
return c.wsHandleProtocolError("invalid 64-bit payload length")
}
r.rem = int(binary.BigEndian.Uint64(tmpBuf))
}
if r.mask {
// Read masking key
tmpBuf, pos, err = wsGet(ior, buf, pos, 4)
if err != nil {
return bufs, err
return err
}
copy(r.mkey[:], tmpBuf)
r.mkpos = 0
@@ -290,7 +361,7 @@ func (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, err
if wsIsControlFrame(frameType) {
pos, err = c.wsHandleControlFrame(r, frameType, ior, buf, pos)
if err != nil {
return bufs, err
return err
}
continue
}
@@ -299,53 +370,26 @@ func (c *client) wsRead(r *wsReadInfo, ior io.Reader, buf []byte) ([][]byte, err
r.fs = false
}
if pos < max {
var b []byte
var n int
n = r.rem
n := r.rem
if pos+n > max {
n = max - pos
}
b = buf[pos : pos+n]
b := buf[pos : pos+n]
pos += n
r.rem -= n
// If needed, unmask the buffer
if r.mask {
r.unmask(b)
}
addToBufs := true
// Handle compressed message
if r.fc {
// Assume that we may have continuation frames or not the full payload.
addToBufs = false
// Make a copy of the buffer before adding it to the list
// of compressed fragments.
r.cbufs = append(r.cbufs, append([]byte(nil), b...))
// When we have the final frame and we have read the full payload,
// we can decompress it.
if r.ff && r.rem == 0 {
b, err = r.decompress()
if err != nil {
return bufs, err
}
r.fc = false
// Now we can add to `bufs`
addToBufs = true
}
if err := handle(b, r.fc, r.ff && r.rem == 0); err != nil {
return err
}
// For non compressed frames, or when we have decompressed the
// whole message.
if addToBufs {
bufs = append(bufs, b)
}
// If payload has been fully read, then indicate that next
// is the start of a frame.
if r.rem == 0 {
r.fs = true
}
}
}
return bufs, nil
return nil
}
func (r *wsReadInfo) Read(dst []byte) (int, error) {
@@ -389,6 +433,9 @@ func (r *wsReadInfo) nextCBuf() []byte {
}
func (r *wsReadInfo) ReadByte() (byte, error) {
for len(r.cbufs) > 0 && len(r.cbufs[0]) == 0 {
r.nextCBuf()
}
if len(r.cbufs) == 0 {
return 0, io.EOF
}
@@ -398,33 +445,71 @@ func (r *wsReadInfo) ReadByte() (byte, error) {
return b, nil
}
func (r *wsReadInfo) decompress() ([]byte, error) {
r.coff = 0
// As per https://tools.ietf.org/html/rfc7692#section-7.2.2
// add 0x00, 0x00, 0xff, 0xff and then a final block so that flate reader
// does not report unexpected EOF.
func (c *client) wsDecompressAndParse(r *wsReadInfo, b []byte, final bool, mpay int) error {
limit := wsMaxMessageSize(mpay)
if len(b) > 0 {
if r.csz+uint64(len(b)) > limit {
return ErrMaxPayload
}
r.cbufs = append(r.cbufs, append([]byte(nil), b...))
r.csz += uint64(len(b))
}
if !final {
return nil
}
if r.csz+uint64(len(compressLastBlock)) > limit {
return ErrMaxPayload
}
r.cbufs = append(r.cbufs, compressLastBlock)
// Get a decompressor from the pool and bind it to this object (wsReadInfo)
// that provides Read() and ReadByte() APIs that will consume the compressed
// buffers (r.cbufs).
r.csz += uint64(len(compressLastBlock))
r.coff = 0
d, _ := decompressorPool.Get().(io.ReadCloser)
if d == nil {
d = flate.NewReader(r)
} else {
d.(flate.Resetter).Reset(r, nil)
}
// This will do the decompression.
b, err := io.ReadAll(d)
decompressorPool.Put(d)
// Now reset the compressed buffers list.
r.cbufs = nil
return b, err
defer func() {
d.Close()
decompressorPool.Put(d)
r.cbufs = nil
r.coff = 0
r.csz = 0
}()
lr := io.LimitedReader{R: d, N: int64(mpay + 1)}
buf := make([]byte, 32*1024)
total := 0
for {
n, err := lr.Read(buf)
if n > 0 {
pn := n
if total+n > mpay {
pn = mpay - total
}
if pn > 0 {
if err := c.parse(buf[:pn]); err != nil {
return err
}
}
total += n
if total > mpay {
return ErrMaxPayload
}
}
if err == nil {
continue
}
if err == io.EOF {
return nil
}
return err
}
}
// Handles the PING, PONG and CLOSE websocket control frames.
//
// Client lock MUST NOT be held on entry.
func (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.Reader, buf []byte, pos int) (int, error) {
func (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.Reader, buf []byte, pos uint64) (uint64, error) {
var payload []byte
var err error
@@ -443,6 +528,9 @@ func (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.R
status := wsCloseStatusNoStatusReceived
var body string
lp := len(payload)
if lp == 1 {
return pos, c.wsHandleProtocolError("close frame payload cannot be 1 byte")
}
// If there is a payload, the status is represented as a 2-byte
// unsigned integer (in network byte order). Then, there may be an
// optional body.
@@ -450,6 +538,9 @@ func (c *client) wsHandleControlFrame(r *wsReadInfo, frameType wsOpCode, nc io.R
if hasStatus {
// Decode the status
status = int(binary.BigEndian.Uint16(payload[:wsCloseSatusSize]))
if !wsIsValidCloseStatus(status) {
return pos, c.wsHandleProtocolError(fmt.Sprintf("invalid close status code %v", status))
}
// Now if there is a body, capture it and make sure this is a valid UTF-8.
if hasBody {
body = string(payload[wsCloseSatusSize:])
@@ -686,6 +777,21 @@ func (c *client) wsHandleProtocolError(message string) error {
return errors.New(message)
}
func wsIsValidCloseStatus(code int) bool {
switch code {
case wsCloseStatusNoStatusReceived, 1004, 1006, wsCloseStatusTLSHandshake:
return false
}
if code < 1000 || code >= 5000 {
return false
}
// 1016-2999 are currently reserved.
if code >= 1016 && code <= 2999 {
return false
}
return true
}
// Create a close message with the given `status` and `body`.
// If the `body` is more than the maximum allows control frame payload size,
// it is truncated and "..." is added at the end (as a hint that message
@@ -745,6 +851,10 @@ func (s *Server) wsUpgrade(w http.ResponseWriter, r *http.Request) (*wsUpgradeRe
if key == _EMPTY_ {
return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "key missing")
}
decoded, err := base64.StdEncoding.DecodeString(key)
if err != nil || len(decoded) != 16 {
return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid websocket key")
}
// Point 6.
if !wsHeaderContains(r.Header, "Sec-Websocket-Version", "13") {
return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "invalid version")
@@ -766,7 +876,10 @@ func (s *Server) wsUpgrade(w http.ResponseWriter, r *http.Request) (*wsUpgradeRe
// We will do masking if asked (unless we reject for tests)
noMasking := r.Header.Get(wsNoMaskingHeader) == wsNoMaskingValue && !wsTestRejectNoMasking
h := w.(http.Hijacker)
h, ok := w.(http.Hijacker)
if !ok {
return nil, wsReturnHTTPError(w, r, http.StatusBadRequest, "websocket upgrade not supported")
}
conn, brw, err := h.Hijack()
if err != nil {
if conn != nil {
@@ -814,9 +927,11 @@ func (s *Server) wsUpgrade(w http.ResponseWriter, r *http.Request) (*wsUpgradeRe
// Check for X-Forwarded-For header
if cips, ok := r.Header[wsXForwardedForHeader]; ok {
cip := cips[0]
if net.ParseIP(cip) != nil {
ws.clientIP = cip
if len(cips) > 0 {
cip := cips[0]
if net.ParseIP(cip) != nil {
ws.clientIP = cip
}
}
}
@@ -946,7 +1061,11 @@ func (w *srvWebsocket) checkOrigin(r *http.Request) error {
if err != nil {
return err
}
if oh != rh || op != rp {
rs := "http"
if r.TLS != nil {
rs = "https"
}
if oh != rh || op != rp || !strings.EqualFold(u.Scheme, rs) {
return errors.New("not same origin")
}
// I guess it is possible to have cases where one wants to check
@@ -955,9 +1074,16 @@ func (w *srvWebsocket) checkOrigin(r *http.Request) error {
}
if !listEmpty {
w.mu.RLock()
ao := w.allowedOrigins[oh]
origins := w.allowedOrigins[oh]
w.mu.RUnlock()
if ao == nil || u.Scheme != ao.scheme || op != ao.port {
var allowed bool
for _, ao := range origins {
if u.Scheme == ao.scheme && op == ao.port {
allowed = true
break
}
}
if !allowed {
return errors.New("not in the allowed list")
}
}
@@ -1011,7 +1137,17 @@ func validateWebsocketOptions(o *Options) error {
}
// Make sure that allowed origins, if specified, can be parsed.
for _, ao := range wo.AllowedOrigins {
if _, err := url.Parse(ao); err != nil {
u, err := url.ParseRequestURI(ao)
if err != nil {
return fmt.Errorf("unable to parse allowed origin: %v", err)
}
if u.Scheme != "http" && u.Scheme != "https" {
return fmt.Errorf("unable to parse allowed origin %q: allowed origins must be absolute URLs with http or https scheme", ao)
}
if u.Host == _EMPTY_ {
return fmt.Errorf("unable to parse allowed origin %q: host is required", ao)
}
if _, _, err := wsGetHostAndPort(u.Scheme == "https", u.Host); err != nil {
return fmt.Errorf("unable to parse allowed origin: %v", err)
}
}
@@ -1083,9 +1219,9 @@ func (s *Server) wsSetOriginOptions(o *WebsocketOpts) {
}
h, p, _ := wsGetHostAndPort(u.Scheme == "https", u.Host)
if ws.allowedOrigins == nil {
ws.allowedOrigins = make(map[string]*allowedOrigin, len(o.AllowedOrigins))
ws.allowedOrigins = make(map[string][]*allowedOrigin, len(o.AllowedOrigins))
}
ws.allowedOrigins[h] = &allowedOrigin{scheme: u.Scheme, port: p}
ws.allowedOrigins[h] = append(ws.allowedOrigins[h], &allowedOrigin{scheme: u.Scheme, port: p})
}
}
@@ -1306,7 +1442,7 @@ func (s *Server) createWSClient(conn net.Conn, ws *websocket) *client {
return c
}
if opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn {
if opts.MaxConn < 0 || (opts.MaxConn > 0 && len(s.clients) >= opts.MaxConn) {
s.mu.Unlock()
c.maxConnExceeded()
return nil
@@ -1384,7 +1520,7 @@ func (c *client) wsCollapsePtoNB() (net.Buffers, int64) {
cp.Reset(buf)
}
var csz int
for _, b := range nb {
for i, b := range nb {
for len(b) > 0 {
n, err := cp.Write(b)
if err != nil {
@@ -1396,7 +1532,10 @@ func (c *client) wsCollapsePtoNB() (net.Buffers, int64) {
}
b = b[n:]
}
nbPoolPut(b) // No longer needed as contents written to compressor.
// Use original slice since capacity will change to zero
// in the loop after consuming the buffer, which will make
// nbPoolPut discard it.
nbPoolPut(nb[i])
}
if err := cp.Flush(); err != nil {
c.Errorf("Error during compression: %v", err)
+12 -12
View File
@@ -23,7 +23,7 @@ A [Go](http://golang.org) client for the [NATS messaging system](https://nats.io
go get github.com/nats-io/nats.go@latest
# To get a specific version:
go get github.com/nats-io/nats.go@v1.47.0
go get github.com/nats-io/nats.go@v1.49.0
# Note that the latest major version for NATS Server is v2:
go get github.com/nats-io/nats-server/v2@latest
@@ -134,7 +134,7 @@ The simplest form is to use the helper method UserCredentials(credsFilepath).
nc, err := nats.Connect(url, nats.UserCredentials("user.creds"))
```
The helper methods creates two callback handlers to present the user JWT and sign the nonce challenge from the server.
The helper method creates two callback handlers to present the user JWT and sign the nonce challenge from the server.
The core client library never has direct access to your private key and simply performs the callback for signing the server challenge.
The helper will load and wipe and erase memory it uses for each connect or reconnect.
@@ -177,7 +177,7 @@ nc, err := nats.Connect("tls://nats.demo.io:4443")
// We provide a helper method to make this case easier.
nc, err = nats.Connect("tls://localhost:4443", nats.RootCAs("./configs/certs/ca.pem"))
// If the server requires client certificate, there is an helper function for that too:
// If the server requires client certificate, there is a helper function for that too:
cert := nats.ClientCert("./configs/certs/client-cert.pem", "./configs/certs/client-key.pem")
nc, err = nats.Connect("tls://localhost:4443", cert)
@@ -210,17 +210,17 @@ if err != nil {
// "*" matches any token, at any level of the subject.
nc.Subscribe("foo.*.baz", func(m *Msg) {
fmt.Printf("Msg received on [%s] : %s\n", m.Subject, string(m.Data));
fmt.Printf("Msg received on [%s] : %s\n", m.Subject, string(m.Data))
})
nc.Subscribe("foo.bar.*", func(m *Msg) {
fmt.Printf("Msg received on [%s] : %s\n", m.Subject, string(m.Data));
fmt.Printf("Msg received on [%s] : %s\n", m.Subject, string(m.Data))
})
// ">" matches any length of the tail of a subject, and can only be the last token
// E.g. 'foo.>' will match 'foo.bar', 'foo.bar.baz', 'foo.foo.bar.bax.22'
nc.Subscribe("foo.>", func(m *Msg) {
fmt.Printf("Msg received on [%s] : %s\n", m.Subject, string(m.Data));
fmt.Printf("Msg received on [%s] : %s\n", m.Subject, string(m.Data))
})
// Matches all of the above
@@ -237,7 +237,7 @@ nc.Publish("foo.bar.baz", []byte("Hello World"))
// Normal subscribers will continue to work as expected.
nc.QueueSubscribe("foo", "job_workers", func(_ *Msg) {
received += 1;
received += 1
})
```
@@ -267,9 +267,9 @@ fmt.Println("All clear!")
// FlushTimeout specifies a timeout value as well.
err := nc.FlushTimeout(1*time.Second)
if err != nil {
fmt.Println("All clear!")
} else {
fmt.Println("Flushed timed out!")
} else {
fmt.Println("All clear!")
}
// Auto-unsubscribe after MAX_WANTED messages received
@@ -285,7 +285,7 @@ nc1.Subscribe("foo", func(m *Msg) {
fmt.Printf("Received a message: %s\n", string(m.Data))
})
nc2.Publish("foo", []byte("Hello World!"));
nc2.Publish("foo", []byte("Hello World!"))
```
@@ -339,7 +339,7 @@ nc, err = nats.Connect("nats://localhost:4222", nats.UserInfo("foo", "bar"))
// For token based authentication:
nc, err = nats.Connect("nats://localhost:4222", nats.Token("S3cretT0ken"))
// You can even pass the two at the same time in case one of the server
// You can even pass the two at the same time in case one of the servers
// in the mesh requires token instead of user name and password.
nc, err = nats.Connect("nats://localhost:4222",
nats.UserInfo("foo", "bar"),
@@ -372,7 +372,7 @@ msg, err := sub.NextMsgWithContext(ctx)
```
## Backwards compatibility
## Backward compatibility
In the development of nats.go, we are committed to maintaining backward compatibility and ensuring a stable and reliable experience for all users. In general, we follow the standard go compatibility guidelines.
However, it's important to clarify our stance on certain types of changes:
+1 -1
View File
@@ -95,7 +95,7 @@ func (nc *Conn) oldRequestWithContext(ctx context.Context, subj string, hdr, dat
s.AutoUnsubscribe(1)
defer s.Unsubscribe()
err = nc.publish(subj, inbox, hdr, data)
err = nc.publish(subj, inbox, false, hdr, data)
if err != nil {
return nil, err
}
+2 -2
View File
@@ -107,7 +107,7 @@ func (c *EncodedConn) Publish(subject string, v any) error {
if err != nil {
return err
}
return c.Conn.publish(subject, _EMPTY_, nil, b)
return c.Conn.publish(subject, _EMPTY_, false, nil, b)
}
// PublishRequest will perform a Publish() expecting a response on the
@@ -120,7 +120,7 @@ func (c *EncodedConn) PublishRequest(subject, reply string, v any) error {
if err != nil {
return err
}
return c.Conn.publish(subject, reply, nil, b)
return c.Conn.publish(subject, reply, true, nil, b)
}
// Request will create an Inbox and perform a Request() call
+9 -9
View File
@@ -4,19 +4,19 @@ go 1.24.0
require (
github.com/golang/protobuf v1.4.2
github.com/klauspost/compress v1.18.0
github.com/klauspost/compress v1.18.2
github.com/nats-io/jwt/v2 v2.8.0
github.com/nats-io/nats-server/v2 v2.12.0
github.com/nats-io/nkeys v0.4.11
github.com/nats-io/nats-server/v2 v2.12.3
github.com/nats-io/nkeys v0.4.12
github.com/nats-io/nuid v1.0.1
google.golang.org/protobuf v1.23.0
)
require (
github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op // indirect
github.com/google/go-tpm v0.9.5 // indirect
github.com/minio/highwayhash v1.0.3 // indirect
golang.org/x/crypto v0.42.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/time v0.13.0 // indirect
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op // indirect
github.com/google/go-tpm v0.9.7 // indirect
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 // indirect
golang.org/x/crypto v0.46.0 // indirect
golang.org/x/sys v0.39.0 // indirect
golang.org/x/time v0.14.0 // indirect
)
+18 -18
View File
@@ -1,5 +1,5 @@
github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0=
github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op h1:Ucf+QxEKMbPogRO5guBNe5cgd9uZgfoJLOYs8WWhtjM=
github.com/antithesishq/antithesis-sdk-go v0.5.0-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E=
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
@@ -12,27 +12,27 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-tpm v0.9.5 h1:ocUmnDebX54dnW+MQWGQRbdaAcJELsa6PqZhJ48KwVU=
github.com/google/go-tpm v0.9.5/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q=
github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/google/go-tpm v0.9.7 h1:u89J4tUUeDTlH8xxC3CTW7OHZjbjKoHdQ9W7gCUhtxA=
github.com/google/go-tpm v0.9.7/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY=
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76 h1:KGuD/pM2JpL9FAYvBrnBBeENKZNh6eNtjqytV6TYjnk=
github.com/minio/highwayhash v1.0.4-0.20251030100505-070ab1a87a76/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ=
github.com/nats-io/jwt/v2 v2.8.0 h1:K7uzyz50+yGZDO5o772eRE7atlcSEENpL7P+b74JV1g=
github.com/nats-io/jwt/v2 v2.8.0/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA=
github.com/nats-io/nats-server/v2 v2.12.0 h1:OIwe8jZUqJFrh+hhiyKu8snNib66qsx806OslqJuo74=
github.com/nats-io/nats-server/v2 v2.12.0/go.mod h1:nr8dhzqkP5E/lDwmn+A2CvQPMd1yDKXQI7iGg3lAvww=
github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
github.com/nats-io/nats-server/v2 v2.12.3 h1:KRv+1n7lddMVgkJPQer+pt36TcO0ENxjilBmeWdjcHs=
github.com/nats-io/nats-server/v2 v2.12.3/go.mod h1:MQXjG9WjyXKz9koWzUc3jYUMKD8x3CLmTNy91IQQz3Y=
github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/time v0.13.0 h1:eUlYslOIt32DgYD6utsuUeHs4d7AsEYLuIAdg7FlYgI=
golang.org/x/time v0.13.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI=
golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+749
View File
@@ -0,0 +1,749 @@
# Migrating from Legacy JetStream API to `jetstream` Package
This guide helps you migrate from the legacy JetStream API in the `nats` package
(`nats.JetStreamContext`) to the new `jetstream` package
(`github.com/nats-io/nats.go/jetstream`).
- [Why Migrate?](#why-migrate)
- [Getting Started](#getting-started)
- [Stream Management](#stream-management)
- [Consumer Management](#consumer-management)
- [Publishing](#publishing)
- [Consuming Messages](#consuming-messages)
- [Replacing js.Subscribe()](#replacing-jssubscribe)
- [Replacing js.PullSubscribe()](#replacing-jspullsubscribe)
- [Ordered Consumers](#ordered-consumers)
- [Push Consumers](#push-consumers)
- [Subscription Options Mapping](#subscription-options-mapping)
- [Error Handling in Consume/Messages](#error-handling-in-consumemessages)
- [Message Acknowledgement](#message-acknowledgement)
- [KeyValue Store](#keyvalue-store)
- [Object Store](#object-store)
## Why Migrate?
The legacy JetStream API (`nats.JetStreamContext`) is deprecated. The `jetstream`
package provides a cleaner, more predictable API with several key improvements:
- **Explicit resource management.** Streams and consumers are created and managed
explicitly. The legacy `js.Subscribe()` implicitly created consumers behind
the scenes, leading to surprising behavior.
- **Pull consumers as the default.** Pull consumers with `Consume()` and
`Messages()` provide the same continuous message delivery as the legacy push-based
`Subscribe()`, but with better flow control and no slow consumer issues.
- **`context.Context` throughout.** All API calls accept `context.Context` for
timeout and cancellation, replacing the mix of `MaxWait`, `AckWait`, and
`Context()` options.
- **Clear interface separation.** Instead of one large `JetStreamContext` interface,
functionality is split across focused interfaces: `JetStream`, `Stream` and
`Consumer`.
## Getting Started
The core NATS connection remains unchanged. Only the JetStream initialization
differs:
```go
import (
"github.com/nats-io/nats.go"
"github.com/nats-io/nats.go/jetstream"
)
nc, _ := nats.Connect(nats.DefaultURL)
```
**Legacy:**
```go
js, _ := nc.JetStream()
// With domain
js, _ := nc.JetStream(nats.Domain("hub"))
// With custom API prefix
js, _ := nc.JetStream(nats.APIPrefix("myprefix"))
```
**New:**
```go
js, _ := jetstream.New(nc)
// With domain
js, _ := jetstream.NewWithDomain(nc, "hub")
// With custom API prefix
js, _ := jetstream.NewWithAPIPrefix(nc, "myprefix")
```
### Initialization Options
| Legacy | New |
|-----------------------------------|--------------------------------------------|
| `nats.Domain(domain)` | `jetstream.NewWithDomain(nc, domain)` |
| `nats.APIPrefix(prefix)` | `jetstream.NewWithAPIPrefix(nc, prefix)` |
| `nats.PublishAsyncMaxPending(n)` | `jetstream.WithPublishAsyncMaxPending(n)` |
| `nats.PublishAsyncErrHandler(cb)` | `jetstream.WithPublishAsyncErrHandler(cb)` |
## Stream Management
`StreamConfig` is essentially the same struct — it just lives in the `jetstream`
package now. The new API takes `StreamConfig` by value (not pointer) and
management methods return a `Stream` handle instead of `*StreamInfo`.
| Legacy | New | Notes |
|---------------------------------|-------------------------------------|-------------------------------------------------------------------|
| `js.AddStream(cfg)` | `js.CreateStream(ctx, cfg)` | Also: `CreateOrUpdateStream()` |
| `js.UpdateStream(cfg)` | `js.UpdateStream(ctx, cfg)` | |
| `js.DeleteStream(name)` | `js.DeleteStream(ctx, name)` | |
| `js.StreamInfo(name)` | `s.Info(ctx)` / `s.CachedInfo()` | Get stream handle first via `js.Stream(ctx, name)` |
| `js.PurgeStream(name, opts...)` | `s.Purge(ctx, opts...)` | Options: `WithPurgeSubject`, `WithPurgeSequence`, `WithPurgeKeep` |
| `js.GetMsg(name, seq)` | `s.GetMsg(ctx, seq)` | |
| `js.GetLastMsg(name, subj)` | `s.GetLastMsgForSubject(ctx, subj)` | |
| `js.DeleteMsg(name, seq)` | `s.DeleteMsg(ctx, seq)` | Also: `s.SecureDeleteMsg()` |
| `js.Streams()` | `js.ListStreams(ctx)` | Returns lister with `.Info()` channel and `.Err()` |
| `js.StreamNames()` | `js.StreamNames(ctx)` | Returns lister with `.Name()` channel and `.Err()` |
The key architectural difference is that stream-specific operations (purge, get/delete
messages) now live on the `Stream` interface instead of the top-level context. Get
a stream handle first, then operate on it:
```go
s, _ := js.Stream(ctx, "ORDERS")
s.Purge(ctx)
msg, _ := s.GetMsg(ctx, 100)
```
## Consumer Management
The biggest conceptual change: in the legacy API, `js.Subscribe()` would
implicitly create consumers. In the new API, consumer creation is always explicit
and separate from message consumption.
| Legacy | New | Notes |
|------------------------------------------|----------------------------------------|----------------------------------------------------------------|
| `js.AddConsumer(stream, cfg)` | `js.CreateConsumer(ctx, stream, cfg)` | Also: `CreateOrUpdateConsumer()`, `UpdateConsumer()` |
| `js.Subscribe(subj, handler)` (implicit) | No equivalent | Must create consumer explicitly first |
| `js.ConsumerInfo(stream, name)` | `cons.Info(ctx)` / `cons.CachedInfo()` | Get consumer handle first via `js.Consumer(ctx, stream, name)` |
| `js.DeleteConsumer(stream, name)` | `js.DeleteConsumer(ctx, stream, name)` | |
| `js.Consumers(stream)` | `s.ListConsumers(ctx)` | Returns lister with `.Info()` channel and `.Err()` |
| `js.ConsumerNames(stream)` | `s.ConsumerNames(ctx)` | Returns lister with `.Name()` channel and `.Err()` |
Consumer management is available at two levels:
- On `JetStream` — requires stream name as parameter (e.g. `js.CreateConsumer(ctx, "ORDERS", cfg)`), bypassing the need to fetch a stream
- On `Stream` — no stream name needed (e.g. `s.CreateConsumer(ctx, cfg)`)
The new API provides three creation methods:
- `CreateConsumer` — fails if the consumer already exists with different config
- `UpdateConsumer` - fails if the consumer does not exist
- `CreateOrUpdateConsumer` — creates or updates as needed
**Additional notes on consumer behavior:**
- The default ack policy changed between the APIs. In the legacy API,
`AddConsumer()` defaulted to `AckNone`. In the new API, the default is
`AckExplicit`.
- In the legacy API, `sub.Unsubscribe()` on an implicitly created
consumer would automatically delete that consumer on the server. The new API
does not perform any automatic cleanup - consumers must be deleted explicitly
via `DeleteConsumer()`, or via `InactiveThreshold` on the consumer
config to let the server remove it automatically after a period of inactivity.
Push consumers use separate methods: `CreatePushConsumer`, `CreateOrUpdatePushConsumer`,
`UpdatePushConsumer`, and `PushConsumer` (for getting a handle).
```go
s, _ := js.Stream(ctx, "ORDERS")
cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
})
```
## Publishing
Publishing is largely the same, with the addition of `context.Context` for
synchronous operations.
### Synchronous Publish
**Legacy:**
```go
ack, _ := js.Publish("ORDERS.new", []byte("hello"))
ack, _ = js.PublishMsg(&nats.Msg{
Subject: "ORDERS.new",
Data: []byte("hello"),
})
```
**New:**
```go
ack, _ := js.Publish(ctx, "ORDERS.new", []byte("hello"))
ack, _ = js.PublishMsg(ctx, &nats.Msg{
Subject: "ORDERS.new",
Data: []byte("hello"),
})
```
### Async Publish
**Legacy:**
```go
ackF, _ := js.PublishAsync("ORDERS.new", []byte("hello"))
select {
case ack := <-ackF.Ok():
fmt.Println(ack.Sequence)
case err := <-ackF.Err():
fmt.Println(err)
}
// Wait for all pending acks
<-js.PublishAsyncComplete()
```
**New:**
```go
// Async publish does not take context (returns immediately)
ackF, _ := js.PublishAsync("ORDERS.new", []byte("hello"))
select {
case ack := <-ackF.Ok():
fmt.Println(ack.Sequence)
case err := <-ackF.Err():
fmt.Println(err)
}
<-js.PublishAsyncComplete()
```
### Publish Options
| Legacy | New |
|------------------------------------------|---------------------------------------------------|
| `nats.MsgId(id)` | `jetstream.WithMsgID(id)` |
| `nats.ExpectStream(name)` | `jetstream.WithExpectStream(name)` |
| `nats.ExpectLastSequence(seq)` | `jetstream.WithExpectLastSequence(seq)` |
| `nats.ExpectLastSequencePerSubject(seq)` | `jetstream.WithExpectLastSequencePerSubject(seq)` |
| `nats.ExpectLastMsgId(id)` | `jetstream.WithExpectLastMsgID(id)` |
| `nats.RetryWait(dur)` | `jetstream.WithRetryWait(dur)` |
| `nats.RetryAttempts(n)` | `jetstream.WithRetryAttempts(n)` |
| `nats.StallWait(dur)` | `jetstream.WithStallWait(dur)` |
## Consuming Messages
This is the most significant area of change. The legacy API offered many
subscription flavors (`Subscribe`, `SubscribeSync`, `QueueSubscribe`,
`ChanSubscribe`, `PullSubscribe`) that blurred the line between consumer
creation, stream lookup and message consumption. The new API separates these
concerns: first create a consumer, then choose how to receive messages.
With the exception of PullSubscribe, all legacy subscription flavors utilized push consumers under the hood. The new API recommends pull consumers for all use cases, as they provide better flow control and no risk of slow consumer issues. Pull-based consumption is available via `Consume()` and `Messages()`, which maintain persistent pull subscriptions with pre-buffering for efficient continuous delivery. Push consumers are still supported for users who prefer that model, but pull consumers are the recommended default.
### Replacing `js.Subscribe()`
The legacy `js.Subscribe()` created a push consumer behind the scenes (unless
explicitly specified otherwise via `nats.Bind()` or `nats.Durable()`) and
delivered messages either via a callback. In the new API, the recommended
replacement is a **pull consumer** with `Consume()` or `Messages()`. These
provide the same continuous delivery with better flow control.
#### Legacy: callback subscription
```go
sub, _ := js.Subscribe("ORDERS.*", func(msg *nats.Msg) {
fmt.Printf("Received: %s\n", string(msg.Data))
msg.Ack()
}, nats.Durable("processor"), nats.ManualAck)
defer sub.Unsubscribe()
```
#### New: callback with `Consume()`
`Consume()` is the closest equivalent to `js.Subscribe()` — it delivers messages
to a callback function continuously.
```go
s, _ := js.Stream(ctx, "ORDERS")
cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
FilterSubject: "ORDERS.*",
})
cc, _ := cons.Consume(func(msg jetstream.Msg) {
fmt.Printf("Received: %s\n", string(msg.Data()))
msg.Ack()
})
defer cc.Stop()
```
> Note: `ManualAck()` is not needed — messages are never auto-acknowledged in
> the new API.
#### New: iterator with `Messages()`
`Messages()` provides an iterator-based approach, useful when you want explicit
control over when the next message is fetched.
```go
cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
FilterSubject: "ORDERS.*",
})
iter, _ := cons.Messages()
for {
msg, err := iter.Next()
if err != nil {
// handle error
}
fmt.Printf("Received: %s\n", string(msg.Data()))
msg.Ack()
}
// Call iter.Stop() when done
```
Both `Consume()` and `Messages()` maintain overlapping pull requests to the
server, providing efficient continuous delivery without gaps.
#### Legacy: synchronous subscription
```go
sub, _ := js.SubscribeSync("ORDERS.*", nats.Durable("processor"))
msg, _ := sub.NextMsg(time.Second)
```
**New:** Use `Messages()` and call `Next()`:
```go
cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
FilterSubject: "ORDERS.*",
})
iter, _ := cons.Messages()
msg, _ := iter.Next()
```
#### Legacy: queue subscription
```go
// Multiple instances share work via a queue group
sub, _ := js.QueueSubscribe("ORDERS.*", "workers", handler,
nats.Durable("processor"))
```
**New with pull consumers:** With pull consumers, there is no need for an
explicit queue group. Multiple application instances (or goroutines) calling
`Consume()` or `Messages()` on the same durable consumer will naturally
distribute messages among themselves — the server tracks pending acknowledgements
and avoids delivering the same message to multiple consumers:
```go
cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
})
cc, _ := cons.Consume(handler)
defer cc.Stop()
```
**New with push consumers:** If you need push-based queue semantics, set
`DeliverGroup` on a push consumer — this is the direct equivalent of the legacy
queue group:
```go
cons, _ := s.CreateOrUpdatePushConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
DeliverSubject: "deliver.orders",
DeliverGroup: "workers",
})
cc, _ := cons.Consume(handler)
defer cc.Stop()
```
> **Note:** Push consumers with `DeliverGroup` cannot be flow controlled. If you
> experience slow consumer issues, consider using pull-based consumers instead —
> multiple instances on the same durable consumer achieve the same work
> distribution without the slow consumer risk.
#### Legacy: channel subscription
```go
ch := make(chan *nats.Msg, 64)
sub, _ := js.ChanSubscribe("ORDERS.*", ch, nats.Durable("processor"))
for msg := range ch {
msg.Ack()
}
```
**New:** There is no direct channel-based equivalent. Use `Consume()` or
`Messages()` instead.
### Replacing `js.PullSubscribe()`
The legacy pull subscription required creating a subscription and then calling
`Fetch()` in a loop.
#### Legacy: pull subscribe + fetch loop
```go
sub, _ := js.PullSubscribe("ORDERS.*", "processor")
for {
msgs, _ := sub.Fetch(10, nats.MaxWait(5*time.Second))
for _, msg := range msgs {
fmt.Printf("Received: %s\n", string(msg.Data))
msg.Ack()
}
}
```
**New with `Fetch()`/`FetchNoWait()` (one-off batch):**
If you specifically need one-off batch fetching, `Fetch()` is available directly
on the consumer — no separate subscription step:
```go
cons, _ := s.CreateOrUpdateConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
FilterSubject: "ORDERS.*",
})
// non-blocking, returns a `FetchResult` that provides messages and error
msgs, _ := cons.Fetch(10, jetstream.FetchMaxWait(5*time.Second))
for msg := range msgs.Messages() {
fmt.Printf("Received: %s\n", string(msg.Data()))
msg.Ack()
}
if msgs.Error() != nil {
// handle error
}
```
> **Warning:** `Fetch()`, `FetchNoWait()`, and `FetchBytes()` are one-off,
> single pull requests. They do not perform pre-buffering optimizations. For
> continuous message processing, always prefer `Consume()` or `Messages()`.
> When using `FetchBytes()`, the requested byte size must stay under the
> client's max pending bytes limit (64MB by default), otherwise it will trigger
> slow consumer errors on the underlying subscription.
### Ordered Consumers
Ordered consumers provide strictly ordered, gap-free message delivery. The library
automatically recreates the underlying consumer on sequence gaps or heartbeat
failures.
**Legacy:**
```go
sub, _ := js.Subscribe("ORDERS.*", handler, nats.OrderedConsumer())
```
**New:**
```go
cons, _ := js.OrderedConsumer(ctx, "ORDERS", jetstream.OrderedConsumerConfig{
FilterSubjects: []string{"ORDERS.*"},
})
// Use the same consumption methods as regular consumers
cc, _ := cons.Consume(func(msg jetstream.Msg) {
fmt.Printf("Received: %s\n", string(msg.Data()))
})
defer cc.Stop()
```
### Push Consumers
Pull consumers are recommended for most use cases, but push consumers are also
supported. Push consumers require `DeliverSubject` in their config and only
support `Consume()` (not `Fetch()` or `Messages()`).
**Legacy:**
```go
sub, _ := js.Subscribe("ORDERS.*", handler,
nats.Durable("processor"),
nats.DeliverSubject("deliver.orders"),
nats.IdleHeartbeat(30*time.Second),
)
```
**New:**
```go
cons, _ := s.CreateOrUpdatePushConsumer(ctx, jetstream.ConsumerConfig{
Durable: "processor",
FilterSubject: "ORDERS.*",
DeliverSubject: "deliver.orders",
IdleHeartbeat: 30 * time.Second,
})
cc, _ := cons.Consume(func(msg jetstream.Msg) {
fmt.Printf("Received: %s\n", string(msg.Data()))
msg.Ack()
})
defer cc.Stop()
```
### Subscription Options Mapping
Most legacy `SubOpt` options map directly to `ConsumerConfig` fields. Since
consumer creation is explicit, these are set at creation time rather than passed
as subscription options.
| Legacy SubOpt | New ConsumerConfig field |
|-------------------------------------|---------------------------------------------------------------------------|
| `nats.Durable("name")` | `Durable: "name"` |
| `nats.ConsumerName("name")` | `Name: "name"` |
| `nats.Description("desc")` | `Description: "desc"` |
| `nats.DeliverAll()` | `DeliverPolicy: jetstream.DeliverAllPolicy` |
| `nats.DeliverLast()` | `DeliverPolicy: jetstream.DeliverLastPolicy` |
| `nats.DeliverLastPerSubject()` | `DeliverPolicy: jetstream.DeliverLastPerSubjectPolicy` |
| `nats.DeliverNew()` | `DeliverPolicy: jetstream.DeliverNewPolicy` |
| `nats.StartSequence(seq)` | `DeliverPolicy: jetstream.DeliverByStartSequencePolicy, OptStartSeq: seq` |
| `nats.StartTime(t)` | `DeliverPolicy: jetstream.DeliverByStartTimePolicy, OptStartTime: &t` |
| `nats.AckExplicit()` | `AckPolicy: jetstream.AckExplicitPolicy` |
| `nats.AckAll()` | `AckPolicy: jetstream.AckAllPolicy` |
| `nats.AckNone()` | `AckPolicy: jetstream.AckNonePolicy` |
| `nats.ManualAck()` | Not needed (messages are never auto-acked) |
| `nats.MaxDeliver(n)` | `MaxDeliver: n` |
| `nats.MaxAckPending(n)` | `MaxAckPending: n` |
| `nats.BackOff(durations)` | `BackOff: durations` |
| `nats.ReplayOriginal()` | `ReplayPolicy: jetstream.ReplayOriginalPolicy` |
| `nats.ReplayInstant()` | `ReplayPolicy: jetstream.ReplayInstantPolicy` |
| `nats.RateLimit(bps)` | `RateLimit: bps` |
| `nats.HeadersOnly()` | `HeadersOnly: true` |
| `nats.InactiveThreshold(dur)` | `InactiveThreshold: dur` |
| `nats.ConsumerFilterSubjects(s...)` | `FilterSubjects: s` |
| `nats.ConsumerReplicas(n)` | `Replicas: n` |
| `nats.ConsumerMemoryStorage()` | `MemoryStorage: true` |
The following options have no direct equivalent — use the consumer handle
directly instead:
| Legacy SubOpt | New equivalent |
|-------------------------------|---------------------------------------------------------------------|
| `nats.Bind(stream, consumer)` | `js.Consumer(ctx, stream, consumer)` or `s.Consumer(ctx, consumer)` |
| `nats.BindStream(stream)` | Use `js.Stream(ctx, stream)` to get a stream handle |
| `nats.OrderedConsumer()` | `js.OrderedConsumer(ctx, stream, cfg)` |
### Consume/Messages Options
`Consume()` and `Messages()` accept options that control pull request behavior:
| Option | Description |
|----------------------------|--------------------------------------------------------------|
| `PullMaxMessages(n)` | Max messages buffered (default: 500) |
| `PullMaxBytes(n)` | Max bytes buffered (mutually exclusive with PullMaxMessages) |
| `PullExpiry(dur)` | Pull request timeout (default: 30s) |
| `PullHeartbeat(dur)` | Idle heartbeat interval |
| `PullThresholdMessages(n)` | Refill threshold (default: 50% of max) |
| `PullThresholdBytes(n)` | Byte-based refill threshold |
| `StopAfter(n)` | Auto-stop after N messages |
| `ConsumeErrHandler(fn)` | Custom error handler |
### Error Handling in Consume/Messages
Both `Consume()` and `Messages()` handle server-sent status messages internally.
Some errors are terminal (stop consumption), while others are recoverable
(consumption continues).
**Terminal errors** — consumption stops automatically:
- `ErrConsumerDeleted` — the consumer was deleted on the server
- `ErrBadRequest` — invalid request (e.g. misconfigured consumer)
- Connection closed — for `Consume()` this surfaces as `ErrConnectionClosed`;
for `Messages()`, `Next()` returns `ErrMsgIteratorClosed`
**Recoverable errors** — reported via error handler, consumption continues:
- `ErrNoHeartbeat` — missed idle heartbeats from server; a new pull request
is issued automatically
- `ErrConsumerLeadershipChanged` — consumer moved to a different server in the
cluster; pending counts are reset
- `nats.ErrNoResponders` — no JetStream service available (temporary)
#### Error handling with `Consume()`
Use `ConsumeErrHandler` to be notified about both terminal and recoverable errors:
```go
cc, _ := cons.Consume(func(msg jetstream.Msg) {
msg.Ack()
}, jetstream.ConsumeErrHandler(func(cc jetstream.ConsumeContext, err error) {
if errors.Is(err, jetstream.ErrConsumerDeleted) ||
errors.Is(err, jetstream.ErrBadRequest) {
log.Fatalf("terminal consumer error: %v", err)
}
log.Printf("recoverable consumer error: %v", err)
}))
defer cc.Stop()
```
#### Error handling with `Messages()`
With `Messages()`, terminal errors are returned directly by `Next()`. By default,
`ErrNoHeartbeat` is also returned by `Next()` (controlled by
`WithMessagesErrOnMissingHeartbeat`), but it is not terminal — you can continue
calling `Next()`:
```go
iter, _ := cons.Messages()
for {
msg, err := iter.Next()
if err != nil {
if errors.Is(err, jetstream.ErrMsgIteratorClosed) {
// iterator was stopped (either explicitly or due to connection close)
break
}
if errors.Is(err, jetstream.ErrNoHeartbeat) {
// recoverable — new pull request is issued, keep going
log.Println("missed heartbeat, re-pulling")
continue
}
// ErrConsumerDeleted, ErrBadRequest are terminal
log.Fatalf("terminal error: %v", err)
}
msg.Ack()
}
```
## Message Acknowledgement
Ack methods are similar, with minor naming changes. The main difference is that
message fields are accessed via methods instead of struct fields.
| Legacy | New |
|-------------------------|------------------------------|
| `msg.Ack()` | Unchanged |
| `msg.AckSync()` | `msg.DoubleAck(ctx)` |
| `msg.Nak()` | Unchanged |
| `msg.NakWithDelay(dur)` | Unchanged |
| `msg.InProgress()` | Unchanged |
| `msg.Term()` | Unchanged |
| N/A | `msg.TermWithReason(reason)` |
| `msg.Metadata()` | Unchanged |
### Accessing Message Data
**Legacy:** Direct struct fields on `*nats.Msg`:
```go
fmt.Println(string(msg.Data))
fmt.Println(msg.Subject)
fmt.Println(msg.Header.Get("key"))
```
**New:** Methods on `jetstream.Msg` interface:
```go
fmt.Println(string(msg.Data()))
fmt.Println(msg.Subject())
fmt.Println(msg.Headers().Get("key"))
```
## KeyValue Store
The KV API is nearly identical. The main changes are:
1. All methods take `context.Context` as the first parameter
2. New `CreateOrUpdateKeyValue()` and `UpdateKeyValue()` methods
3. Types live in the `jetstream` package
**Legacy:**
```go
js, _ := nc.JetStream()
kv, _ := js.CreateKeyValue(&nats.KeyValueConfig{
Bucket: "profiles",
})
kv.Put("sue.color", []byte("blue"))
entry, _ := kv.Get("sue.color")
fmt.Println(string(entry.Value()))
watcher, _ := kv.Watch("sue.*")
defer watcher.Stop()
```
**New:**
```go
js, _ := jetstream.New(nc)
kv, _ := js.CreateKeyValue(ctx, jetstream.KeyValueConfig{
Bucket: "profiles",
})
kv.Put(ctx, "sue.color", []byte("blue"))
entry, _ := kv.Get(ctx, "sue.color")
fmt.Println(string(entry.Value()))
watcher, _ := kv.Watch(ctx, "sue.*")
defer watcher.Stop()
```
### KV Management Methods
| Legacy | New |
|-----------------------------|---------------------------------------|
| `js.KeyValue(bucket)` | `js.KeyValue(ctx, bucket)` |
| `js.CreateKeyValue(cfg)` | `js.CreateKeyValue(ctx, cfg)` |
| N/A | `js.UpdateKeyValue(ctx, cfg)` |
| N/A | `js.CreateOrUpdateKeyValue(ctx, cfg)` |
| `js.DeleteKeyValue(bucket)` | `js.DeleteKeyValue(ctx, bucket)` |
| `js.KeyValueStoreNames()` | `js.KeyValueStoreNames(ctx)` |
| `js.KeyValueStores()` | `js.KeyValueStores(ctx)` |
## Object Store
Same pattern as KV — all methods gain `context.Context`, types move to `jetstream`
package.
**Legacy:**
```go
js, _ := nc.JetStream()
os, _ := js.CreateObjectStore(&nats.ObjectStoreConfig{
Bucket: "configs",
})
os.PutString("config-1", "data")
result, _ := os.Get("config-1")
data, _ := io.ReadAll(result)
```
**New:**
```go
js, _ := jetstream.New(nc)
os, _ := js.CreateObjectStore(ctx, jetstream.ObjectStoreConfig{
Bucket: "configs",
})
os.PutString(ctx, "config-1", "data")
result, _ := os.Get(ctx, "config-1")
data, _ := io.ReadAll(result)
```
### Object Store Management Methods
| Legacy | New |
|--------------------------------|------------------------------------------|
| `js.ObjectStore(bucket)` | `js.ObjectStore(ctx, bucket)` |
| `js.CreateObjectStore(cfg)` | `js.CreateObjectStore(ctx, cfg)` |
| N/A | `js.UpdateObjectStore(ctx, cfg)` |
| N/A | `js.CreateOrUpdateObjectStore(ctx, cfg)` |
| `js.DeleteObjectStore(bucket)` | `js.DeleteObjectStore(ctx, bucket)` |
+22 -25
View File
@@ -121,7 +121,7 @@ func main() {
messageCounter++
}
fmt.Printf("received %d messages\n", messageCounter)
fmt.Printf("Received %d messages\n", messageCounter)
if msgs.Error() != nil {
fmt.Println("Error during Fetch(): ", msgs.Error())
@@ -224,7 +224,7 @@ _ = s.Purge(ctx, jetstream.WithPurgeSequence(100))
_ = s.Purge(ctx, jetstream.WithPurgeKeep(10))
```
- Get and messages from stream
- Get and delete messages from a stream
```go
// get message from stream with sequence number == 100
@@ -240,7 +240,7 @@ _ = s.DeleteMsg(ctx, 100)
- Get information about a stream
```go
// Fetches latest stream info from server
// Fetches the latest stream info from server
info, _ := s.Info(ctx)
fmt.Println(info.Config.Name)
@@ -310,7 +310,7 @@ cons2 := js.CreateOrUpdateConsumer(ctx, "ORDERS", jetstream.ConsumerConfig{
// or an illegal property is to be updated (e.g. AckPolicy)
updated, _ := js.UpdateConsumer(ctx, "ORDERS", jetstream.ConsumerConfig{
AckPolicy: jetstream.AckExplicitPolicy,
Description: "updated consumer"
Description: "updated consumer",
})
// get consumer handle
@@ -336,7 +336,7 @@ cons, _ := stream.CreateConsumer(ctx, jetstream.ConsumerConfig{
})
// get consumer handle
cons, _ = stream.Consumer(ctx, "ORDERS", "foo")
cons, _ = stream.Consumer(ctx, "foo")
// delete a consumer
stream.DeleteConsumer(ctx, "foo")
@@ -395,20 +395,19 @@ js, _ := jetstream.New(nc)
// create a consumer (this is an idempotent operation)
cons, _ := js.OrderedConsumer(ctx, "ORDERS", jetstream.OrderedConsumerConfig{
// Filter results from "ORDERS" stream by specific subject
FilterSubjects: []{"ORDERS.A"},
FilterSubjects: []string{"ORDERS.A"},
})
```
### Receiving messages from pull consumers
The `Consumer` interface covers allows fetching messages on demand, with
pre-defined batch size on bytes limit, or continuous push-like receiving of
The `Consumer` interface allows fetching messages on demand, with a
pre-defined batch size or byte limit, or continuous push-like receiving of
messages.
#### __Single fetch__
This pattern pattern allows fetching a defined number of messages in a single
RPC.
This pattern allows fetching a defined number of messages in a single RPC.
- Using `Fetch` or `FetchBytes`, consumer will return up to the provided number
of messages/bytes. By default, `Fetch()` will wait 30 seconds before timing out
@@ -481,10 +480,10 @@ single messages on demand.
Subject filtering is achieved by configuring a consumer with a `FilterSubject`
value.
##### Using `Consume()` receive messages in a callback
##### Using `Consume()` to receive messages in a callback
```go
cons, _ := js.CreateOrUpdateConsumer("ORDERS", jetstream.ConsumerConfig{
cons, _ := js.CreateOrUpdateConsumer(ctx, "ORDERS", jetstream.ConsumerConfig{
AckPolicy: jetstream.AckExplicitPolicy,
// receive messages from ORDERS.A subject only
FilterSubject: "ORDERS.A"
@@ -498,7 +497,7 @@ consContext, _ := c.Consume(func(msg jetstream.Msg) {
defer consContext.Stop()
```
Similarly to `Messages()`, `Consume()` can be supplied with options to modify
Similar to `Messages()`, `Consume()` can be supplied with options to modify
the behavior of a single pull request:
- `PullMaxMessages(int)` - up to provided number of messages will be buffered
@@ -511,7 +510,6 @@ the behavior of a single pull request:
request. If the value is set too low, the consumer will stall and not be able
to consume messages.
- `PullExpiry(time.Duration)` - timeout on a single pull request to the server
type PullThresholdMessages int
- `PullThresholdMessages(int)` - amount of messages which triggers refilling the
buffer
- `PullThresholdBytes(int)` - amount of bytes which triggers refilling the
@@ -521,10 +519,10 @@ request. An error will be triggered if at least 2 heartbeats are missed
- `ConsumeErrHandler(func (ConsumeContext, error))` - when used, sets a
custom error handler on `Consume()`, allowing e.g. tracking missing
heartbeats.
- `PullMaxMessagesWithBytesLimit` - up to the provided number of messages will
be buffered and a single fetch size will be limited to the provided value.
- `PullMaxMessagesWithBytesLimit(int, int)` - up to the provided number of messages
will be buffered and a single fetch size will be limited to the provided value.
This is an advanced option and should be used with caution. Most of the time,
`PullMaxMessages` or `PullMaxBytes` should be used instead. Note that he byte
`PullMaxMessages` or `PullMaxBytes` should be used instead. Note that the byte
limit should never be set to a value lower than the maximum message size that
can be expected from the server. If the byte limit is lower than the maximum
message size, the consumer will stall and not be able to consume messages.
@@ -568,7 +566,6 @@ iter, _ := cons.Messages(jetstream.PullMaxMessages(10), jetstream.PullMaxBytes(1
request. If the value is set too low, the consumer will stall and not be able
to consume messages.
- `PullExpiry(time.Duration)` - timeout on a single pull request to the server
type PullThresholdMessages int
- `PullThresholdMessages(int)` - amount of messages which triggers refilling the
buffer
- `PullThresholdBytes(int)` - amount of bytes which triggers refilling the
@@ -576,10 +573,10 @@ type PullThresholdMessages int
- `PullHeartbeat(time.Duration)` - idle heartbeat duration for a single pull
request. An error will be triggered if at least 2 heartbeats are missed (unless
`WithMessagesErrOnMissingHeartbeat(false)` is used)
- `PullMaxMessagesWithBytesLimit` - up to the provided number of messages will
be buffered and a single fetch size will be limited to the provided value.
- `PullMaxMessagesWithBytesLimit(int, int)` - up to the provided number of messages
will be buffered and a single fetch size will be limited to the provided value.
This is an advanced option and should be used with caution. Most of the time,
`PullMaxMessages` or `PullMaxBytes` should be used instead. Note that he byte
`PullMaxMessages` or `PullMaxBytes` should be used instead. Note that the byte
limit should never be set to a value lower than the maximum message size that
can be expected from the server. If the byte limit is lower than the maximum
message size, the consumer will stall and not be able to consume messages.
@@ -623,7 +620,7 @@ can be set to prevent the consumer from receiving more messages than it can
handle.
```go
cons, _ := js.CreateOrUpdatePushConsumer("ORDERS", jetstream.ConsumerConfig{
cons, _ := js.CreateOrUpdatePushConsumer(ctx, "ORDERS", jetstream.ConsumerConfig{
DeliverSubject: nats.NewInbox()
AckPolicy: jetstream.AckExplicitPolicy,
// receive messages from ORDERS.A subject only
@@ -672,7 +669,7 @@ setting various headers. Additionally, for `PublishMsg()` headers can be set
directly on `nats.Msg`.
```go
// All 3 implementations are work identically
// All 3 implementations work identically
ack, err := js.PublishMsg(ctx, &nats.Msg{
Data: []byte("hello"),
Subject: "ORDERS.new",
@@ -973,14 +970,14 @@ js.DeleteObjectStore(ctx, "configs")
Object Stores support Watchers, which can be used to watch for changes on
objects in a given bucket. Watcher will receive a notification on a channel when
a change occurs. By default, watcher will return latest information for all
a change occurs. By default, watcher will return the latest information for all
objects in a bucket. After sending all initial values, watcher will send nil on
the channel to signal that all initial values have been sent and it will start
sending updates when changes occur.
>__NOTE:__ Watchers do not retrieve values for objects, only metadata (containing
>information such as object name, bucket name, object size etc.). If object data
>is required, `Get` method should be used.
>is required, the `Get` method should be used.
Watcher supports several configuration options:
+3 -3
View File
@@ -50,11 +50,11 @@ const (
// apiConsumerCreateT is used to create consumers.
apiConsumerCreateT = "CONSUMER.CREATE.%s.%s"
// apiConsumerCreateT is used to create consumers.
// apiConsumerCreateWithFilterSubjectT is used to create consumers with a filter subject.
// it accepts stream name, consumer name and filter subject
apiConsumerCreateWithFilterSubjectT = "CONSUMER.CREATE.%s.%s.%s"
// apiConsumerInfoT is used to create consumers.
// apiConsumerInfoT is used to retrieve consumer information.
apiConsumerInfoT = "CONSUMER.INFO.%s.%s"
// apiRequestNextT is the prefix for the request next message(s) for a consumer in worker/pull mode.
@@ -96,7 +96,7 @@ const (
// apiMsgGetT is the endpoint to get a message.
apiMsgGetT = "STREAM.MSG.GET.%s"
// apiMsgGetT is the endpoint to perform a direct get of a message.
// apiDirectMsgGetT is the endpoint to perform a direct get of a message.
apiDirectMsgGetT = "DIRECT.GET.%s"
// apiDirectMsgGetLastBySubjectT is the endpoint to perform a direct get of a message by subject.
+31 -16
View File
@@ -44,13 +44,13 @@ type (
// deletes and restarts. They provide limited configuration options
// using [OrderedConsumerConfig].
//
// Consumer provides method for optimized continuous consumption of messages
// Consumer provides methods for optimized continuous consumption of messages
// using Consume and Messages methods, as well as simple one-off messages
// retrieval using Fetch and Next methods.
Consumer interface {
// Fetch is used to retrieve up to a provided number of messages from a
// stream. This method will send a single request and deliver either all
// requested messages unless time out is met earlier. Fetch timeout
// requested messages unless the timeout is met earlier. Fetch timeout
// defaults to 30 seconds and can be configured using FetchMaxWait
// option.
//
@@ -67,11 +67,16 @@ type (
// without additional checks. After the channel is closed,
// MessageBatch.Error() should be checked to see if there was an error
// during message delivery (e.g. missing heartbeat).
//
// NOTE: Fetch has worse performance when used to continuously retrieve
// messages in comparison to Messages or Consume methods, as it does not
// perform any optimizations (e.g. overlapping pull requests) and new
// subscription is created for each execution.
Fetch(batch int, opts ...FetchOpt) (MessageBatch, error)
// FetchBytes is used to retrieve up to a provided bytes from the
// stream. This method will send a single request and deliver the
// provided number of bytes unless time out is met earlier. FetchBytes
// FetchBytes is used to retrieve up to a provided number of bytes from
// the stream. This method will send a single request and deliver the
// provided number of bytes unless the timeout is met earlier. FetchBytes
// timeout defaults to 30 seconds and can be configured using
// FetchMaxWait option.
//
@@ -79,7 +84,7 @@ type (
// 10 seconds. For shorter requests, the idle heartbeat is disabled.
// This can be configured using FetchHeartbeat option. If a client does
// not receive a heartbeat message from a stream for more than 2 times
// the idle heartbeat setting, Fetch will return ErrNoHeartbeat.
// the idle heartbeat setting, FetchBytes will return [ErrNoHeartbeat].
//
// FetchBytes is non-blocking and returns MessageBatch, exposing a channel
// for delivered messages.
@@ -88,6 +93,11 @@ type (
// without additional checks. After the channel is closed,
// MessageBatch.Error() should be checked to see if there was an error
// during message delivery (e.g. missing heartbeat).
//
// NOTE: FetchBytes has worse performance when used to continuously
// retrieve messages in comparison to Messages or Consume methods, as it
// does not perform any optimizations (e.g. overlapping pull requests)
// and new subscription is created for each execution.
FetchBytes(maxBytes int, opts ...FetchOpt) (MessageBatch, error)
// FetchNoWait is used to retrieve up to a provided number of messages
@@ -102,6 +112,11 @@ type (
// without additional checks. After the channel is closed,
// MessageBatch.Error() should be checked to see if there was an error
// during message delivery (e.g. missing heartbeat).
//
// NOTE: FetchNoWait has worse performance when used to continuously
// retrieve messages in comparison to Messages or Consume methods, as it
// does not perform any optimizations (e.g. overlapping pull requests)
// and new subscription is created for each execution.
FetchNoWait(batch int) (MessageBatch, error)
// Consume will continuously receive messages and handle them
@@ -112,7 +127,7 @@ type (
// option, which provides information about errors encountered during
// consumption (both transient and terminal)
// - Consume can be configured to stop after a certain number of
// messages is received using StopAfter option.
// messages have been received using StopAfter option.
// - Consume can be optimized for throughput or memory usage using
// PullExpiry, PullMaxMessages, PullMaxBytes and PullHeartbeat options.
// Unless there is a specific use case, these options should not be used.
@@ -121,8 +136,8 @@ type (
// the consumer.
Consume(handler MessageHandler, opts ...PullConsumeOpt) (ConsumeContext, error)
// Messages returns MessagesContext, allowing continuously iterating
// over messages on a stream. Messages can be configured using
// Messages returns MessagesContext, allowing continuous iteration
// over messages in a stream. Messages can be configured using
// PullMessagesOpt options:
//
// - Messages can be optimized for throughput or memory usage using
@@ -134,8 +149,8 @@ type (
Messages(opts ...PullMessagesOpt) (MessagesContext, error)
// Next is used to retrieve the next message from the consumer. This
// method will block until the message is retrieved or timeout is
// reached.
// method will block until the message is retrieved or the timeout
// is reached.
Next(opts ...FetchOpt) (Msg, error)
// Info fetches current ConsumerInfo from the server.
@@ -481,12 +496,12 @@ func resumeConsumer(ctx context.Context, js *jetStream, stream, consumer string)
return pauseConsumer(ctx, js, stream, consumer, nil)
}
func validateConsumerName(dur string) error {
if dur == "" {
return fmt.Errorf("%w: '%s'", ErrInvalidConsumerName, "name is required")
func validateConsumerName(name string) error {
if name == "" {
return fmt.Errorf("%w: name is required", ErrInvalidConsumerName)
}
if strings.ContainsAny(dur, ">*. /\\") {
return fmt.Errorf("%w: '%s'", ErrInvalidConsumerName, dur)
if strings.ContainsAny(name, ">*. /\\") {
return fmt.Errorf("%w: '%s'", ErrInvalidConsumerName, name)
}
return nil
}
+21 -21
View File
@@ -216,14 +216,14 @@ type (
// settings from stream's ConsumerLimits. If neither are set, server
// default is 5 seconds.
//
// A consumer is considered inactive there are not pull requests
// received by the server (for pull consumers), or no interest detected
// on deliver subject (for push consumers), not if there are no
// A consumer is considered inactive if no pull requests are received by
// the server (for pull consumers), or no interest is detected on the
// deliver subject (for push consumers), not if there are no
// messages to be delivered.
InactiveThreshold time.Duration `json:"inactive_threshold,omitempty"`
// Replicas the number of replicas for the consumer's state. By default,
// consumers inherit the number of replicas from the stream.
// Replicas is the number of replicas for the consumer's state. By
// default, consumers inherit the number of replicas from the stream.
Replicas int `json:"num_replicas"`
// MemoryStorage is a flag to force the consumer to use memory storage
@@ -243,19 +243,19 @@ type (
// PauseUntil is for suspending the consumer until the deadline.
PauseUntil *time.Time `json:"pause_until,omitempty"`
// PriorityPolicy represents he priority policy the consumer is set to.
// PriorityPolicy represents the priority policy the consumer is set to.
// Requires nats-server v2.11.0 or later.
PriorityPolicy PriorityPolicy `json:"priority_policy,omitempty"`
// PinnedTTL represents the time after which the client will be unpinned
// if no new pull requests are sent.Used with PriorityPolicyPinned.
// if no new pull requests are sent. Used with PriorityPolicyPinned.
// Requires nats-server v2.11.0 or later.
PinnedTTL time.Duration `json:"priority_timeout,omitempty"`
// PriorityGroups is a list of priority groups this consumer supports.
PriorityGroups []string `json:"priority_groups,omitempty"`
// Fields specific for push consumers:
// Fields specific to push consumers:
// DeliverSubject is the subject to deliver messages to for push consumers
DeliverSubject string `json:"deliver_subject,omitempty"`
@@ -264,9 +264,9 @@ type (
DeliverGroup string `json:"deliver_group,omitempty"`
// FlowControl is a flag to enable flow control for the consumer.
// When set, server will regularly send an empty message with Status
// header 100 and a reply subject, consumers must reply to these
// messages to control the rate of message delivery
// When set, the server will regularly send an empty message with status
// header 100 and a reply subject. Consumers must reply to these
// messages to control the rate of message delivery.
FlowControl bool `json:"flow_control,omitempty"`
// IdleHeartbeat enables push consumer idle heartbeat messages.
@@ -302,9 +302,9 @@ type (
// ReplayPolicy defines the rate at which messages are sent to the
// consumer. If ReplayOriginalPolicy is set, messages are sent in the
// same intervals in which they were stored on stream. This can be used
// e.g. to simulate production traffic in development environments. If
// ReplayInstantPolicy is set, messages are sent as fast as possible.
// same intervals in which they were stored on the stream. This can be
// used e.g. to simulate production traffic in development environments.
// If ReplayInstantPolicy is set, messages are sent as fast as possible.
// Defaults to ReplayInstantPolicy.
ReplayPolicy ReplayPolicy `json:"replay_policy"`
@@ -317,12 +317,12 @@ type (
// (and no payload). Defaults to false.
HeadersOnly bool `json:"headers_only,omitempty"`
// Maximum number of attempts for the consumer to be recreated in a
// single recreation cycle. Defaults to unlimited.
// MaxResetAttempts is the maximum number of attempts to recreate the
// consumer in a single recovery cycle. Defaults to unlimited.
MaxResetAttempts int
// Metadata is a set of application-defined key-value pairs for
// associating metadata on the consumer. This feature requires
// associating metadata with the consumer. This feature requires
// nats-server v2.10.0 or later.
Metadata map[string]string `json:"metadata,omitempty"`
@@ -386,7 +386,7 @@ func (p *PriorityPolicy) UnmarshalJSON(data []byte) error {
case jsonString("prioritized"):
*p = PriorityPolicyPrioritized
default:
return fmt.Errorf("nats: can not unmarshal %q", data)
return fmt.Errorf("nats: cannot unmarshal %q", data)
}
return nil
}
@@ -446,7 +446,7 @@ func (p *DeliverPolicy) UnmarshalJSON(data []byte) error {
case jsonString("last_per_subject"):
*p = DeliverLastPerSubjectPolicy
default:
return fmt.Errorf("nats: can not unmarshal %q", data)
return fmt.Errorf("nats: cannot unmarshal %q", data)
}
return nil
@@ -509,7 +509,7 @@ func (p *AckPolicy) UnmarshalJSON(data []byte) error {
case jsonString("explicit"):
*p = AckExplicitPolicy
default:
return fmt.Errorf("nats: can not unmarshal %q", data)
return fmt.Errorf("nats: cannot unmarshal %q", data)
}
return nil
}
@@ -554,7 +554,7 @@ func (p *ReplayPolicy) UnmarshalJSON(data []byte) error {
case jsonString("original"):
*p = ReplayOriginalPolicy
default:
return fmt.Errorf("nats: can not unmarshal %q", data)
return fmt.Errorf("nats: cannot unmarshal %q", data)
}
return nil
}
+6 -6
View File
@@ -151,7 +151,7 @@ var (
// FilterSubjects are specified when creating consumer.
ErrDuplicateFilterSubjects JetStreamError = &jsError{apiErr: &APIError{ErrorCode: JSErrCodeDuplicateFilterSubjects, Description: "consumer cannot have both FilterSubject and FilterSubjects specified", Code: 500}}
// ErrDuplicateFilterSubjects is returned when filter subjects overlap when
// ErrOverlappingFilterSubjects is returned when filter subjects overlap when
// creating consumer.
ErrOverlappingFilterSubjects JetStreamError = &jsError{apiErr: &APIError{ErrorCode: JSErrCodeOverlappingFilterSubjects, Description: "consumer subject filters cannot overlap", Code: 500}}
@@ -167,8 +167,8 @@ var (
// already created in the server.
ErrConsumerMultipleFilterSubjectsNotSupported JetStreamError = &jsError{message: "multiple consumer filter subjects not supported by nats-server"}
// ErrConsumerNotFound is an error returned when consumer with given name
// does not exist.
// ErrConsumerNameAlreadyInUse is an error returned when attempting to create
// a consumer with a name that is already in use.
ErrConsumerNameAlreadyInUse JetStreamError = &jsError{message: "consumer name already in use"}
// ErrNotPullConsumer is returned when attempting to fetch or create pull
@@ -289,8 +289,8 @@ var (
// shutdown.
ErrServerShutdown JetStreamError = &jsError{message: "server shutdown"}
// ErrOrderedConsumerReset is returned when resetting ordered consumer fails
// due to too many attempts.
// ErrOrderedConsumerReset indicates that the ordered consumer was
// automatically reset and recreated to preserve message ordering.
ErrOrderedConsumerReset JetStreamError = &jsError{message: "recreating ordered consumer"}
// ErrOrderConsumerUsedAsFetch is returned when ordered consumer was already
@@ -353,7 +353,7 @@ var (
// deleted.
ErrKeyDeleted JetStreamError = &jsError{message: "key was deleted"}
// ErrHistoryToLarge is returned when provided history limit is larger than
// ErrHistoryTooLarge is returned when provided history limit is larger than
// 64.
ErrHistoryTooLarge JetStreamError = &jsError{message: "history limited to a max of 64"}
+28 -27
View File
@@ -34,9 +34,9 @@ type (
//
// - Publishing messages to a stream using [Publisher].
// - Managing streams using [StreamManager].
// - Managing consumers using [StreamConsumerManager]. Those are the same
// methods as on [Stream], but are available as a shortcut to a consumer
// bypassing stream lookup.
// - Managing consumers using [StreamConsumerManager]. These are the same
// methods available on [Stream], but exposed here as a shortcut that bypasses
// stream lookup.
// - Managing KeyValue stores using [KeyValueManager].
// - Managing Object Stores using [ObjectStoreManager].
//
@@ -95,8 +95,8 @@ type (
// be bound to a stream) and nats.Message.
//
// PublishMsgAsync does not guarantee that the message has been
// sent to the server and thus messages can be stored in the stream
// received by the server. It only guarantees that the message has been
// sent to the server and thus messages can be stored in the stream
// out of order in case of retries.
PublishMsgAsync(msg *nats.Msg, opts ...PublishOpt) (PubAckFuture, error)
@@ -110,7 +110,7 @@ type (
// server.
PublishAsyncComplete() <-chan struct{}
// CleanupPublisher will cleanup the publishing side of JetStreamContext.
// CleanupPublisher will clean up the publishing side of JetStreamContext.
//
// This will unsubscribe from the internal reply subject if needed.
// All pending async publishes will fail with ErrJetStreamContextClosed.
@@ -129,7 +129,7 @@ type (
// CreateOrUpdateStream and Stream methods return a [Stream] interface, allowing
// to operate on a stream.
StreamManager interface {
// CreateStream creates a new stream with given config and returns an
// CreateStream creates a new stream with the given config and returns an
// interface to operate on it. If stream with given name already exists
// and its configuration differs from the provided one,
// ErrStreamNameAlreadyInUse is returned.
@@ -139,7 +139,7 @@ type (
// ErrStreamNotFound is returned.
UpdateStream(ctx context.Context, cfg StreamConfig) (Stream, error)
// CreateOrUpdateStream creates a stream with given config. If stream
// CreateOrUpdateStream creates a stream with the given config. If stream
// already exists, it will be updated (if possible).
CreateOrUpdateStream(ctx context.Context, cfg StreamConfig) (Stream, error)
@@ -147,7 +147,7 @@ type (
// If stream does not exist, ErrStreamNotFound is returned.
Stream(ctx context.Context, stream string) (Stream, error)
// StreamNameBySubject returns a stream name stream listening on given
// StreamNameBySubject returns the name of the stream that listens on the given
// subject. If no stream is bound to given subject, ErrStreamNotFound
// is returned.
StreamNameBySubject(ctx context.Context, subject string) (string, error)
@@ -156,11 +156,11 @@ type (
// exist, ErrStreamNotFound is returned.
DeleteStream(ctx context.Context, stream string) error
// ListStreams returns StreamInfoLister, enabling iterating over a
// ListStreams returns StreamInfoLister, enabling iteration over a
// channel of stream infos.
ListStreams(context.Context, ...StreamListOpt) StreamInfoLister
// StreamNames returns a StreamNameLister, enabling iterating over a
// StreamNames returns a StreamNameLister, enabling iteration over a
// channel of stream names.
StreamNames(context.Context, ...StreamListOpt) StreamNameLister
}
@@ -169,11 +169,11 @@ type (
// available as a part of [JetStream] interface. This is an alternative to
// [Stream] interface, allowing to bypass stream lookup. CreateConsumer,
// UpdateConsumer, CreateOrUpdateConsumer and Consumer methods return a
// [Consumer] interface, allowing to operate on a consumer (e.g. consume
// [Consumer] interface, allowing operation on a consumer (e.g. consume
// messages).
StreamConsumerManager interface {
// CreateOrUpdateConsumer creates a consumer on a given stream with
// given config. If consumer already exists, it will be updated (if
// the given config. If consumer already exists, it will be updated (if
// possible). Consumer interface is returned, allowing to operate on a
// consumer (e.g. fetch messages).
CreateOrUpdateConsumer(ctx context.Context, stream string, cfg ConsumerConfig) (Consumer, error)
@@ -191,10 +191,11 @@ type (
// returned, allowing to operate on a consumer (e.g. fetch messages).
UpdateConsumer(ctx context.Context, stream string, cfg ConsumerConfig) (Consumer, error)
// OrderedConsumer returns an OrderedConsumer instance. OrderedConsumer
// are managed by the library and provide a simple way to consume
// messages from a stream. Ordered consumers are ephemeral in-memory
// pull consumers and are resilient to deletes and restarts.
// OrderedConsumer returns a client-managed ordered consumer for the given stream.
// Ordered consumers use ephemeral pull consumers and automatically reset
// themselves when ordering is lost. The client tracks state in memory and
// recreates the underlying consumer as needed, making them resilient to deletes
// and restarts while ensuring message order is preserved.
OrderedConsumer(ctx context.Context, stream string, cfg OrderedConsumerConfig) (Consumer, error)
// Consumer returns an interface to an existing consumer, allowing processing
@@ -213,7 +214,7 @@ type (
ResumeConsumer(ctx context.Context, stream string, consumer string) (*ConsumerPauseResponse, error)
// CreateOrUpdatePushConsumer creates a push consumer on a given stream with
// given config. If consumer already exists, it will be updated (if
// the given config. If consumer already exists, it will be updated (if
// possible). Consumer interface is returned, allowing to consume messages.
CreateOrUpdatePushConsumer(ctx context.Context, stream string, cfg ConsumerConfig) (PushConsumer, error)
@@ -234,7 +235,7 @@ type (
// of messages. If consumer does not exist, ErrConsumerNotFound is
// returned.
//
// It returns ErrNotPushConsumer if the consumer is not a push consumer (deliver subject is not set).
// It returns ErrNotPushConsumer if the consumer is not a push consumer (delivery subject is not set).
PushConsumer(ctx context.Context, stream string, consumer string) (PushConsumer, error)
}
@@ -565,7 +566,7 @@ func (js *jetStream) Options() JetStreamOptions {
return opts
}
// CreateStream creates a new stream with given config and returns an
// CreateStream creates a new stream with the given config and returns an
// interface to operate on it. If stream with given name already exists,
// ErrStreamNameAlreadyInUse is returned.
func (js *jetStream) CreateStream(ctx context.Context, cfg StreamConfig) (Stream, error) {
@@ -673,7 +674,7 @@ func convertStreamConfigDomains(cfg StreamConfig) (StreamConfig, error) {
}
}
// Check sources for the same.
// Check sources for the same conversion.
if len(ncfg.Sources) > 0 {
ncfg.Sources = append([]*StreamSource(nil), ncfg.Sources...)
for i, ss := range ncfg.Sources {
@@ -754,7 +755,7 @@ func (js *jetStream) UpdateStream(ctx context.Context, cfg StreamConfig) (Stream
}, nil
}
// CreateOrUpdateStream creates a stream with given config. If stream
// CreateOrUpdateStream creates a stream with the given config. If stream
// already exists, it will be updated (if possible).
func (js *jetStream) CreateOrUpdateStream(ctx context.Context, cfg StreamConfig) (Stream, error) {
s, err := js.UpdateStream(ctx, cfg)
@@ -823,7 +824,7 @@ func (js *jetStream) DeleteStream(ctx context.Context, name string) error {
}
// CreateOrUpdateConsumer creates a consumer on a given stream with
// given config. If consumer already exists, it will be updated (if
// the given config. If consumer already exists, it will be updated (if
// possible). Consumer interface is returned, allowing to operate on a
// consumer (e.g. fetch messages).
func (js *jetStream) CreateOrUpdateConsumer(ctx context.Context, stream string, cfg ConsumerConfig) (Consumer, error) {
@@ -905,7 +906,7 @@ func (js *jetStream) DeleteConsumer(ctx context.Context, stream string, name str
}
// CreateOrUpdatePushConsumer creates a push consumer on a given stream with
// given config. If consumer already exists, it will be updated (if
// the given config. If consumer already exists, it will be updated (if
// possible). Consumer interface is returned, allowing to consume messages.
func (js *jetStream) CreateOrUpdatePushConsumer(ctx context.Context, stream string, cfg ConsumerConfig) (PushConsumer, error) {
if err := validateStreamName(stream); err != nil {
@@ -1014,7 +1015,7 @@ func (js *jetStream) AccountInfo(ctx context.Context) (*AccountInfo, error) {
return &resp.AccountInfo, nil
}
// ListStreams returns StreamInfoLister, enabling iterating over a
// ListStreams returns StreamInfoLister, enabling iteration over a
// channel of stream infos.
func (js *jetStream) ListStreams(ctx context.Context, opts ...StreamListOpt) StreamInfoLister {
l := &streamLister{
@@ -1068,7 +1069,7 @@ func (s *streamLister) Err() error {
return s.err
}
// StreamNames returns a StreamNameLister, enabling iterating over a
// StreamNames returns a StreamNameLister, enabling iteration over a
// channel of stream names.
func (js *jetStream) StreamNames(ctx context.Context, opts ...StreamListOpt) StreamNameLister {
l := &streamLister{
@@ -1112,7 +1113,7 @@ func (js *jetStream) StreamNames(ctx context.Context, opts ...StreamListOpt) Str
return l
}
// StreamNameBySubject returns a stream name stream listening on given
// StreamNameBySubject returns the name of the stream bound to the given
// subject. If no stream is bound to given subject, ErrStreamNotFound
// is returned.
func (js *jetStream) StreamNameBySubject(ctx context.Context, subject string) (string, error) {
@@ -1264,7 +1265,7 @@ func (js *jetStream) cleanupReplySub() {
js.publisher.replySub = nil
}
if js.publisher.connStatusCh != nil {
close(js.publisher.connStatusCh)
js.conn.RemoveStatusListener(js.publisher.connStatusCh)
js.publisher.connStatusCh = nil
}
js.publisher.Unlock()
+2 -2
View File
@@ -250,7 +250,7 @@ type PullMaxBytes int
func (max PullMaxBytes) configureConsume(opts *consumeOpts) error {
if max <= 0 {
return fmt.Errorf("%w: max bytes must be greater then 0", ErrInvalidOption)
return fmt.Errorf("%w: max bytes must be greater than 0", ErrInvalidOption)
}
opts.MaxBytes = int(max)
return nil
@@ -258,7 +258,7 @@ func (max PullMaxBytes) configureConsume(opts *consumeOpts) error {
func (max PullMaxBytes) configureMessages(opts *consumeOpts) error {
if max <= 0 {
return fmt.Errorf("%w: max bytes must be greater then 0", ErrInvalidOption)
return fmt.Errorf("%w: max bytes must be greater than 0", ErrInvalidOption)
}
opts.MaxBytes = int(max)
return nil
+44 -6
View File
@@ -178,7 +178,7 @@ type (
// the key value store in a streaming fashion (on a channel).
ListKeys(ctx context.Context, opts ...WatchOpt) (KeyLister, error)
// ListKeysFiltered ListKeysWithFilters returns a KeyLister for filtered keys in the bucket.
// ListKeysFiltered returns a KeyLister for filtered keys in the bucket.
ListKeysFiltered(ctx context.Context, filters ...string) (KeyLister, error)
// History will return all historical values for the key (up to
@@ -241,11 +241,16 @@ type (
// subject after it's stored.
RePublish *RePublish `json:"republish,omitempty"`
// Mirror defines the consiguration for mirroring another KeyValue
// Mirror defines the configuration for mirroring another KeyValue
// store.
Mirror *StreamSource `json:"mirror,omitempty"`
// Sources defines the configuration for sources of a KeyValue store.
// If no subject transforms are defined, it is assumed that a source is
// also a KV store and subject transforms will be set to correctly map
// keys from the source KV to the current one. If subject transforms are
// defined, they will be used as is. This allows using non-kv streams as
// sources.
Sources []*StreamSource `json:"sources,omitempty"`
// Compression sets the underlying stream compression.
@@ -323,6 +328,9 @@ type (
// Metadata returns the metadata associated with the bucket.
Metadata() map[string]string
// Config returns the configuration for the bucket.
Config() KeyValueConfig
}
// KeyWatcher is what is returned when doing a watch. It can be used to
@@ -471,7 +479,6 @@ const (
kvSubjectsTmpl = "$KV.%s.>"
kvSubjectsPreTmpl = "$KV.%s."
kvSubjectsPreDomainTmpl = "%s.$KV.%s."
kvNoPending = "0"
)
const (
@@ -685,8 +692,14 @@ func (js *jetStream) prepareKeyValueConfig(ctx context.Context, cfg KeyValueConf
scfg.Mirror = m
scfg.MirrorDirect = true
} else if len(cfg.Sources) > 0 {
// For now we do not allow direct subjects for sources. If that is desired a user could use stream API directly.
for _, ss := range cfg.Sources {
// if subject transforms are already set, then use as is.
// this allows for full control of the source, e.g. using non-KV streams.
// Note that in this case, the Name is not modified and full stream name must be provided.
if len(ss.SubjectTransforms) > 0 {
scfg.Sources = append(scfg.Sources, ss)
continue
}
var sourceBucketName string
if strings.HasPrefix(ss.Name, kvBucketNamePre) {
sourceBucketName = ss.Name[len(kvBucketNamePre):]
@@ -821,6 +834,27 @@ func (s *KeyValueBucketStatus) LimitMarkerTTL() time.Duration {
return s.info.Config.SubjectDeleteMarkerTTL
}
// Config returns the configuration for the bucket.
func (s *KeyValueBucketStatus) Config() KeyValueConfig {
return KeyValueConfig{
Bucket: s.bucket,
Description: s.info.Config.Description,
MaxValueSize: s.info.Config.MaxMsgSize,
History: uint8(s.info.Config.MaxMsgsPerSubject),
TTL: s.info.Config.MaxAge,
MaxBytes: s.info.Config.MaxBytes,
Storage: s.info.Config.Storage,
Replicas: s.info.Config.Replicas,
Placement: s.info.Config.Placement,
RePublish: s.info.Config.RePublish,
Mirror: s.info.Config.Mirror,
Sources: s.info.Config.Sources,
Compression: s.info.Config.Compression != NoCompression,
Metadata: s.info.Config.Metadata,
LimitMarkerTTL: s.info.Config.SubjectDeleteMarkerTTL,
}
}
// Metadata returns the metadata associated with the bucket.
func (s *KeyValueBucketStatus) Metadata() map[string]string {
return s.info.Config.Metadata
@@ -1170,6 +1204,8 @@ func (w *watcher) Stop() error {
return w.sub.Unsubscribe()
}
// WatchFiltered will watch for any updates to keys that match the provided
// key filters. It can be configured with the same options as Watch.
func (kv *kvs) WatchFiltered(ctx context.Context, keys []string, opts ...WatchOpt) (KeyWatcher, error) {
for _, key := range keys {
if !searchKeyValid(key) {
@@ -1291,6 +1327,8 @@ func (kv *kvs) WatchFiltered(ctx context.Context, keys []string, opts ...WatchOp
return nil, err
}
sub.SetClosedHandler(func(_ string) {
w.mu.Lock()
defer w.mu.Unlock()
close(w.updates)
})
// If there were no pending messages at the time of the creation
@@ -1352,7 +1390,7 @@ type keyLister struct {
keys chan string
}
// Keys will return all keys.
// ListKeys will return all keys.
func (kv *kvs) ListKeys(ctx context.Context, opts ...WatchOpt) (KeyLister, error) {
opts = append(opts, IgnoreDeletes(), MetaOnly())
watcher, err := kv.WatchAll(ctx, opts...)
@@ -1379,7 +1417,7 @@ func (kv *kvs) ListKeys(ctx context.Context, opts ...WatchOpt) (KeyLister, error
return kl, nil
}
// ListKeysWithFilters returns a channel of keys matching the provided filters using WatchFiltered.
// ListKeysFiltered returns a KeyLister for filtered keys in the bucket.
func (kv *kvs) ListKeysFiltered(ctx context.Context, filters ...string) (KeyLister, error) {
watcher, err := kv.WatchFiltered(ctx, filters, IgnoreDeletes(), MetaOnly())
if err != nil {
+5 -5
View File
@@ -29,7 +29,7 @@ func (opt watchOptFn) configureWatcher(opts *watchOpts) error {
func IncludeHistory() WatchOpt {
return watchOptFn(func(opts *watchOpts) error {
if opts.updatesOnly {
return fmt.Errorf("%w: include history can not be used with updates only", ErrInvalidOption)
return fmt.Errorf("%w: include history cannot be used with updates only", ErrInvalidOption)
}
opts.includeHistory = true
return nil
@@ -41,14 +41,14 @@ func IncludeHistory() WatchOpt {
func UpdatesOnly() WatchOpt {
return watchOptFn(func(opts *watchOpts) error {
if opts.includeHistory {
return fmt.Errorf("%w: updates only can not be used with include history", ErrInvalidOption)
return fmt.Errorf("%w: updates only cannot be used with include history", ErrInvalidOption)
}
opts.updatesOnly = true
return nil
})
}
// IgnoreDeletes will have the key watcher not pass any deleted keys.
// IgnoreDeletes will prevent the key watcher from passing any deleted keys.
func IgnoreDeletes() WatchOpt {
return watchOptFn(func(opts *watchOpts) error {
opts.ignoreDeletes = true
@@ -56,7 +56,7 @@ func IgnoreDeletes() WatchOpt {
})
}
// MetaOnly instructs the key watcher to retrieve only the entry meta data, not
// MetaOnly instructs the key watcher to retrieve only the entry metadata, not
// the entry value.
func MetaOnly() WatchOpt {
return watchOptFn(func(opts *watchOpts) error {
@@ -121,7 +121,7 @@ func (opt createOptFn) configureCreate(opts *createOpts) error {
}
// KeyTTL sets the TTL for the key. This is the time after which the key will be
// automatically deleted. The TTL is set when the key is created and can not be
// automatically deleted. The TTL is set when the key is created and cannot be
// changed later. This requires LimitMarkerTTL to be enabled on the bucket.
func KeyTTL(ttl time.Duration) KVCreateOpt {
return createOptFn(func(opts *createOpts) error {
+6 -6
View File
@@ -180,7 +180,7 @@ const (
// level. Server will reject the message if it is not the case.
//
// This can be set when publishing messages using [WithExpectLastSequence]
// option. option.
// option.
ExpectedLastSeqHeader = "Nats-Expected-Last-Sequence"
// ExpectedLastSubjSeqHeader contains the expected last sequence number on
@@ -227,7 +227,7 @@ const (
// SequenceHeader contains the original sequence number of the message.
SequenceHeader = "Nats-Sequence"
// TimeStampHeader contains the original timestamp of the message.
// TimeStampHeaer contains the original timestamp of the message.
TimeStampHeaer = "Nats-Time-Stamp"
// SubjectHeader contains the original subject the message was published to.
@@ -291,7 +291,7 @@ func (m *jetStreamMsg) Headers() nats.Header {
return m.msg.Header
}
// Subject returns a subject on which a message is published.
// Subject returns a subject on which a message was published/received.
func (m *jetStreamMsg) Subject() string {
return m.msg.Subject
}
@@ -408,9 +408,9 @@ func (m *jetStreamMsg) checkReply() error {
return nil
}
// Returns if the given message is a user message or not, and if
// checkSts() is true, returns appropriate error based on the
// content of the status (404, etc..)
// checkMsg returns whether the given message is a user message or a control message.
// If the status header is present, it returns an appropriate error based
// on the status code (404, etc.)
func checkMsg(msg *nats.Msg) (bool, error) {
// If payload or no header, consider this a user message
if len(msg.Data) > 0 || len(msg.Header) == 0 {
+10 -1
View File
@@ -675,8 +675,17 @@ func (obs *obs) Put(ctx context.Context, meta ObjectMeta, r io.Reader) (*ObjectI
return perr
}
opts := []JetStreamOpt{
WithPublishAsyncErrHandler(func(js JetStream, _ *nats.Msg, err error) { setErr(err) }),
}
// if context deadline is not set, use default JetStream timeout (per publish)
if _, ok := ctx.Deadline(); !ok {
opts = append(opts, WithPublishAsyncTimeout(obs.js.opts.DefaultTimeout))
}
// Create our own JS context to handle errors etc.
pubJS, err := New(obs.js.conn, WithPublishAsyncErrHandler(func(js JetStream, _ *nats.Msg, err error) { setErr(err) }))
pubJS, err := New(obs.js.conn, opts...)
if err != nil {
return nil, err
}
+5 -1
View File
@@ -105,7 +105,11 @@ func (p *pushConsumer) Consume(handler MessageHandler, opts ...PushConsumeOpt) (
}
var err error
sub.subscription, err = p.js.conn.Subscribe(p.info.Config.DeliverSubject, internalHandler)
if p.info.Config.DeliverGroup != "" {
sub.subscription, err = p.js.conn.QueueSubscribe(p.info.Config.DeliverSubject, p.info.Config.DeliverGroup, internalHandler)
} else {
sub.subscription, err = p.js.conn.Subscribe(p.info.Config.DeliverSubject, internalHandler)
}
if err != nil {
return nil, err
}
+1 -1
View File
@@ -48,7 +48,7 @@ type (
GetMsg(ctx context.Context, seq uint64, opts ...GetMsgOpt) (*RawStreamMsg, error)
// GetLastMsgForSubject retrieves the last raw stream message stored in
// JetStream on a given subject subject.
// JetStream on a given subject.
GetLastMsgForSubject(ctx context.Context, subject string) (*RawStreamMsg, error)
// DeleteMsg deletes a message from a stream.
+10 -8
View File
@@ -296,8 +296,7 @@ type jsOpts struct {
}
const (
defaultRequestWait = 5 * time.Second
defaultAccountCheck = 20 * time.Second
defaultRequestWait = 5 * time.Second
)
// JetStream returns a JetStreamContext for messaging and stream management.
@@ -1132,7 +1131,7 @@ func (js *js) PublishMsgAsync(m *Msg, opts ...PubOpt) (PubAckFuture, error) {
if err != nil {
return nil, err
}
if err := js.nc.publish(m.Subject, reply, hdr, m.Data); err != nil {
if err := js.nc.publish(m.Subject, reply, false, hdr, m.Data); err != nil {
js.clearPAF(id)
return nil, err
}
@@ -2835,12 +2834,15 @@ func ConsumerFilterSubjects(subjects ...string) SubOpt {
func (sub *Subscription) ConsumerInfo() (*ConsumerInfo, error) {
sub.mu.Lock()
// TODO(dlc) - Better way to mark especially if we attach.
if sub.jsi == nil || sub.jsi.consumer == _EMPTY_ {
if sub.jsi.ordered {
sub.mu.Unlock()
if sub.jsi == nil {
sub.mu.Unlock()
return nil, ErrTypeSubscription
} else if sub.jsi.consumer == _EMPTY_ {
ordered := sub.jsi.ordered
sub.mu.Unlock()
if ordered {
return nil, ErrConsumerInfoOnOrderedReset
}
sub.mu.Unlock()
return nil, ErrTypeSubscription
}
@@ -3560,7 +3562,7 @@ func (js *js) apiRequestWithContext(ctx context.Context, subj string, data []byt
}
if js.opts.shouldTrace {
ctrace := js.opts.ctrace
if ctrace.RequestSent != nil {
if ctrace.ResponseReceived != nil {
ctrace.ResponseReceived(subj, resp.Data, resp.Header)
}
}
+21 -1
View File
@@ -105,6 +105,9 @@ type KeyValueStatus interface {
// IsCompressed indicates if the data is compressed on disk
IsCompressed() bool
// Config returns the original configuration used to create the bucket
Config() KeyValueConfig
}
// KeyWatcher is what is returned when doing a watch.
@@ -354,7 +357,6 @@ const (
kvSubjectsTmpl = "$KV.%s.>"
kvSubjectsPreTmpl = "$KV.%s."
kvSubjectsPreDomainTmpl = "%s.$KV.%s."
kvNoPending = "0"
)
// Regex for valid keys and buckets.
@@ -1209,6 +1211,24 @@ func (s *KeyValueBucketStatus) Bytes() uint64 { return s.nfo.State.Bytes }
// IsCompressed indicates if the data is compressed on disk
func (s *KeyValueBucketStatus) IsCompressed() bool { return s.nfo.Config.Compression != NoCompression }
func (s *KeyValueBucketStatus) Config() KeyValueConfig {
return KeyValueConfig{
Bucket: s.bucket,
Description: s.nfo.Config.Description,
MaxValueSize: s.nfo.Config.MaxMsgSize,
History: uint8(s.nfo.Config.MaxMsgsPerSubject),
TTL: s.nfo.Config.MaxAge,
MaxBytes: s.nfo.Config.MaxBytes,
Storage: s.nfo.Config.Storage,
Replicas: s.nfo.Config.Replicas,
Placement: s.nfo.Config.Placement,
RePublish: s.nfo.Config.RePublish,
Mirror: s.nfo.Config.Mirror,
Sources: s.nfo.Config.Sources,
Compression: s.nfo.Config.Compression != NoCompression,
}
}
// Status retrieves the status and configuration of a bucket
func (kv *kvs) Status() (KeyValueStatus, error) {
nfo, err := kv.js.StreamInfo(kv.stream)
+377 -70
View File
@@ -48,7 +48,7 @@ import (
// Default Constants
const (
Version = "1.47.0"
Version = "1.49.0"
DefaultURL = "nats://127.0.0.1:4222"
DefaultPort = 4222
DefaultMaxReconnect = 60
@@ -152,6 +152,8 @@ var (
ErrConnectionNotTLS = errors.New("nats: connection is not tls")
ErrMaxSubscriptionsExceeded = errors.New("nats: server maximum subscriptions exceeded")
ErrWebSocketHeadersAlreadySet = errors.New("nats: websocket connection headers already set")
ErrServerNotInPool = errors.New("nats: selected server is not in the pool")
ErrMixingWebsocketSchemes = errors.New("nats: mixing of websocket and non websocket URLs is not allowed")
)
// GetDefaultOptions returns default configuration options for the client.
@@ -242,7 +244,7 @@ type SignatureHandler func([]byte) ([]byte, error)
// AuthTokenHandler is used to generate a new token.
type AuthTokenHandler func() string
// UserInfoCB is used to pass the username and password when establishing connection.
// UserInfoCB is used to pass the username and password when establishing a connection.
type UserInfoCB func() (string, string)
// ReconnectDelayHandler is used to get from the user the desired
@@ -254,6 +256,25 @@ type ReconnectDelayHandler func(attempts int) time.Duration
// WebSocketHeadersHandler is an optional callback handler for generating token used for WebSocket connections.
type WebSocketHeadersHandler func() (http.Header, error)
// ReconnectToServerHandler is used to determine the server to reconnect to during
// the reconnection process. The handler receives a snapshot of available servers
// in the pool and should return a pointer to one of the servers from the provided
// slice and a delay before attempting the connection.
//
// Return values:
// - *Server: The server to connect to. Must be a pointer to an element from the
// provided servers slice. If the returned server is not in the pool, the library
// will fire ReconnectErrCB with ErrServerNotInPool and fall back to default
// server selection.
// - time.Duration: The delay before attempting the connection. If zero, the
// connection attempt is made immediately. If non-zero, the library sleeps
// for exactly that duration before attempting.
//
// MaxReconnect limits are enforced automatically: servers exceeding the configured
// MaxReconnect attempts are removed from the pool before the handler is called.
// To disable this limit, set MaxReconnect to a negative value.
type ReconnectToServerHandler func([]Server, ServerInfo) (*Server, time.Duration)
// asyncCB is used to preserve order for async callbacks.
type asyncCB struct {
f func()
@@ -350,7 +371,7 @@ type Options struct {
// Defaults to 60.
MaxReconnect int
// ReconnectWait sets the time to backoff after attempting a reconnect
// ReconnectWait sets the time to back off after attempting a reconnect
// to a server that we were already connected to previously.
// Defaults to 2s.
ReconnectWait time.Duration
@@ -377,7 +398,7 @@ type Options struct {
// Defaults to 2s.
Timeout time.Duration
// DrainTimeout sets the timeout for a Drain Operation to complete.
// DrainTimeout sets the timeout for a drain operation to complete.
// Defaults to 30s.
DrainTimeout time.Duration
@@ -431,18 +452,25 @@ type Options struct {
AsyncErrorCB ErrHandler
// ReconnectErrCB sets the callback that is invoked whenever a
// reconnect attempt failed
// reconnect attempt fails.
ReconnectErrCB ConnErrHandler
// ReconnectToServerCB is called before reconnection attempt.
// It is used to determine the server to reconnect to out of
// the list of available servers.
// If a reconnect attempt is not successful, this callback will
// be called again before the next attempt.
ReconnectToServerCB ReconnectToServerHandler
// ReconnectBufSize is the size of the backing bufio during reconnect.
// Once this has been exhausted publish operations will return an error.
// Defaults to 8388608 bytes (8MB).
ReconnectBufSize int
// SubChanLen is the size of the buffered channel used between the socket
// Go routine and the message delivery for SyncSubscriptions.
// goroutine and the message delivery for SyncSubscriptions.
// NOTE: This does not affect AsyncSubscriptions which are
// dictated by PendingLimits()
// dictated by PendingLimits().
// Defaults to 65536.
SubChanLen int
@@ -470,7 +498,8 @@ type Options struct {
// Token sets the token to be used when connecting to a server.
Token string
// TokenHandler designates the function used to generate the token to be used when connecting to a server.
// TokenHandler designates the function used to generate the token
// used when connecting to a server.
TokenHandler AuthTokenHandler
// Dialer allows a custom net.Dialer when forming connections.
@@ -532,8 +561,17 @@ type Options struct {
// WebSocketConnectionHeaders is an optional http request headers to be sent with the WebSocket request.
WebSocketConnectionHeaders http.Header
// WebSocketConnectionHeadersHandler is an optional callback handler for generating token used for WebSocket connections.
// WebSocketConnectionHeadersHandler is an optional callback handler for generating token used for WebSocket connections.
WebSocketConnectionHeadersHandler WebSocketHeadersHandler
// SkipSubjectValidation will disable publish subject validation.
// NOTE: This is not recommended in general, as the performance gain is minimal
// and may lead to breaking protocol.
SkipSubjectValidation bool
// IgnoreDiscoveredServers will disable adding advertised server URLs
// from INFO messages to the server pool.
IgnoreDiscoveredServers bool
}
const (
@@ -572,14 +610,14 @@ type Conn struct {
// Modifying the configuration of a running Conn is a race.
Opts Options
wg sync.WaitGroup
srvPool []*srv
current *srv
srvPool []*Server
current *Server
urls map[string]struct{} // Keep track of all known URLs (used by processInfo)
conn net.Conn
bw *natsWriter
br *natsReader
fch chan struct{}
info serverInfo
info ServerInfo
ssid int64
subsMu sync.RWMutex
subs map[int64]*Subscription
@@ -810,18 +848,27 @@ type Statistics struct {
Reconnects uint64
}
// Tracks individual backend servers.
type srv struct {
url *url.URL
// Server represents a server in the pool of servers that the client can connect to.
type Server struct {
URL *url.URL
Reconnects int
didConnect bool
reconnects int
lastErr error
isImplicit bool
tlsName string
}
// The INFO block received from the server.
type serverInfo struct {
func (s Server) clone() Server {
c := s
if s.URL != nil {
u := *s.URL
c.URL = &u
}
return c
}
// ServerInfo represents the information about the server that is sent in the INFO protocol message.
type ServerInfo struct {
ID string `json:"server_id"`
Name string `json:"server_name"`
Proto int `json:"proto"`
@@ -902,7 +949,7 @@ func Name(name string) Option {
}
}
// InProcessServer is an Option that will try to establish a direction to a NATS server
// InProcessServer is an Option that will try to establish a connection to a NATS server
// running within the process instead of dialing via TCP.
func InProcessServer(server InProcessConnProvider) Option {
return func(o *Options) error {
@@ -1082,6 +1129,19 @@ func CustomReconnectDelay(cb ReconnectDelayHandler) Option {
}
}
// ReconnectToServer is an Option to set a custom server selection callback
// for reconnection attempts. The callback receives a snapshot of available
// servers and must return a server from the pool to connect to and a delay
// duration before attempting the connection.
//
// See ReconnectToServerHandler for detailed documentation and usage examples.
func ReconnectToServer(cb ReconnectToServerHandler) Option {
return func(o *Options) error {
o.ReconnectToServerCB = cb
return nil
}
}
// PingInterval is an Option to set the period for client ping commands.
// Defaults to 2m.
func PingInterval(t time.Duration) Option {
@@ -1512,6 +1572,29 @@ func WebSocketConnectionHeadersHandler(cb WebSocketHeadersHandler) Option {
}
}
// SkipSubjectValidation is an Option to skip subject validation when
// publishing messages.
// By default, subject validation is performed to ensure that subjects
// are valid according to NATS subject syntax (no spaces newlines and tabs).
// NOTE: It is not recommended to use this option as the performance gain
// is minimal and disabling subject validation can lead to breaking protocol
// rules.
func SkipSubjectValidation() Option {
return func(o *Options) error {
o.SkipSubjectValidation = true
return nil
}
}
// IgnoreDiscoveredServers is an Option to disable adding advertised
// server URLs from INFO messages to the server pool.
func IgnoreDiscoveredServers() Option {
return func(o *Options) error {
o.IgnoreDiscoveredServers = true
return nil
}
}
// Handler processing
// SetDisconnectHandler will set the disconnect event handler.
@@ -1771,7 +1854,7 @@ const (
)
// Return the currently selected server
func (nc *Conn) currentServer() (int, *srv) {
func (nc *Conn) currentServer() (int, *Server) {
for i, s := range nc.srvPool {
if s == nil {
continue
@@ -1785,7 +1868,7 @@ func (nc *Conn) currentServer() (int, *srv) {
// Pop the current server and put onto the end of the list. Select head of list as long
// as number of reconnect attempts under MaxReconnect.
func (nc *Conn) selectNextServer() (*srv, error) {
func (nc *Conn) selectNextServer() (*Server, error) {
i, s := nc.currentServer()
if i < 0 {
return nil, ErrNoServers
@@ -1794,7 +1877,7 @@ func (nc *Conn) selectNextServer() (*srv, error) {
num := len(sp)
copy(sp[i:num-1], sp[i+1:num])
maxReconnect := nc.Opts.MaxReconnect
if maxReconnect < 0 || s.reconnects < maxReconnect {
if maxReconnect < 0 || s.Reconnects < maxReconnect {
nc.srvPool[num-1] = s
} else {
nc.srvPool = sp[0 : num-1]
@@ -1830,7 +1913,7 @@ const tlsScheme = "tls"
// Server Options. We will randomize the server pool unless
// the NoRandomize flag is set.
func (nc *Conn) setupServerPool() error {
nc.srvPool = make([]*srv, 0, srvPoolSize)
nc.srvPool = make([]*Server, 0, srvPoolSize)
nc.urls = make(map[string]struct{}, srvPoolSize)
// Create srv objects from each url string in nc.Opts.Servers
@@ -1867,7 +1950,7 @@ func (nc *Conn) setupServerPool() error {
// Check for Scheme hint to move to TLS mode.
for _, srv := range nc.srvPool {
if srv.url.Scheme == tlsScheme || srv.url.Scheme == wsSchemeTLS {
if srv.URL.Scheme == tlsScheme || srv.URL.Scheme == wsSchemeTLS {
// FIXME(dlc), this is for all in the pool, should be case by case.
nc.Opts.Secure = true
if nc.Opts.TLSConfig == nil {
@@ -1898,8 +1981,10 @@ func hostIsIP(u *url.URL) bool {
return net.ParseIP(u.Hostname()) != nil
}
// addURLToPool adds an entry to the server pool
func (nc *Conn) addURLToPool(sURL string, implicit, saveTLSName bool) error {
// parseServerURL parses a server URL string into a Server struct.
// It handles scheme defaults and port defaults. Does not validate websocket consistency.
// Returns the parsed Server and whether it's a websocket URL.
func (nc *Conn) parseServerURL(sURL string, implicit, saveTLSName bool) (*Server, error) {
if !strings.Contains(sURL, "://") {
sURL = fmt.Sprintf("%s://%s", nc.connScheme(), sURL)
}
@@ -1910,7 +1995,7 @@ func (nc *Conn) addURLToPool(sURL string, implicit, saveTLSName bool) error {
for i := 0; i < 2; i++ {
u, err = url.Parse(sURL)
if err != nil {
return err
return nil, err
}
if u.Port() != "" {
break
@@ -1930,19 +2015,9 @@ func (nc *Conn) addURLToPool(sURL string, implicit, saveTLSName bool) error {
}
}
isWS := isWebsocketScheme(u)
// We don't support mix and match of websocket and non websocket URLs.
// If this is the first URL, then we accept and switch the global state
// to websocket. After that, we will know how to reject mixed URLs.
if len(nc.srvPool) == 0 {
nc.ws = isWS
} else if isWS && !nc.ws || !isWS && nc.ws {
return errors.New("mixing of websocket and non websocket URLs is not allowed")
}
var tlsName string
if implicit {
curl := nc.current.url
curl := nc.current.URL
// Check to see if we do not have a url.User but current connected
// url does. If so copy over.
if u.User == nil && curl.User != nil {
@@ -1956,9 +2031,29 @@ func (nc *Conn) addURLToPool(sURL string, implicit, saveTLSName bool) error {
}
}
s := &srv{url: u, isImplicit: implicit, tlsName: tlsName}
s := &Server{URL: u, isImplicit: implicit, tlsName: tlsName}
return s, nil
}
// addURLToPool adds an entry to the server pool
func (nc *Conn) addURLToPool(sURL string, implicit, saveTLSName bool) error {
s, err := nc.parseServerURL(sURL, implicit, saveTLSName)
if err != nil {
return err
}
// We don't support mix and match of websocket and non websocket URLs.
// If this is the first URL, then we accept and switch the global state
// to websocket. After that, we will know how to reject mixed URLs.
isWS := isWebsocketScheme(s.URL)
if len(nc.srvPool) == 0 {
nc.ws = isWS
} else if isWS != nc.ws {
return ErrMixingWebsocketSchemes
}
nc.srvPool = append(nc.srvPool, s)
nc.urls[u.Host] = struct{}{}
nc.urls[s.URL.Host] = struct{}{}
return nil
}
@@ -2154,7 +2249,7 @@ func (nc *Conn) createConn() (err error) {
// We will auto-expand host names if they resolve to multiple IPs
hosts := []string{}
u := nc.current.url
u := nc.current.URL
if !nc.Opts.SkipHostLookup && net.ParseIP(u.Hostname()) == nil {
addrs, _ := net.LookupHost(u.Hostname())
@@ -2240,7 +2335,7 @@ func (nc *Conn) makeTLSConn() error {
if nc.current.tlsName != _EMPTY_ {
tlsCopy.ServerName = nc.current.tlsName
} else {
h, _, _ := net.SplitHostPort(nc.current.url.Host)
h, _, _ := net.SplitHostPort(nc.current.URL.Host)
tlsCopy.ServerName = h
}
}
@@ -2338,7 +2433,7 @@ func (nc *Conn) ConnectedUrl() string {
if nc.status != CONNECTED {
return _EMPTY_
}
return nc.current.url.String()
return nc.current.URL.String()
}
// ConnectedUrlRedacted reports the connected server's URL with passwords redacted
@@ -2353,7 +2448,7 @@ func (nc *Conn) ConnectedUrlRedacted() string {
if nc.status != CONNECTED {
return _EMPTY_
}
return nc.current.url.Redacted()
return nc.current.URL.Redacted()
}
// ConnectedAddr returns the connected server's IP
@@ -2568,7 +2663,7 @@ func (nc *Conn) connect() (bool, error) {
if err == nil {
nc.current.didConnect = true
nc.current.reconnects = 0
nc.current.Reconnects = 0
nc.current.lastErr = nil
break
} else {
@@ -2685,7 +2780,7 @@ func (nc *Conn) sendProto(proto string) {
func (nc *Conn) connectProto() (string, error) {
o := nc.Opts
var nkey, sig, user, pass, token, ujwt string
u := nc.current.url.User
u := nc.current.URL.User
if u != nil {
// if no password, assume username is authToken
if _, ok := u.Password(); !ok {
@@ -2980,17 +3075,88 @@ func (nc *Conn) doReconnect(err error, forceReconnect bool) {
}
for i := 0; len(nc.srvPool) > 0; {
cur, err := nc.selectNextServer()
if err != nil {
nc.err = err
break
var err error
var cur *Server
var callbackDelay time.Duration
var useCallbackDelay bool
if nc.Opts.ReconnectToServerCB != nil {
// Enforce MaxReconnect limits before calling the callback
maxReconnect := nc.Opts.MaxReconnect
if maxReconnect >= 0 {
// Remove servers that have exceeded MaxReconnect attempts
filtered := make([]*Server, 0, len(nc.srvPool))
for _, srv := range nc.srvPool {
if srv != nil && srv.Reconnects < maxReconnect {
filtered = append(filtered, srv)
}
}
nc.srvPool = filtered
}
// Check if we still have servers after filtering
if len(nc.srvPool) == 0 {
nc.err = ErrNoServers
break
}
// Copy server values to avoid caller modifying internal state
srvVals := make([]Server, len(nc.srvPool))
for idx, srv := range nc.srvPool {
if srv != nil {
srvVals[idx] = srv.clone()
}
}
var selectedSrv *Server
selectedSrv, callbackDelay = nc.Opts.ReconnectToServerCB(srvVals, nc.info)
if selectedSrv != nil {
idx := slices.IndexFunc(nc.srvPool, func(srv *Server) bool {
return srv != nil && srv.URL.String() == selectedSrv.URL.String()
})
if idx != -1 {
cur = nc.srvPool[idx]
nc.current = cur
useCallbackDelay = true
} else if reconnectErrCB := nc.Opts.ReconnectErrCB; reconnectErrCB != nil {
nc.ach.push(func() { reconnectErrCB(nc, ErrServerNotInPool) })
}
}
}
var doSleep bool
if cur == nil {
cur, err = nc.selectNextServer()
if err != nil {
nc.err = err
break
}
doSleep = i+1 >= len(nc.srvPool) && !forceReconnect
}
doSleep := i+1 >= len(nc.srvPool) && !forceReconnect
forceReconnect = false
nc.mu.Unlock()
if !doSleep {
if useCallbackDelay {
if callbackDelay > 0 {
i = 0
if rt == nil {
rt = time.NewTimer(callbackDelay)
} else {
rt.Reset(callbackDelay)
}
select {
case <-rqch:
rt.Stop()
nc.mu.Lock()
nc.rqch = make(chan struct{})
nc.mu.Unlock()
case <-rt.C:
}
} else {
runtime.Gosched()
}
} else if !doSleep {
i++
// Release the lock to give a chance to a concurrent nc.Close() to break the loop.
runtime.Gosched()
@@ -3036,7 +3202,7 @@ func (nc *Conn) doReconnect(err error, forceReconnect bool) {
}
// Mark that we tried a reconnect
cur.reconnects++
cur.Reconnects++
// Try to create a new connection
err = nc.createConn()
@@ -3071,7 +3237,7 @@ func (nc *Conn) doReconnect(err error, forceReconnect bool) {
// Clear out server stats for the server we connected to..
cur.didConnect = true
cur.reconnects = 0
cur.Reconnects = 0
// Send existing subscription state
nc.resendSubscriptions()
@@ -3739,7 +3905,7 @@ func (nc *Conn) processInfo(info string) error {
if info == _EMPTY_ {
return nil
}
var ncInfo serverInfo
var ncInfo ServerInfo
if err := json.Unmarshal([]byte(info), &ncInfo); err != nil {
return err
}
@@ -3750,7 +3916,7 @@ func (nc *Conn) processInfo(info string) error {
// if advertise is disabled on that server, or servers that
// did not include themselves in the async INFO protocol.
// If empty, do not remove the implicit servers from the pool.
if len(nc.info.ConnectURLs) == 0 {
if len(nc.info.ConnectURLs) == 0 || nc.Opts.IgnoreDiscoveredServers {
if !nc.initc && ncInfo.LameDuckMode && nc.Opts.LameDuckModeHandler != nil {
nc.ach.push(func() { nc.Opts.LameDuckModeHandler(nc) })
}
@@ -3773,7 +3939,7 @@ func (nc *Conn) processInfo(info string) error {
sp := nc.srvPool
for i := 0; i < len(sp); i++ {
srv := sp[i]
curl := srv.url.Host
curl := srv.URL.Host
// Check if this URL is in the INFO protocol
_, inInfo := tmp[curl]
// Remove from the temp map so that at the end we are left with only
@@ -3781,7 +3947,7 @@ func (nc *Conn) processInfo(info string) error {
delete(tmp, curl)
// Keep servers that were set through Options, but also the one that
// we are currently connected to (even if it is a discovered server).
if !srv.isImplicit || srv.url == nc.current.url {
if !srv.isImplicit || srv.URL == nc.current.URL {
continue
}
if !inInfo {
@@ -3793,7 +3959,7 @@ func (nc *Conn) processInfo(info string) error {
}
}
// Figure out if we should save off the current non-IP hostname if we encounter a bare IP.
saveTLS := nc.current != nil && !hostIsIP(nc.current.url)
saveTLS := nc.current != nil && !hostIsIP(nc.current.URL)
// If there are any left in the tmp map, these are new (or restarted) servers
// and need to be added to the pool.
@@ -3916,7 +4082,7 @@ func (nc *Conn) kickFlusher() {
// argument is left untouched and needs to be correctly interpreted on
// the receiver.
func (nc *Conn) Publish(subj string, data []byte) error {
return nc.publish(subj, _EMPTY_, nil, data)
return nc.publish(subj, _EMPTY_, false, nil, data)
}
// Header represents the optional Header for a NATS message,
@@ -4059,27 +4225,71 @@ func (nc *Conn) PublishMsg(m *Msg) error {
if err != nil {
return err
}
return nc.publish(m.Subject, m.Reply, hdr, m.Data)
validateReply := m.Reply != _EMPTY_
return nc.publish(m.Subject, m.Reply, validateReply, hdr, m.Data)
}
// PublishRequest will perform a Publish() expecting a response on the
// reply subject. Use Request() for automatically waiting for a response
// inline.
func (nc *Conn) PublishRequest(subj, reply string, data []byte) error {
return nc.publish(subj, reply, nil, data)
return nc.publish(subj, reply, true, nil, data)
}
// Used for handrolled Itoa
const digits = "0123456789"
// validateSubject checks if the subject contains characters that break the NATS protocol.
// Uses an adaptive algorithm: manual loop for short subjects (< 16 chars) and
// SIMD-optimized strings.IndexByte for longer subjects.
func validateSubject(subj string) error {
if subj == "" {
return ErrBadSubject
}
// Adaptive threshold based on benchmark data showing crossover at ~15-20 characters.
const lengthThreshold = 16
if len(subj) < lengthThreshold {
// Fast path for short subjects (< 16 chars)
// Short-circuit on non-control characters.
for i := range len(subj) {
c := subj[i]
if c <= ' ' && (c == ' ' || c == '\t' || c == '\r' || c == '\n') {
return ErrBadSubject
}
}
return nil
}
// Optimized path for long subjects (>= 16 chars)
// Uses SIMD-optimized strings.IndexByte (processes 16+ bytes per instruction)
if strings.IndexByte(subj, ' ') >= 0 ||
strings.IndexByte(subj, '\t') >= 0 ||
strings.IndexByte(subj, '\r') >= 0 ||
strings.IndexByte(subj, '\n') >= 0 {
return ErrBadSubject
}
return nil
}
// publish is the internal function to publish messages to a nats-server.
// Sends a protocol data message by queuing into the bufio writer
// and kicking the flush go routine. These writes should be protected.
func (nc *Conn) publish(subj, reply string, hdr, data []byte) error {
func (nc *Conn) publish(subj, reply string, validateReply bool, hdr, data []byte) error {
if nc == nil {
return ErrInvalidConnection
}
if subj == "" {
if !nc.Opts.SkipSubjectValidation {
if err := validateSubject(subj); err != nil {
return err
}
if validateReply {
if err := validateSubject(reply); err != nil {
return ErrBadSubject
}
}
} else if subj == _EMPTY_ {
return ErrBadSubject
}
nc.mu.Lock()
@@ -4245,7 +4455,7 @@ func (nc *Conn) createNewRequestAndSend(subj string, hdr, data []byte) (chan *Ms
}
nc.mu.Unlock()
if err := nc.publish(subj, respInbox, hdr, data); err != nil {
if err := nc.publish(subj, respInbox, false, hdr, data); err != nil {
return nil, token, err
}
@@ -4341,7 +4551,7 @@ func (nc *Conn) oldRequest(subj string, hdr, data []byte, timeout time.Duration)
s.AutoUnsubscribe(1)
defer s.Unsubscribe()
err = nc.publish(subj, inbox, hdr, data)
err = nc.publish(subj, inbox, false, hdr, data)
if err != nil {
return nil, err
}
@@ -4702,7 +4912,7 @@ func (s *Subscription) IsValid() bool {
//
// For a JetStream subscription, if the library has created the JetStream
// consumer, the library will send a DeleteConsumer request to the server
// when the Drain operation completes. If a failure occurs when deleting
// when the drain operation completes. If a failure occurs when deleting
// the JetStream consumer, an error will be reported to the asynchronous
// error callback.
// If you do not wish the JetStream consumer to be automatically deleted,
@@ -5852,7 +6062,7 @@ func (nc *Conn) getServers(implicitOnly bool) []string {
if implicitOnly && !nc.srvPool[i].isImplicit {
continue
}
url := nc.srvPool[i].url
url := nc.srvPool[i].URL
servers = append(servers, fmt.Sprintf("%s://%s", url.Scheme, url.Host))
}
return servers
@@ -6008,6 +6218,103 @@ func (nc *Conn) Barrier(f func()) error {
return nil
}
// ServerPool returns a copy of the current server pool for the connection.
//
// This function should not be called from within connection callbacks to avoid
// potential deadlocks.
func (nc *Conn) ServerPool() []Server {
nc.mu.RLock()
defer nc.mu.RUnlock()
servers := make([]Server, len(nc.srvPool))
for i, srv := range nc.srvPool {
if srv != nil {
// Return a copy to avoid exposing internal state
servers[i] = srv.clone()
}
}
return servers
}
// SetServerPool allows updating the server pool for the connection. This
// replaces the existing pool with the provided list of server URLs. If the
// current server is not in the new pool, the client will switch to a server in
// the new pool on the next reconnect attempt. This function is thread-safe and
// can be called while the connection is active. It will return an error if the
// connection is closed or if any of the provided URLs are invalid.
//
// This function does not trigger an immediate reconnect. The new server
// pool will be used on the next reconnect attempt.
// If you want to trigger an immediate reconnect to apply the new server pool,
// you can call [Conn.ForceReconnect] after this function.
//
// Unless [IgnoreDiscoveredServers] is used true, the client will continue to
// discover and add new servers to the pool as it receives INFO messages from
// the server. If you want to prevent this behavior and only use the servers
// provided in SetServerPool, use [IgnoreDiscoveredServers].
//
// This function should not be called from within connection callbacks to avoid
// potential deadlocks.
func (nc *Conn) SetServerPool(servers []string) error {
nc.mu.Lock()
defer nc.mu.Unlock()
if nc.isClosed() {
return ErrConnectionClosed
}
// Parse and validate all URLs first (without modifying state)
newPool := make([]*Server, 0, len(servers))
newURLs := make(map[string]struct{})
for _, addr := range servers {
s, err := nc.parseServerURL(addr, false, false)
if err != nil {
return err
}
if isWebsocketScheme(s.URL) != nc.ws {
return ErrMixingWebsocketSchemes
}
newPool = append(newPool, s)
newURLs[s.URL.Host] = struct{}{}
}
// Preserve state from existing pool entries
for _, newSrv := range newPool {
if idx := slices.IndexFunc(nc.srvPool, func(oldSrv *Server) bool {
return oldSrv != nil && oldSrv.URL.String() == newSrv.URL.String()
}); idx != -1 {
newSrv.Reconnects = nc.srvPool[idx].Reconnects
newSrv.didConnect = nc.srvPool[idx].didConnect
newSrv.lastErr = nc.srvPool[idx].lastErr
}
}
nc.srvPool = newPool
nc.urls = newURLs
// Update nc.current to point to the corresponding server in the new pool
// This is important because currentServer() uses pointer equality
if nc.current != nil {
currentURL := nc.current.URL.String()
found := false
for _, s := range newPool {
if s.URL.String() == currentURL {
// Update nc.current to point to the server instance in the new pool
nc.current = s
found = true
break
}
}
if !found && len(newPool) > 0 {
// Current server not in new pool - point to first server in new pool
// This ensures selectNextServer() can find it and properly rotate
nc.current = newPool[0]
}
}
return nil
}
// GetClientIP returns the client IP as known by the server.
// Supported as of server version 2.1.6.
func (nc *Conn) GetClientIP() (net.IP, error) {
@@ -6070,8 +6377,8 @@ func (nc *Conn) RemoveStatusListener(ch chan (Status)) {
}
for _, listeners := range nc.statListeners {
for l := range listeners {
delete(listeners, l)
for range listeners {
delete(listeners, ch)
}
}
}
+4 -2
View File
@@ -1,3 +1,4 @@
version: 2
project_name: nkeys
release:
github:
@@ -41,7 +42,8 @@ archives:
- name_template: '{{ .ProjectName }}-v{{ .Version }}-{{ .Os }}-{{ .Arch }}{{ if .Arm
}}v{{ .Arm }}{{ end }}'
wrap_in_directory: true
format: zip
formats:
- zip
files:
- README.md
- LICENSE
@@ -50,7 +52,7 @@ checksum:
name_template: '{{ .ProjectName }}-v{{ .Version }}-checksums.txt'
snapshot:
name_template: 'dev'
version_template: dev
nfpms:
- file_name_template: '{{ .ProjectName }}-v{{ .Version }}-{{ .Arch }}{{ if .Arm
+3
View File
@@ -38,6 +38,9 @@ type chacha20poly1305 struct {
// New returns a ChaCha20-Poly1305 AEAD that uses the given 256-bit key.
func New(key []byte) (cipher.AEAD, error) {
if fips140Enforced() {
return nil, errors.New("chacha20poly1305: use of ChaCha20Poly1305 is not allowed in FIPS 140-only mode")
}
if len(key) != KeySize {
return nil, errors.New("chacha20poly1305: bad key length")
}
+9
View File
@@ -0,0 +1,9 @@
// Copyright 2025 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !go1.26
package chacha20poly1305
func fips140Enforced() bool { return false }
+11
View File
@@ -0,0 +1,11 @@
// Copyright 2025 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build go1.26
package chacha20poly1305
import "crypto/fips140"
func fips140Enforced() bool { return fips140.Enforced() }
+3
View File
@@ -22,6 +22,9 @@ type xchacha20poly1305 struct {
// preferred when nonce uniqueness cannot be trivially ensured, or whenever
// nonces are randomly generated.
func NewX(key []byte) (cipher.AEAD, error) {
if fips140Enforced() {
return nil, errors.New("chacha20poly1305: use of ChaCha20Poly1305 is not allowed in FIPS 140-only mode")
}
if len(key) != KeySize {
return nil, errors.New("chacha20poly1305: bad key length")
}
+3
View File
@@ -196,6 +196,9 @@ func Key(password, salt []byte, N, r, p, keyLen int) ([]byte, error) {
if N <= 1 || N&(N-1) != 0 {
return nil, errors.New("scrypt: N must be > 1 and a power of 2")
}
if r <= 0 || p <= 0 {
return nil, errors.New("scrypt: parameters must be > 0")
}
if uint64(r)*uint64(p) >= 1<<30 || r > maxInt/128/p || r > maxInt/256 || N > maxInt/128/r {
return nil, errors.New("scrypt: parameters are too large")
}
+1 -1
View File
@@ -36,7 +36,7 @@ func (s *server) processRequestBytes(reqData []byte) []byte {
return []byte{agentFailure}
}
if err == nil && rep == nil {
if rep == nil {
return []byte{agentSuccess}
}
+3 -3
View File
@@ -261,7 +261,7 @@ func modPathOK(r rune) bool {
// importPathOK reports whether r can appear in a package import path element.
//
// Import paths are intermediate between module paths and file paths: we allow
// Import paths are intermediate between module paths and file paths: we
// disallow characters that would be confusing or ambiguous as arguments to
// 'go get' (such as '@' and ' ' ), but allow certain characters that are
// otherwise-unambiguous on the command line and historically used for some
@@ -802,8 +802,8 @@ func MatchPrefixPatterns(globs, target string) bool {
for globs != "" {
// Extract next non-empty glob in comma-separated list.
var glob string
if i := strings.Index(globs, ","); i >= 0 {
glob, globs = globs[:i], globs[i+1:]
if before, after, ok := strings.Cut(globs, ","); ok {
glob, globs = before, after
} else {
glob, globs = globs, ""
}
+2 -2
View File
@@ -45,8 +45,8 @@ func IsValid(v string) bool {
// Canonical returns the canonical formatting of the semantic version v.
// It fills in any missing .MINOR or .PATCH and discards build metadata.
// Two semantic versions compare equal only if their canonical formattings
// are identical strings.
// Two semantic versions compare equal only if their canonical formatting
// is an identical string.
// The canonical invalid semantic version is the empty string.
func Canonical(v string) string {
p, ok := parse(v)
-2
View File
@@ -2,8 +2,6 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build go1.23
package html
import "iter"
+1
View File
@@ -11,6 +11,7 @@ import (
// A NodeType is the type of a Node.
type NodeType uint32
//go:generate stringer -type NodeType
const (
ErrorNode NodeType = iota
TextNode
+31
View File
@@ -0,0 +1,31 @@
// Code generated by "stringer -type NodeType"; DO NOT EDIT.
package html
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[ErrorNode-0]
_ = x[TextNode-1]
_ = x[DocumentNode-2]
_ = x[ElementNode-3]
_ = x[CommentNode-4]
_ = x[DoctypeNode-5]
_ = x[RawNode-6]
_ = x[scopeMarkerNode-7]
}
const _NodeType_name = "ErrorNodeTextNodeDocumentNodeElementNodeCommentNodeDoctypeNodeRawNodescopeMarkerNode"
var _NodeType_index = [...]uint8{0, 9, 17, 29, 40, 51, 62, 69, 84}
func (i NodeType) String() string {
idx := int(i) - 0
if i < 0 || idx >= len(_NodeType_index)-1 {
return "NodeType(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _NodeType_name[_NodeType_index[idx]:_NodeType_index[idx+1]]
}
+20
View File
@@ -0,0 +1,20 @@
// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build !go1.27
package http2
import "net/http"
// Support for go.dev/issue/75500 is added in Go 1.27. In case anyone uses
// x/net with versions before Go 1.27, we return true here so that their write
// scheduler will still be the round-robin write scheduler rather than the RFC
// 9218 write scheduler. That way, older users of Go will not see a sudden
// change of behavior just from importing x/net.
//
// TODO(nsh): remove this file after x/net go.mod is at Go 1.27.
func clientPriorityDisabled(_ *http.Server) bool {
return true
}
+13
View File
@@ -0,0 +1,13 @@
// Copyright 2026 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
//go:build go1.27
package http2
import "net/http"
func clientPriorityDisabled(s *http.Server) bool {
return s.DisableClientPriority
}
+153 -35
View File
@@ -11,11 +11,13 @@ import (
"fmt"
"io"
"log"
"slices"
"strings"
"sync"
"golang.org/x/net/http/httpguts"
"golang.org/x/net/http2/hpack"
"golang.org/x/net/internal/httpsfv"
)
const frameHeaderLen = 9
@@ -23,33 +25,36 @@ const frameHeaderLen = 9
var padZeros = make([]byte, 255) // zeros for padding
// A FrameType is a registered frame type as defined in
// https://httpwg.org/specs/rfc7540.html#rfc.section.11.2
// https://httpwg.org/specs/rfc7540.html#rfc.section.11.2 and other future
// RFCs.
type FrameType uint8
const (
FrameData FrameType = 0x0
FrameHeaders FrameType = 0x1
FramePriority FrameType = 0x2
FrameRSTStream FrameType = 0x3
FrameSettings FrameType = 0x4
FramePushPromise FrameType = 0x5
FramePing FrameType = 0x6
FrameGoAway FrameType = 0x7
FrameWindowUpdate FrameType = 0x8
FrameContinuation FrameType = 0x9
FrameData FrameType = 0x0
FrameHeaders FrameType = 0x1
FramePriority FrameType = 0x2
FrameRSTStream FrameType = 0x3
FrameSettings FrameType = 0x4
FramePushPromise FrameType = 0x5
FramePing FrameType = 0x6
FrameGoAway FrameType = 0x7
FrameWindowUpdate FrameType = 0x8
FrameContinuation FrameType = 0x9
FramePriorityUpdate FrameType = 0x10
)
var frameNames = [...]string{
FrameData: "DATA",
FrameHeaders: "HEADERS",
FramePriority: "PRIORITY",
FrameRSTStream: "RST_STREAM",
FrameSettings: "SETTINGS",
FramePushPromise: "PUSH_PROMISE",
FramePing: "PING",
FrameGoAway: "GOAWAY",
FrameWindowUpdate: "WINDOW_UPDATE",
FrameContinuation: "CONTINUATION",
FrameData: "DATA",
FrameHeaders: "HEADERS",
FramePriority: "PRIORITY",
FrameRSTStream: "RST_STREAM",
FrameSettings: "SETTINGS",
FramePushPromise: "PUSH_PROMISE",
FramePing: "PING",
FrameGoAway: "GOAWAY",
FrameWindowUpdate: "WINDOW_UPDATE",
FrameContinuation: "CONTINUATION",
FramePriorityUpdate: "PRIORITY_UPDATE",
}
func (t FrameType) String() string {
@@ -125,21 +130,24 @@ var flagName = map[FrameType]map[Flags]string{
type frameParser func(fc *frameCache, fh FrameHeader, countError func(string), payload []byte) (Frame, error)
var frameParsers = [...]frameParser{
FrameData: parseDataFrame,
FrameHeaders: parseHeadersFrame,
FramePriority: parsePriorityFrame,
FrameRSTStream: parseRSTStreamFrame,
FrameSettings: parseSettingsFrame,
FramePushPromise: parsePushPromise,
FramePing: parsePingFrame,
FrameGoAway: parseGoAwayFrame,
FrameWindowUpdate: parseWindowUpdateFrame,
FrameContinuation: parseContinuationFrame,
FrameData: parseDataFrame,
FrameHeaders: parseHeadersFrame,
FramePriority: parsePriorityFrame,
FrameRSTStream: parseRSTStreamFrame,
FrameSettings: parseSettingsFrame,
FramePushPromise: parsePushPromise,
FramePing: parsePingFrame,
FrameGoAway: parseGoAwayFrame,
FrameWindowUpdate: parseWindowUpdateFrame,
FrameContinuation: parseContinuationFrame,
FramePriorityUpdate: parsePriorityUpdateFrame,
}
func typeFrameParser(t FrameType) frameParser {
if int(t) < len(frameParsers) {
return frameParsers[t]
if f := frameParsers[t]; f != nil {
return f
}
}
return parseUnknownFrame
}
@@ -1180,9 +1188,34 @@ type PriorityFrame struct {
PriorityParam
}
var defaultRFC9218Priority = PriorityParam{
incremental: 0,
urgency: 3,
// defaultRFC9218Priority determines what priority we should use as the default
// value.
//
// According to RFC 9218, by default, streams should be given an urgency of 3
// and should be non-incremental. However, making streams non-incremental by
// default would be a huge change to our historical behavior where we would
// round-robin writes across streams. When streams are non-incremental, we
// would process streams of the same urgency one-by-one to completion instead.
//
// To avoid such a sudden change which might break some HTTP/2 users, this
// function allows the caller to specify whether they can actually use the
// default value as specified in RFC 9218. If not, this function will return a
// priority value where streams are incremental by default instead: effectively
// a round-robin between stream of the same urgency.
//
// As an example, a server might not be able to use the RFC 9218 default value
// when it's not sure that the client it is serving is aware of RFC 9218.
func defaultRFC9218Priority(canUseDefault bool) PriorityParam {
if canUseDefault {
return PriorityParam{
urgency: 3,
incremental: 0,
}
}
return PriorityParam{
urgency: 3,
incremental: 1,
}
}
// Note that HTTP/2 has had two different prioritization schemes, and
@@ -1266,6 +1299,74 @@ func (f *Framer) WritePriority(streamID uint32, p PriorityParam) error {
return f.endWrite()
}
// PriorityUpdateFrame is a PRIORITY_UPDATE frame as described in
// https://www.rfc-editor.org/rfc/rfc9218.html#name-the-priority_update-frame.
type PriorityUpdateFrame struct {
FrameHeader
Priority string
PrioritizedStreamID uint32
}
func parseRFC9218Priority(s string, canUseDefault bool) (p PriorityParam, ok bool) {
p = defaultRFC9218Priority(canUseDefault)
ok = httpsfv.ParseDictionary(s, func(key, val, _ string) {
switch key {
case "u":
if u, ok := httpsfv.ParseInteger(val); ok && u >= 0 && u <= 7 {
p.urgency = uint8(u)
}
case "i":
if i, ok := httpsfv.ParseBoolean(val); ok {
if i {
p.incremental = 1
} else {
p.incremental = 0
}
}
}
})
if !ok {
return defaultRFC9218Priority(canUseDefault), ok
}
return p, true
}
func parsePriorityUpdateFrame(_ *frameCache, fh FrameHeader, countError func(string), payload []byte) (Frame, error) {
if fh.StreamID != 0 {
countError("frame_priority_update_non_zero_stream")
return nil, connError{ErrCodeProtocol, "PRIORITY_UPDATE frame with non-zero stream ID"}
}
if len(payload) < 4 {
countError("frame_priority_update_bad_length")
return nil, connError{ErrCodeFrameSize, fmt.Sprintf("PRIORITY_UPDATE frame payload size was %d; want at least 4", len(payload))}
}
v := binary.BigEndian.Uint32(payload[:4])
streamID := v & 0x7fffffff // mask off high bit
if streamID == 0 {
countError("frame_priority_update_prioritizing_zero_stream")
return nil, connError{ErrCodeProtocol, "PRIORITY_UPDATE frame with prioritized stream ID of zero"}
}
return &PriorityUpdateFrame{
FrameHeader: fh,
PrioritizedStreamID: streamID,
Priority: string(payload[4:]),
}, nil
}
// WritePriorityUpdate writes a PRIORITY_UPDATE frame.
//
// It will perform exactly one Write to the underlying Writer.
// It is the caller's responsibility to not call other Write methods concurrently.
func (f *Framer) WritePriorityUpdate(streamID uint32, priority string) error {
if !validStreamID(streamID) && !f.AllowIllegalWrites {
return errStreamID
}
f.startWrite(FramePriorityUpdate, 0, 0)
f.writeUint32(streamID)
f.writeBytes([]byte(priority))
return f.endWrite()
}
// A RSTStreamFrame allows for abnormal termination of a stream.
// See https://httpwg.org/specs/rfc7540.html#rfc.section.6.4
type RSTStreamFrame struct {
@@ -1547,6 +1648,23 @@ func (mh *MetaHeadersFrame) PseudoFields() []hpack.HeaderField {
return mh.Fields
}
func (mh *MetaHeadersFrame) rfc9218Priority(priorityAware bool) (p PriorityParam, priorityAwareAfter, hasIntermediary bool) {
var s string
for _, field := range mh.Fields {
if field.Name == "priority" {
s = field.Value
priorityAware = true
}
if slices.Contains([]string{"via", "forwarded", "x-forwarded-for"}, field.Name) {
hasIntermediary = true
}
}
// No need to check for ok. parseRFC9218Priority will return a default
// value if there is no priority field or if the field cannot be parsed.
p, _ = parseRFC9218Priority(s, priorityAware && !hasIntermediary)
return p, priorityAware, hasIntermediary
}
func (mh *MetaHeadersFrame) checkPseudos() error {
var isRequest, isResponse bool
pf := mh.PseudoFields()
+2
View File
@@ -169,6 +169,7 @@ const (
SettingMaxFrameSize SettingID = 0x5
SettingMaxHeaderListSize SettingID = 0x6
SettingEnableConnectProtocol SettingID = 0x8
SettingNoRFC7540Priorities SettingID = 0x9
)
var settingName = map[SettingID]string{
@@ -179,6 +180,7 @@ var settingName = map[SettingID]string{
SettingMaxFrameSize: "MAX_FRAME_SIZE",
SettingMaxHeaderListSize: "MAX_HEADER_LIST_SIZE",
SettingEnableConnectProtocol: "ENABLE_CONNECT_PROTOCOL",
SettingNoRFC7540Priorities: "NO_RFC7540_PRIORITIES",
}
func (s SettingID) String() string {

Some files were not shown because too many files have changed in this diff Show More