Initial QSfera import

This commit is contained in:
Курнат Андрей
2026-06-07 10:20:04 +03:00
commit 2315f25754
16485 changed files with 4826827 additions and 0 deletions
@@ -0,0 +1,15 @@
# Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, built with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
# Dependency directories (remove the comment below to include it)
# vendor/
+95
View File
@@ -0,0 +1,95 @@
version: "2"
linters:
default: all
disable:
- cyclop
- depguard
- dupl
- errorlint
- exhaustive
- forbidigo
- funcorder
- funlen
- gochecknoglobals
- gochecknoinits
- gocognit
- gocritic
- gocyclo
- godot
- godox
- gosec
- gosmopolitan
- govet
- inamedparam
- ireturn
- lll
- maintidx
- makezero
- mnd
- nakedret
- nestif
- nlreturn
- noinlineerr
- nonamedreturns
- paralleltest
- tagliatelle
- testpackage
- thelper
- varnamelen
- wrapcheck
- wsl
- wsl_v5
settings:
govet:
disable:
- shadow
- fieldalignment
enable-all: true
exclusions:
generated: lax
presets:
- comments
- common-false-positives
- legacy
- std-error-handling
rules:
- linters:
- staticcheck
path: /*.go
text: 'ST1003: should not use underscores in package names'
- linters:
- revive
path: /*.go
text: don't use an underscore in package name
- linters:
- contextcheck
- exhaustruct
path: /*.go
- linters:
- errcheck
path: /main.go
- linters:
- errcheck
- errchkjson
- forcetypeassert
path: /*_test.go
- linters:
- forbidigo
path: /*_example_test.go
paths:
- third_party$
- builtin$
- examples$
issues:
max-issues-per-linter: 0
max-same-issues: 0
formatters:
enable:
- gofmt
- goimports
exclusions:
generated: lax
paths:
- third_party$
- builtin$
- examples$
+34
View File
@@ -0,0 +1,34 @@
Changes
=======
v3.0.2 05 Dev 2025
* Code changes mainly due to upgraded linter.
* github.com/lestrrat-go/option upgraded to v2
v3.0.1 18 Aug 2025
* Refresh() no longer requires the resource to be ready.
v3.0.0 5 Jun 2025
[Breaking Changes]
* The entire API has been re-imagined for Go versions that allow typed parameters
v2.0.0 19 Feb 2024
[Breaking Changes]
* `Fetcher` type is no longer available. You probably want to provide
a customg HTTP client instead via httprc.WithHTTPClient()).
*
v1.0.4 19 Jul 2022
* Fix sloppy API breakage
v1.0.3 19 Jul 2022
* Fix queue insertion in the middle of the queue (#7)
v1.0.2 13 Jun 2022
* Properly release a lock when the fetch fails (#5)
v1.0.1 29 Mar 2022
* Bump dependency for github.com/lestrrat-go/httpcc to v1.0.1
v1.0.0 29 Mar 2022
* Initial release, refactored out of github.com/lestrrat-go/jwx
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 lestrrat
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
+172
View File
@@ -0,0 +1,172 @@
# github.com/lestrrat-go/httprc/v3 ![](https://github.com/lestrrat-go/httprc/v3/workflows/CI/badge.svg) [![Go Reference](https://pkg.go.dev/badge/github.com/lestrrat-go/httprc/v3.svg)](https://pkg.go.dev/github.com/lestrrat-go/httprc/v3)
`httprc` is a HTTP "Refresh" Cache. Its aim is to cache a remote resource that
can be fetched via HTTP, but keep the cached content up-to-date based on periodic
refreshing.
# Client
A `httprc.Client` object is comprised of 3 parts: The user-facing controller API,
the main controller loop, and set of workers that perform the actual fetching.
The user-facing controller API is the object returned when you call `(httprc.Client).Start`.
```go
ctrl, _ := client.Start(ctx)
```
# Controller API
The controller API gives you access to the controller backend that runs asynchronously.
All methods take a `context.Context` object because they potentially block. You should
be careful to use `context.WithTimeout` to properly set a timeout if you cannot tolerate
a blocking operation.
# Main Controller Loop
The main controller loop is run asynchronously to the controller API. It is single threaded,
and it has two reponsibilities.
The first is to receive commands from the controller API,
and appropriately modify the state of the goroutine, i.e. modify the list of resources
it is watching, performing forced refreshes, etc.
The other is to periodically wake up and go through the list of resources and re-fetch
ones that are past their TTL (in reality, each resource carry a "next-check" time, not
a TTL). The main controller loop itself does nothing more: it just kicks these checks periodically.
The interval between fetches is changed dynamically based on either the metadata carried
with the HTTP responses, such as `Cache-Control` and `Expires` headers, or a constant
interval set by the user for a given resource. Between these values, the main controller loop
will pick the shortest interval (but no less than 1 second) and checks if resources
need updating based on that value.
For example, if a resource A has an expiry of 10 minutes and if resource has an expiry of 5
minutes, the main controller loop will attempt to wake up roughly every 5 minutes to check
on the resources.
When the controller loop detects that a resource needs to be checked for freshness,
it will send the resource to the worker pool to be synced.
# Interval calculation
After the resource is synced, the next fetch is scheduled. The interval to the next
fetch is calculated either by using constant intervals, or by heuristics using values
from the `http.Response` object.
If the constant interval is specified, no extra calculation is performed. If you specify
a constant interval of 15 minutes, the resource will be checked every 15 minutes. This is
predictable and reliable, but not necessarily efficient.
If you do not specify a constant interval, the HTTP response is analyzed for
values in `Cache-Control` and `Expires` headers. These values will be compared against
a maximum and minimum interval values, which default to 30 days and 15 minutes, respectively.
If the values obtained from the headers fall within that range, the value from the header is
used. If the value is larger than the maximum, the maximum is used. If the value is lower
than the minimum, the minimum is used.
# SYNOPSIS
<!-- INCLUDE(client_example_test.go) -->
```go
package httprc_test
import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"time"
"github.com/lestrrat-go/httprc/v3"
)
func ExampleClient() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
type HelloWorld struct {
Hello string `json:"hello"`
}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
json.NewEncoder(w).Encode(map[string]string{"hello": "world"})
}))
options := []httprc.NewClientOption{
// By default the client will allow all URLs (which is what the option
// below is explicitly specifying). If you want to restrict what URLs
// are allowed, you can specify another whitelist.
//
// httprc.WithWhitelist(httprc.NewInsecureWhitelist()),
}
// If you would like to handle errors from asynchronous workers, you can specify a error sink.
// This is disabled in this example because the trace logs are dynamic
// and thus would interfere with the runnable example test.
// options = append(options, httprc.WithErrorSink(errsink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))))
// If you would like to see the trace logs, you can specify a trace sink.
// This is disabled in this example because the trace logs are dynamic
// and thus would interfere with the runnable example test.
// options = append(options, httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))))
// Create a new client
cl := httprc.NewClient(options...)
// Start the client, and obtain a Controller object
ctrl, err := cl.Start(ctx)
if err != nil {
fmt.Println(err.Error())
return
}
// The following is required if you want to make sure that there are no
// dangling goroutines hanging around when you exit. For example, if you
// are running tests to check for goroutine leaks, you should call this
// function before the end of your test.
defer ctrl.Shutdown(time.Second)
// Create a new resource that is synchronized every so often
//
// By default the client will attempt to fetch the resource once
// as soon as it can, and then if no other metadata is provided,
// it will fetch the resource every 15 minutes.
//
// If the resource responds with a Cache-Control/Expires header,
// the client will attempt to respect that, and will try to fetch
// the resource again based on the values obatained from the headers.
r, err := httprc.NewResource[HelloWorld](srv.URL, httprc.JSONTransformer[HelloWorld]())
if err != nil {
fmt.Println(err.Error())
return
}
// Add the resource to the controller, so that it starts fetching.
// By default, a call to `Add()` will block until the first fetch
// succeeds, via an implicit call to `r.Ready()`
// You can change this behavior if you specify the `WithWaitReady(false)`
// option.
ctrl.Add(ctx, r)
// if you specified `httprc.WithWaitReady(false)` option, the fetch will happen
// "soon", but you're not guaranteed that it will happen before the next
// call to `Lookup()`. If you want to make sure that the resource is ready,
// you can call `Ready()` like so:
/*
{
tctx, tcancel := context.WithTimeout(ctx, time.Second)
defer tcancel()
if err := r.Ready(tctx); err != nil {
fmt.Println(err.Error())
return
}
}
*/
m := r.Resource()
fmt.Println(m.Hello)
// OUTPUT:
// world
}
```
source: [client_example_test.go](https://github.com/lestrrat-go/httprc/blob/refs/heads/v3/client_example_test.go)
<!-- END INCLUDE -->
+235
View File
@@ -0,0 +1,235 @@
package httprc
import (
"context"
"fmt"
"sync"
"time"
)
func (c *ctrlBackend) adjustInterval(ctx context.Context, req adjustIntervalRequest) {
interval := roundupToSeconds(time.Until(req.resource.Next()))
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got adjust request (current tick interval=%s, next for %q=%s)", c.tickInterval, req.resource.URL(), interval))
if interval < time.Second {
interval = time.Second
}
if c.tickInterval < interval {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: no adjusting required (time to next check %s > current tick interval %s)", interval, c.tickInterval))
} else {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: adjusting tick interval to %s", interval))
c.tickInterval = interval
c.check.Reset(interval)
}
}
func (c *ctrlBackend) addResource(ctx context.Context, req addRequest) {
r := req.resource
if _, ok := c.items[r.URL()]; ok {
// Already exists
sendReply(ctx, req.reply, struct{}{}, errResourceAlreadyExists)
return
}
c.items[r.URL()] = r
if r.MaxInterval() == 0 {
r.SetMaxInterval(c.defaultMaxInterval)
}
if r.MinInterval() == 0 {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: set minimum interval to %s", c.defaultMinInterval))
r.SetMinInterval(c.defaultMinInterval)
}
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: added resource %q", r.URL()))
sendReply(ctx, req.reply, struct{}{}, nil)
c.SetTickInterval(time.Nanosecond)
}
func (c *ctrlBackend) rmResource(ctx context.Context, req rmRequest) {
u := req.u
if _, ok := c.items[u]; !ok {
sendReply(ctx, req.reply, struct{}{}, errResourceNotFound)
return
}
delete(c.items, u)
minInterval := oneDay
for _, item := range c.items {
if d := item.MinInterval(); d < minInterval {
minInterval = d
}
}
close(req.reply)
c.check.Reset(minInterval)
}
func (c *ctrlBackend) refreshResource(ctx context.Context, req refreshRequest) {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] START %q", req.u))
defer c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] END %q", req.u))
u := req.u
r, ok := c.items[u]
if !ok {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] %s is not registered", req.u))
sendReply(ctx, req.reply, struct{}{}, errResourceNotFound)
return
}
// Note: We don't wait for r.Ready() here because refresh should work
// regardless of whether the resource has been fetched before. This allows
// refresh to work with resources registered using WithWaitReady(false).
r.SetNext(time.Unix(0, 0))
sendWorkerSynchronous(ctx, c.syncoutgoing, synchronousRequest{
resource: r,
reply: req.reply,
})
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] sync request for %s sent to worker pool", req.u))
}
func (c *ctrlBackend) lookupResource(ctx context.Context, req lookupRequest) {
u := req.u
r, ok := c.items[u]
if !ok {
sendReply(ctx, req.reply, nil, errResourceNotFound)
return
}
sendReply(ctx, req.reply, r, nil)
}
func (c *ctrlBackend) handleRequest(ctx context.Context, req any) {
switch req := req.(type) {
case adjustIntervalRequest:
c.adjustInterval(ctx, req)
case addRequest:
c.addResource(ctx, req)
case rmRequest:
c.rmResource(ctx, req)
case refreshRequest:
c.refreshResource(ctx, req)
case lookupRequest:
c.lookupResource(ctx, req)
default:
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: unknown request type %T", req))
}
}
func sendWorker(ctx context.Context, ch chan Resource, r Resource) {
r.SetBusy(true)
select {
case <-ctx.Done():
case ch <- r:
}
}
func sendWorkerSynchronous(ctx context.Context, ch chan synchronousRequest, r synchronousRequest) {
r.resource.SetBusy(true)
select {
case <-ctx.Done():
case ch <- r:
}
}
func sendReply[T any](ctx context.Context, ch chan backendResponse[T], v T, err error) {
defer close(ch)
select {
case <-ctx.Done():
case ch <- backendResponse[T]{payload: v, err: err}:
}
}
type ctrlBackend struct {
items map[string]Resource
outgoing chan Resource
syncoutgoing chan synchronousRequest
incoming chan any // incoming requests to the controller
traceSink TraceSink
tickInterval time.Duration
check *time.Ticker
defaultMaxInterval time.Duration
defaultMinInterval time.Duration
}
func (c *ctrlBackend) loop(ctx context.Context, readywg, donewg *sync.WaitGroup) {
c.traceSink.Put(ctx, "httprc controller: starting main controller loop")
readywg.Done()
defer c.traceSink.Put(ctx, "httprc controller: stopping main controller loop")
defer donewg.Done()
for {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: waiting for request or tick (tick interval=%s)", c.tickInterval))
select {
case req := <-c.incoming:
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got request %T", req))
c.handleRequest(ctx, req)
case t := <-c.check.C:
c.periodicCheck(ctx, t)
case <-ctx.Done():
return
}
}
}
func (c *ctrlBackend) periodicCheck(ctx context.Context, t time.Time) {
c.traceSink.Put(ctx, "httprc controller: START periodic check")
defer c.traceSink.Put(ctx, "httprc controller: END periodic check")
var minNext time.Time
var dispatched int
minInterval := -1 * time.Second
for _, item := range c.items {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: checking resource %q", item.URL()))
next := item.Next()
if minNext.IsZero() || next.Before(minNext) {
minNext = next
}
if interval := item.MinInterval(); minInterval < 0 || interval < minInterval {
minInterval = interval
}
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q isBusy=%t, next(%s).After(%s)=%t", item.URL(), item.IsBusy(), next, t, next.After(t)))
if item.IsBusy() || next.After(t) {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q is busy or not ready yet, skipping", item.URL()))
continue
}
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q is ready, dispatching to worker pool", item.URL()))
dispatched++
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: dispatching resource %q to worker pool", item.URL()))
sendWorker(ctx, c.outgoing, item)
}
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: dispatched %d resources", dispatched))
// Next check is always at the earliest next check + 1 second.
// The extra second makes sure that we are _past_ the actual next check time
// so we can send the resource to the worker pool
if interval := time.Until(minNext); interval > 0 {
c.SetTickInterval(roundupToSeconds(interval) + time.Second)
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resetting check intervanl to %s", c.tickInterval))
} else {
// if we got here, either we have no resources, or all resources are busy.
// In this state, it's possible that the interval is less than 1 second,
// because we previously set it to a small value for an immediate refresh.
// in this case, we want to reset it to a sane value
if c.tickInterval < time.Second {
c.SetTickInterval(minInterval)
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resetting check intervanl to %s after forced refresh", c.tickInterval))
}
}
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: next check in %s", c.tickInterval))
}
func (c *ctrlBackend) SetTickInterval(d time.Duration) {
// TODO synchronize
if d <= 0 {
d = time.Second // ensure positive interval
}
c.tickInterval = d
c.check.Reset(d)
}
+185
View File
@@ -0,0 +1,185 @@
package httprc
import (
"context"
"net/http"
"sync"
"time"
"github.com/lestrrat-go/httprc/v3/errsink"
"github.com/lestrrat-go/httprc/v3/proxysink"
"github.com/lestrrat-go/httprc/v3/tracesink"
)
// setupSink creates and starts a proxy for the given sink if it's not a Nop sink
// Returns the sink to use and a cancel function that should be chained with the original cancel
func setupSink[T any, S proxysink.Backend[T], NopType any](ctx context.Context, sink S, wg *sync.WaitGroup) (S, context.CancelFunc) {
if _, ok := any(sink).(NopType); ok {
return sink, func() {}
}
proxy := proxysink.New[T](sink)
wg.Add(1)
go func(ctx context.Context, wg *sync.WaitGroup, proxy *proxysink.Proxy[T]) {
defer wg.Done()
proxy.Run(ctx)
}(ctx, wg, proxy)
// proxy can be converted to one of the sink subtypes
s, ok := any(proxy).(S)
if !ok {
panic("type assertion failed: proxy cannot be converted to type S")
}
return s, proxy.Close
}
// Client is the main entry point for the httprc package.
type Client struct {
mu sync.Mutex
httpcl HTTPClient
numWorkers int
running bool
errSink ErrorSink
traceSink TraceSink
wl Whitelist
defaultMaxInterval time.Duration
defaultMinInterval time.Duration
}
// NewClient creates a new `httprc.Client` object.
//
// By default ALL urls are allowed. This may not be suitable for you if
// are using this in a production environment. You are encouraged to specify
// a whitelist using the `WithWhitelist` option.
//
// NOTE: In future versions, this function signature should be changed to
// return an error to properly handle option parsing failures.
func NewClient(options ...NewClientOption) *Client {
//nolint:staticcheck
var errSink ErrorSink = errsink.NewNop()
//nolint:staticcheck
var traceSink TraceSink = tracesink.NewNop()
var wl Whitelist = InsecureWhitelist{}
var httpcl HTTPClient = http.DefaultClient
defaultMinInterval := DefaultMinInterval
defaultMaxInterval := DefaultMaxInterval
numWorkers := DefaultWorkers
for _, option := range options {
switch option.Ident() {
case identHTTPClient{}:
_ = option.Value(&httpcl)
case identWorkers{}:
_ = option.Value(&numWorkers)
case identErrorSink{}:
_ = option.Value(&errSink)
case identTraceSink{}:
_ = option.Value(&traceSink)
case identWhitelist{}:
_ = option.Value(&wl)
}
}
if numWorkers <= 0 {
numWorkers = 1
}
return &Client{
httpcl: httpcl,
numWorkers: numWorkers,
errSink: errSink,
traceSink: traceSink,
wl: wl,
defaultMinInterval: defaultMinInterval,
defaultMaxInterval: defaultMaxInterval,
}
}
// Start sets the client into motion. It will start a number of worker goroutines,
// and return a Controller object that you can use to control the execution of
// the client.
//
// If you attempt to call Start more than once, it will return an error.
func (c *Client) Start(octx context.Context) (Controller, error) {
c.mu.Lock()
if c.running {
c.mu.Unlock()
return nil, errAlreadyRunning
}
c.running = true
c.mu.Unlock()
// DON'T CANCEL THIS IN THIS METHOD! It's the responsibility of the
// controller to cancel this context.
ctx, cancel := context.WithCancel(octx)
var donewg sync.WaitGroup
// start proxy goroutines that will accept sink requests
// and forward them to the appropriate sink
errSink, errCancel := setupSink[error, ErrorSink, errsink.Nop](ctx, c.errSink, &donewg)
traceSink, traceCancel := setupSink[string, TraceSink, tracesink.Nop](ctx, c.traceSink, &donewg)
// Chain the cancel functions
ocancel := cancel
cancel = func() {
ocancel()
errCancel()
traceCancel()
}
chbuf := c.numWorkers + 1
incoming := make(chan any, chbuf)
outgoing := make(chan Resource, chbuf)
syncoutgoing := make(chan synchronousRequest, chbuf)
var readywg sync.WaitGroup
readywg.Add(c.numWorkers)
donewg.Add(c.numWorkers)
for range c.numWorkers {
wrk := worker{
incoming: incoming,
next: outgoing,
nextsync: syncoutgoing,
errSink: errSink,
traceSink: traceSink,
httpcl: c.httpcl,
}
go wrk.Run(ctx, &readywg, &donewg)
}
tickInterval := oneDay
ctrl := &controller{
cancel: cancel,
incoming: incoming,
shutdown: make(chan struct{}),
traceSink: traceSink,
wl: c.wl,
}
backend := &ctrlBackend{
items: make(map[string]Resource),
outgoing: outgoing,
syncoutgoing: syncoutgoing,
incoming: incoming,
traceSink: traceSink,
tickInterval: tickInterval,
check: time.NewTicker(tickInterval),
defaultMinInterval: c.defaultMinInterval,
defaultMaxInterval: c.defaultMaxInterval,
}
donewg.Add(1)
readywg.Add(1)
go backend.loop(ctx, &readywg, &donewg)
go func(wg *sync.WaitGroup, ch chan struct{}) {
wg.Wait()
close(ch)
}(&donewg, ctrl.shutdown)
readywg.Wait()
return ctrl, nil
}
+187
View File
@@ -0,0 +1,187 @@
package httprc
import (
"context"
"fmt"
"time"
)
type Controller interface {
// Add adds a new `http.Resource` to the controller. If the resource already exists,
// it will return an error.
Add(context.Context, Resource, ...AddOption) error
// Lookup a `httprc.Resource` by its URL. If the resource does not exist, it
// will return an error.
Lookup(context.Context, string) (Resource, error)
// Remove a `httprc.Resource` from the controller by its URL. If the resource does
// not exist, it will return an error.
Remove(context.Context, string) error
// Refresh forces a resource to be refreshed immediately. If the resource does
// not exist, or if the refresh fails, it will return an error.
Refresh(context.Context, string) error
ShutdownContext(context.Context) error
Shutdown(time.Duration) error
}
type controller struct {
cancel context.CancelFunc
incoming chan any // incoming requests to the controller
shutdown chan struct{}
traceSink TraceSink
wl Whitelist
}
// Shutdown is a convenience function that calls ShutdownContext with a
// context that has a timeout of `timeout`.
func (c *controller) Shutdown(timeout time.Duration) error {
ctx, cancel := context.WithTimeout(context.Background(), timeout)
defer cancel()
return c.ShutdownContext(ctx)
}
// ShutdownContext stops the client and all associated goroutines, and waits for them
// to finish. If the context is canceled, the function will return immediately:
// there fore you should not use the context you used to start the client (because
// presumably it's already canceled).
//
// Waiting for the client shutdown will also ensure that all sinks are properly
// flushed.
func (c *controller) ShutdownContext(ctx context.Context) error {
c.cancel()
select {
case <-ctx.Done():
return ctx.Err()
case <-c.shutdown:
return nil
}
}
type ctrlRequest[T any] struct {
reply chan T
resource Resource
u string
}
type addRequest ctrlRequest[backendResponse[struct{}]]
type rmRequest ctrlRequest[backendResponse[struct{}]]
type refreshRequest ctrlRequest[backendResponse[struct{}]]
type lookupRequest ctrlRequest[backendResponse[Resource]]
type synchronousRequest ctrlRequest[backendResponse[struct{}]]
type adjustIntervalRequest struct {
resource Resource
}
type backendResponse[T any] struct {
payload T
err error
}
func sendBackend[TReq any, TB any](ctx context.Context, backendCh chan any, v TReq, replyCh chan backendResponse[TB]) (TB, error) {
select {
case <-ctx.Done():
case backendCh <- v:
}
select {
case <-ctx.Done():
var zero TB
return zero, ctx.Err()
case res := <-replyCh:
return res.payload, res.err
}
}
// Lookup returns a resource by its URL. If the resource does not exist, it
// will return an error.
//
// Unfortunately, due to the way typed parameters are handled in Go, we can only
// return a Resource object (and not a ResourceBase[T] object). This means that
// you will either need to use the `Resource.Get()` method or use a type
// assertion to obtain a `ResourceBase[T]` to get to the actual object you are
// looking for
func (c *controller) Lookup(ctx context.Context, u string) (Resource, error) {
reply := make(chan backendResponse[Resource], 1)
req := lookupRequest{
reply: reply,
u: u,
}
return sendBackend[lookupRequest, Resource](ctx, c.incoming, req, reply)
}
// Add adds a new resource to the controller. If the resource already
// exists, it will return an error.
//
// By default this function will automatically wait for the resource to be
// fetched once (by calling `r.Ready()`). Note that the `r.Ready()` call will NOT
// timeout unless you configure your context object with `context.WithTimeout`.
// To disable waiting, you can specify the `WithWaitReady(false)` option.
func (c *controller) Add(ctx context.Context, r Resource, options ...AddOption) error {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: START Add(%q)", r.URL()))
defer c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: END Add(%q)", r.URL()))
waitReady := true
for _, option := range options {
switch option.Ident() {
case identWaitReady{}:
if err := option.Value(&waitReady); err != nil {
return fmt.Errorf(`httprc.Controller.Add: failed to parse WaitReady option: %w`, err)
}
}
}
if !c.wl.IsAllowed(r.URL()) {
return fmt.Errorf(`httprc.Controller.AddResource: cannot add %q: %w`, r.URL(), errBlockedByWhitelist)
}
reply := make(chan backendResponse[struct{}], 1)
req := addRequest{
reply: reply,
resource: r,
}
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: sending add request for %q to backend", r.URL()))
if _, err := sendBackend[addRequest, struct{}](ctx, c.incoming, req, reply); err != nil {
return err
}
if waitReady {
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: waiting for resource %q to be ready", r.URL()))
if err := r.Ready(ctx); err != nil {
return err
}
}
return nil
}
// Remove removes a resource from the controller. If the resource does
// not exist, it will return an error.
func (c *controller) Remove(ctx context.Context, u string) error {
reply := make(chan backendResponse[struct{}], 1)
req := rmRequest{
reply: reply,
u: u,
}
if _, err := sendBackend[rmRequest, struct{}](ctx, c.incoming, req, reply); err != nil {
return err
}
return nil
}
// Refresh forces a resource to be refreshed immediately. If the resource does
// not exist, or if the refresh fails, it will return an error.
//
// This function is synchronous, and will block until the resource has been refreshed.
func (c *controller) Refresh(ctx context.Context, u string) error {
reply := make(chan backendResponse[struct{}], 1)
req := refreshRequest{
reply: reply,
u: u,
}
if _, err := sendBackend[refreshRequest, struct{}](ctx, c.incoming, req, reply); err != nil {
return err
}
return nil
}
+57
View File
@@ -0,0 +1,57 @@
package httprc
import "errors"
var errResourceAlreadyExists = errors.New(`resource already exists`)
func ErrResourceAlreadyExists() error {
return errResourceAlreadyExists
}
var errAlreadyRunning = errors.New(`client is already running`)
func ErrAlreadyRunning() error {
return errAlreadyRunning
}
var errResourceNotFound = errors.New(`resource not found`)
func ErrResourceNotFound() error {
return errResourceNotFound
}
var errTransformerRequired = errors.New(`transformer is required`)
func ErrTransformerRequired() error {
return errTransformerRequired
}
var errURLCannotBeEmpty = errors.New(`URL cannot be empty`)
func ErrURLCannotBeEmpty() error {
return errURLCannotBeEmpty
}
var errUnexpectedStatusCode = errors.New(`unexpected status code`)
func ErrUnexpectedStatusCode() error {
return errUnexpectedStatusCode
}
var errTransformerFailed = errors.New(`failed to transform response body`)
func ErrTransformerFailed() error {
return errTransformerFailed
}
var errRecoveredFromPanic = errors.New(`recovered from panic`)
func ErrRecoveredFromPanic() error {
return errRecoveredFromPanic
}
var errBlockedByWhitelist = errors.New(`blocked by whitelist`)
func ErrBlockedByWhitelist() error {
return errBlockedByWhitelist
}
+59
View File
@@ -0,0 +1,59 @@
package errsink
import (
"context"
"log/slog"
)
type Interface interface {
Put(context.Context, error)
}
// Nop is an ErrorSink that does nothing. It does not require
// any initialization, so the zero value can be used.
type Nop struct{}
// NewNop returns a new NopErrorSink object. The constructor
// is provided for consistency.
func NewNop() Interface {
return Nop{}
}
// Put for NopErrorSink does nothing.
func (Nop) Put(context.Context, error) {}
type SlogLogger interface {
Log(context.Context, slog.Level, string, ...any)
}
type slogSink struct {
logger SlogLogger
}
// NewSlog returns a new ErrorSink that logs errors using the provided slog.Logger
func NewSlog(l SlogLogger) Interface {
return &slogSink{
logger: l,
}
}
func (s *slogSink) Put(ctx context.Context, v error) {
s.logger.Log(ctx, slog.LevelError, v.Error())
}
// FuncSink is an ErrorSink that calls a function with the error.
type FuncSink struct {
fn func(context.Context, error)
}
// NewFunc returns a new FuncSink that calls the provided function with errors.
func NewFunc(fn func(context.Context, error)) Interface {
return &FuncSink{fn: fn}
}
// Put calls the function with the error.
func (f *FuncSink) Put(ctx context.Context, err error) {
if f.fn != nil {
f.fn(ctx, err)
}
}
+90
View File
@@ -0,0 +1,90 @@
package httprc
import (
"context"
"net/http"
"time"
"github.com/lestrrat-go/httprc/v3/errsink"
"github.com/lestrrat-go/httprc/v3/tracesink"
)
// Buffer size constants
const (
// ReadBufferSize is the default buffer size for reading HTTP responses (10MB)
ReadBufferSize = 1024 * 1024 * 10
// MaxBufferSize is the maximum allowed buffer size (1GB)
MaxBufferSize = 1024 * 1024 * 1000
)
// Client worker constants
const (
// DefaultWorkers is the default number of worker goroutines
DefaultWorkers = 5
)
// Interval constants
const (
// DefaultMaxInterval is the default maximum interval between fetches (30 days)
DefaultMaxInterval = 24 * time.Hour * 30
// DefaultMinInterval is the default minimum interval between fetches (15 minutes)
DefaultMinInterval = 15 * time.Minute
// oneDay is used internally for time calculations
oneDay = 24 * time.Hour
)
// utility to round up intervals to the nearest second
func roundupToSeconds(d time.Duration) time.Duration {
if diff := d % time.Second; diff > 0 {
return d + time.Second - diff
}
return d
}
// ErrorSink is an interface that abstracts a sink for errors.
type ErrorSink = errsink.Interface
type TraceSink = tracesink.Interface
// HTTPClient is an interface that abstracts a "net/http".Client, so that
// users can provide their own implementation of the HTTP client, if need be.
type HTTPClient interface {
Do(*http.Request) (*http.Response, error)
}
// Transformer is used to convert the body of an HTTP response into an appropriate
// object of type T.
type Transformer[T any] interface {
Transform(context.Context, *http.Response) (T, error)
}
// TransformFunc is a function type that implements the Transformer interface.
type TransformFunc[T any] func(context.Context, *http.Response) (T, error)
func (f TransformFunc[T]) Transform(ctx context.Context, res *http.Response) (T, error) {
return f(ctx, res)
}
// Resource is a single resource that can be retrieved via HTTP, and (possibly) transformed
// into an arbitrary object type.
//
// Realistically, there is no need for third-parties to implement this interface. This exists
// to provide a way to aggregate `httprc.ResourceBase` objects with different specialized types
// into a single collection.
//
// See ResourceBase for details
type Resource interface { //nolint:interfacebloat
Get(any) error
Next() time.Time
SetNext(time.Time)
URL() string
Sync(context.Context) error
ConstantInterval() time.Duration
MaxInterval() time.Duration
SetMaxInterval(time.Duration)
MinInterval() time.Duration
SetMinInterval(time.Duration)
IsBusy() bool
SetBusy(bool)
Ready(context.Context) error
}
+144
View File
@@ -0,0 +1,144 @@
package httprc
import (
"time"
"github.com/lestrrat-go/option/v2"
)
type NewClientOption interface {
option.Interface
newClientOption()
}
type newClientOption struct {
option.Interface
}
func (newClientOption) newClientOption() {}
type identWorkers struct{}
// WithWorkers specifies the number of concurrent workers to use for the client.
// If n is less than or equal to 0, the client will use a single worker.
func WithWorkers(n int) NewClientOption {
return newClientOption{option.New(identWorkers{}, n)}
}
type identErrorSink struct{}
// WithErrorSink specifies the error sink to use for the client.
// If not specified, the client will use a NopErrorSink.
func WithErrorSink(sink ErrorSink) NewClientOption {
return newClientOption{option.New(identErrorSink{}, sink)}
}
type identTraceSink struct{}
// WithTraceSink specifies the trace sink to use for the client.
// If not specified, the client will use a NopTraceSink.
func WithTraceSink(sink TraceSink) NewClientOption {
return newClientOption{option.New(identTraceSink{}, sink)}
}
type identWhitelist struct{}
// WithWhitelist specifies the whitelist to use for the client.
// If not specified, the client will use a BlockAllWhitelist.
func WithWhitelist(wl Whitelist) NewClientOption {
return newClientOption{option.New(identWhitelist{}, wl)}
}
type NewResourceOption interface {
option.Interface
newResourceOption()
}
type newResourceOption struct {
option.Interface
}
func (newResourceOption) newResourceOption() {}
type NewClientResourceOption interface {
option.Interface
newResourceOption()
newClientOption()
}
type newClientResourceOption struct {
option.Interface
}
func (newClientResourceOption) newResourceOption() {}
func (newClientResourceOption) newClientOption() {}
type identHTTPClient struct{}
// WithHTTPClient specifies the HTTP client to use for the client.
// If not specified, the client will use http.DefaultClient.
//
// This option can be passed to NewClient or NewResource.
func WithHTTPClient(cl HTTPClient) NewClientResourceOption {
return newClientResourceOption{option.New(identHTTPClient{}, cl)}
}
type identMinimumInterval struct{}
// WithMinInterval specifies the minimum interval between fetches.
//
// This option affects the dynamic calculation of the interval between fetches.
// If the value calculated from the http.Response is less than the this value,
// the client will use this value instead.
func WithMinInterval(d time.Duration) NewResourceOption {
return newResourceOption{option.New(identMinimumInterval{}, d)}
}
type identMaximumInterval struct{}
// WithMaxInterval specifies the maximum interval between fetches.
//
// This option affects the dynamic calculation of the interval between fetches.
// If the value calculated from the http.Response is greater than the this value,
// the client will use this value instead.
func WithMaxInterval(d time.Duration) NewResourceOption {
return newResourceOption{option.New(identMaximumInterval{}, d)}
}
type identConstantInterval struct{}
// WithConstantInterval specifies the interval between fetches. When you
// specify this option, the client will fetch the resource at the specified
// intervals, regardless of the response's Cache-Control or Expires headers.
//
// By default this option is disabled.
func WithConstantInterval(d time.Duration) NewResourceOption {
return newResourceOption{option.New(identConstantInterval{}, d)}
}
type AddOption interface {
option.Interface
newAddOption()
}
type addOption struct {
option.Interface
}
func (addOption) newAddOption() {}
type identWaitReady struct{}
// WithWaitReady specifies whether the client should wait for the resource to be
// ready before returning from the Add method.
//
// By default, the client will wait for the resource to be ready before returning.
// If you specify this option with a value of false, the client will not wait for
// the resource to be fully registered, which is usually not what you want.
// This option exists to accommodate for cases where you for some reason want to
// add a resource to the controller, but want to do something else before
// you wait for it. Make sure to call `r.Ready()` later on to ensure that
// the resource is ready before you try to access it.
func WithWaitReady(b bool) AddOption {
return addOption{option.New(identWaitReady{}, b)}
}
+135
View File
@@ -0,0 +1,135 @@
package proxysink
import (
"context"
"sync"
)
type Backend[T any] interface {
Put(context.Context, T)
}
// Proxy is used to send values through a channel. This is used to
// serialize calls to underlying sinks.
type Proxy[T any] struct {
mu *sync.Mutex
cancel context.CancelFunc
ch chan T
cond *sync.Cond
pending []T
backend Backend[T]
closed bool
}
func New[T any](b Backend[T]) *Proxy[T] {
mu := &sync.Mutex{}
return &Proxy[T]{
ch: make(chan T, 1),
mu: mu,
cond: sync.NewCond(mu),
backend: b,
cancel: func() {},
}
}
func (p *Proxy[T]) Run(ctx context.Context) {
defer p.cond.Broadcast()
p.mu.Lock()
ctx, cancel := context.WithCancel(ctx)
p.cancel = cancel
p.mu.Unlock()
go p.controlloop(ctx)
go p.flushloop(ctx)
<-ctx.Done()
}
func (p *Proxy[T]) controlloop(ctx context.Context) {
defer p.cond.Broadcast()
for {
select {
case <-ctx.Done():
return
case r := <-p.ch:
p.mu.Lock()
p.pending = append(p.pending, r)
p.mu.Unlock()
}
p.cond.Broadcast()
}
}
func (p *Proxy[T]) flushloop(ctx context.Context) {
const defaultPendingSize = 10
pending := make([]T, defaultPendingSize)
for {
select {
case <-ctx.Done():
p.mu.Lock()
if len(p.pending) <= 0 {
p.mu.Unlock()
return
}
default:
}
p.mu.Lock()
for len(p.pending) <= 0 {
select {
case <-ctx.Done():
p.mu.Unlock()
return
default:
p.cond.Wait()
}
}
// extract all pending values, and clear the shared slice
if cap(pending) < len(p.pending) {
pending = make([]T, len(p.pending))
} else {
pending = pending[:len(p.pending)]
}
copy(pending, p.pending)
if cap(p.pending) > defaultPendingSize {
p.pending = make([]T, 0, defaultPendingSize)
} else {
p.pending = p.pending[:0]
}
p.mu.Unlock()
for _, v := range pending {
// send to sink serially
p.backend.Put(ctx, v)
}
}
}
func (p *Proxy[T]) Put(ctx context.Context, v T) {
p.mu.Lock()
if p.closed {
p.mu.Unlock()
return
}
p.mu.Unlock()
select {
case <-ctx.Done():
return
case p.ch <- v:
return
}
}
func (p *Proxy[T]) Close() {
p.mu.Lock()
defer p.mu.Unlock()
if !p.closed {
p.closed = true
}
p.cancel()
p.cond.Broadcast()
}
+366
View File
@@ -0,0 +1,366 @@
package httprc
import (
"context"
"fmt"
"io"
"net/http"
"net/url"
"sync"
"sync/atomic"
"time"
"github.com/lestrrat-go/blackmagic"
"github.com/lestrrat-go/httpcc"
"github.com/lestrrat-go/httprc/v3/tracesink"
)
// ResourceBase is a generic Resource type
type ResourceBase[T any] struct {
u string
ready chan struct{} // closed when the resource is ready (i.e. after first successful fetch)
once sync.Once
httpcl HTTPClient
t Transformer[T]
r atomic.Value
next atomic.Value
interval time.Duration
minInterval atomic.Int64
maxInterval atomic.Int64
busy atomic.Bool
}
// NewResource creates a new Resource object which after fetching the
// resource from the URL, will transform the response body using the
// provided Transformer to an object of type T.
//
// This function will return an error if the URL is not a valid URL
// (i.e. it cannot be parsed by url.Parse), or if the transformer is nil.
func NewResource[T any](s string, transformer Transformer[T], options ...NewResourceOption) (*ResourceBase[T], error) {
var httpcl HTTPClient
var interval time.Duration
minInterval := DefaultMinInterval
maxInterval := DefaultMaxInterval
for _, option := range options {
switch option.Ident() {
case identHTTPClient{}:
if err := option.Value(&httpcl); err != nil {
return nil, fmt.Errorf(`httprc.NewResource: failed to parse HTTPClient option: %w`, err)
}
case identMinimumInterval{}:
if err := option.Value(&minInterval); err != nil {
return nil, fmt.Errorf(`httprc.NewResource: failed to parse MinimumInterval option: %w`, err)
}
case identMaximumInterval{}:
if err := option.Value(&maxInterval); err != nil {
return nil, fmt.Errorf(`httprc.NewResource: failed to parse MaximumInterval option: %w`, err)
}
case identConstantInterval{}:
if err := option.Value(&interval); err != nil {
return nil, fmt.Errorf(`httprc.NewResource: failed to parse ConstantInterval option: %w`, err)
}
}
}
if transformer == nil {
return nil, fmt.Errorf(`httprc.NewResource: %w`, errTransformerRequired)
}
if s == "" {
return nil, fmt.Errorf(`httprc.NewResource: %w`, errURLCannotBeEmpty)
}
if _, err := url.Parse(s); err != nil {
return nil, fmt.Errorf(`httprc.NewResource: %w`, err)
}
r := &ResourceBase[T]{
u: s,
httpcl: httpcl,
t: transformer,
interval: interval,
ready: make(chan struct{}),
}
if httpcl != nil {
r.httpcl = httpcl
}
r.minInterval.Store(int64(minInterval))
r.maxInterval.Store(int64(maxInterval))
r.SetNext(time.Unix(0, 0)) // initially, it should be fetched immediately
return r, nil
}
// URL returns the URL of the resource.
func (r *ResourceBase[T]) URL() string {
return r.u
}
// Ready returns an empty error when the resource is ready. If the context
// is canceled before the resource is ready, it will return the error from
// the context.
func (r *ResourceBase[T]) Ready(ctx context.Context) error {
select {
case <-ctx.Done():
return ctx.Err()
case <-r.ready:
return nil
}
}
// Get assigns the value of the resource to the provided pointer.
// If using the `httprc.ResourceBase[T]` type directly, you can use the `Resource()`
// method to get the resource directly.
//
// This method exists because parametric types cannot be assigned to a single object type
// that return different return values of the specialized type. i.e. for resources
// `ResourceBase[A]` and `ResourceBase[B]`, we cannot have a single interface that can
// be assigned to the same interface type `X` that expects a `Resource()` method that
// returns `A` or `B` depending on the type of the resource. When accessing the
// resource through the `httprc.Resource` interface, use this method to obtain the
// stored value.
func (r *ResourceBase[T]) Get(dst any) error {
return blackmagic.AssignIfCompatible(dst, r.Resource())
}
// Resource returns the last fetched resource. If the resource has not been
// fetched yet, this will return the zero value of type T.
//
// If you would rather wait until the resource is fetched, you can use the
// `Ready()` method to wait until the resource is ready (i.e. fetched at least once).
func (r *ResourceBase[T]) Resource() T {
v := r.r.Load()
switch v := v.(type) {
case T:
return v
default:
var zero T
return zero
}
}
func (r *ResourceBase[T]) Next() time.Time {
//nolint:forcetypeassert
return r.next.Load().(time.Time)
}
func (r *ResourceBase[T]) SetNext(v time.Time) {
r.next.Store(v)
}
func (r *ResourceBase[T]) ConstantInterval() time.Duration {
return r.interval
}
func (r *ResourceBase[T]) MaxInterval() time.Duration {
return time.Duration(r.maxInterval.Load())
}
func (r *ResourceBase[T]) MinInterval() time.Duration {
return time.Duration(r.minInterval.Load())
}
func (r *ResourceBase[T]) SetMaxInterval(v time.Duration) {
r.maxInterval.Store(int64(v))
}
func (r *ResourceBase[T]) SetMinInterval(v time.Duration) {
r.minInterval.Store(int64(v))
}
func (r *ResourceBase[T]) SetBusy(v bool) {
r.busy.Store(v)
}
func (r *ResourceBase[T]) IsBusy() bool {
return r.busy.Load()
}
// limitedBody is a wrapper around an io.Reader that will only read up to
// MaxBufferSize bytes. This is provided to prevent the user from accidentally
// reading a huge response body into memory
type limitedBody struct {
rdr io.Reader
close func() error
}
func (l *limitedBody) Read(p []byte) (n int, err error) {
return l.rdr.Read(p)
}
func (l *limitedBody) Close() error {
return l.close()
}
type traceSinkKey struct{}
func withTraceSink(ctx context.Context, sink TraceSink) context.Context {
return context.WithValue(ctx, traceSinkKey{}, sink)
}
func traceSinkFromContext(ctx context.Context) TraceSink {
if v := ctx.Value(traceSinkKey{}); v != nil {
//nolint:forcetypeassert
return v.(TraceSink)
}
return tracesink.Nop{}
}
type httpClientKey struct{}
func withHTTPClient(ctx context.Context, cl HTTPClient) context.Context {
return context.WithValue(ctx, httpClientKey{}, cl)
}
func httpClientFromContext(ctx context.Context) HTTPClient {
if v := ctx.Value(httpClientKey{}); v != nil {
//nolint:forcetypeassert
return v.(HTTPClient)
}
return http.DefaultClient
}
func (r *ResourceBase[T]) Sync(ctx context.Context) error {
traceSink := traceSinkFromContext(ctx)
httpcl := r.httpcl
if httpcl == nil {
httpcl = httpClientFromContext(ctx)
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.u, nil)
if err != nil {
return fmt.Errorf(`httprc.Resource.Sync: failed to create request: %w`, err)
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: fetching %q", r.u))
res, err := httpcl.Do(req)
if err != nil {
return fmt.Errorf(`httprc.Resource.Sync: failed to execute HTTP request: %w`, err)
}
defer res.Body.Close()
next := r.calculateNextRefreshTime(ctx, res)
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: next refresh time for %q is %v", r.u, next))
r.SetNext(next)
if res.StatusCode != http.StatusOK {
return fmt.Errorf(`httprc.Resource.Sync: %w (status code=%d, url=%q)`, errUnexpectedStatusCode, res.StatusCode, r.u)
}
// replace the body of the response with a limited reader that
// will only read up to MaxBufferSize bytes
res.Body = &limitedBody{
rdr: &io.LimitedReader{R: res.Body, N: MaxBufferSize},
close: res.Body.Close,
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: transforming %q", r.u))
v, err := r.transform(ctx, res)
if err != nil {
return fmt.Errorf(`httprc.Resource.Sync: %w: %w`, errTransformerFailed, err)
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: storing new value for %q", r.u))
r.r.Store(v)
r.once.Do(func() { close(r.ready) })
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: stored value for %q", r.u))
return nil
}
func (r *ResourceBase[T]) transform(ctx context.Context, res *http.Response) (ret T, gerr error) {
// Protect the call to Transform with a defer/recover block, so that even
// if the Transform method panics, we can recover from it and return an error
defer func() {
if recovered := recover(); recovered != nil {
gerr = fmt.Errorf(`httprc.Resource.transform: %w: %v`, errRecoveredFromPanic, recovered)
}
}()
return r.t.Transform(ctx, res)
}
func (r *ResourceBase[T]) determineNextFetchInterval(ctx context.Context, name string, fromHeader, minValue, maxValue time.Duration) time.Duration {
traceSink := traceSinkFromContext(ctx)
if fromHeader > maxValue {
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s %s > maximum interval, using maximum interval %s", r.URL(), name, maxValue))
return maxValue
}
if fromHeader < minValue {
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s %s < minimum interval, using minimum interval %s", r.URL(), name, minValue))
return minValue
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Using %s (%s)", r.URL(), name, fromHeader))
return fromHeader
}
func (r *ResourceBase[T]) calculateNextRefreshTime(ctx context.Context, res *http.Response) time.Time {
traceSink := traceSinkFromContext(ctx)
now := time.Now()
// If constant interval is set, use that regardless of what the
// response headers say.
if interval := r.ConstantInterval(); interval > 0 {
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Explicit interval set, using value %s", r.URL(), interval))
return now.Add(interval)
}
if interval := r.extractCacheControlMaxAge(ctx, res); interval > 0 {
return now.Add(interval)
}
if interval := r.extractExpiresInterval(ctx, res); interval > 0 {
return now.Add(interval)
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s No cache-control/expires headers found, using minimum interval", r.URL()))
return now.Add(r.MinInterval())
}
func (r *ResourceBase[T]) extractCacheControlMaxAge(ctx context.Context, res *http.Response) time.Duration {
traceSink := traceSinkFromContext(ctx)
v := res.Header.Get(`Cache-Control`)
if v == "" {
return 0
}
dir, err := httpcc.ParseResponse(v)
if err != nil {
return 0
}
maxAge, ok := dir.MaxAge()
if !ok {
return 0
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Cache-Control=max-age directive set (%d)", r.URL(), maxAge))
return r.determineNextFetchInterval(
ctx,
"max-age",
time.Duration(maxAge)*time.Second,
r.MinInterval(),
r.MaxInterval(),
)
}
func (r *ResourceBase[T]) extractExpiresInterval(ctx context.Context, res *http.Response) time.Duration {
traceSink := traceSinkFromContext(ctx)
v := res.Header.Get(`Expires`)
if v == "" {
return 0
}
expires, err := http.ParseTime(v)
if err != nil {
return 0
}
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Expires header set (%s)", r.URL(), expires))
return r.determineNextFetchInterval(
ctx,
"expires",
time.Until(expires),
r.MinInterval(),
r.MaxInterval(),
)
}
+52
View File
@@ -0,0 +1,52 @@
package tracesink
import (
"context"
"log/slog"
)
type Interface interface {
Put(context.Context, string)
}
// Nop is an ErrorSink that does nothing. It does not require
// any initialization, so the zero value can be used.
type Nop struct{}
// NewNop returns a new NopTraceSink object. The constructor
// is provided for consistency.
func NewNop() Interface {
return Nop{}
}
// Put for NopTraceSink does nothing.
func (Nop) Put(context.Context, string) {}
type slogSink struct {
level slog.Level
logger SlogLogger
}
type SlogLogger interface {
Log(context.Context, slog.Level, string, ...any)
}
// NewSlog returns a new ErrorSink that logs errors using the provided slog.Logger
func NewSlog(l SlogLogger) Interface {
return &slogSink{
level: slog.LevelInfo,
logger: l,
}
}
func (s *slogSink) Put(ctx context.Context, v string) {
s.logger.Log(ctx, s.level, v)
}
// Func is a TraceSink that calls a function with the trace message.
type Func func(context.Context, string)
// Put calls the function with the trace message.
func (f Func) Put(ctx context.Context, msg string) {
f(ctx, msg)
}
+37
View File
@@ -0,0 +1,37 @@
package httprc
import (
"context"
"encoding/json"
"io"
"net/http"
)
type bytesTransformer struct{}
// BytesTransformer returns a Transformer that reads the entire response body
// as a byte slice. This is the default Transformer used by httprc.Client
func BytesTransformer() Transformer[[]byte] {
return bytesTransformer{}
}
func (bytesTransformer) Transform(_ context.Context, res *http.Response) ([]byte, error) {
return io.ReadAll(res.Body)
}
type jsonTransformer[T any] struct{}
// JSONTransformer returns a Transformer that decodes the response body as JSON
// into the provided type T.
func JSONTransformer[T any]() Transformer[T] {
return jsonTransformer[T]{}
}
func (jsonTransformer[T]) Transform(_ context.Context, res *http.Response) (T, error) {
var v T
if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
var zero T
return zero, err
}
return v, nil
}
+113
View File
@@ -0,0 +1,113 @@
package httprc
import (
"regexp"
"sync"
)
// Whitelist is an interface that allows you to determine if a given URL is allowed
// or not. Implementations of this interface can be used to restrict the URLs that
// the client can access.
//
// By default all URLs are allowed, but this may not be ideal in production environments
// for security reasons.
//
// This exists because you might use this module to store resources provided by
// user of your application, in which case you cannot necessarily trust that the
// URLs are safe.
//
// You will HAVE to provide some sort of whitelist.
type Whitelist interface {
IsAllowed(string) bool
}
// WhitelistFunc is a function type that implements the Whitelist interface.
type WhitelistFunc func(string) bool
func (f WhitelistFunc) IsAllowed(u string) bool { return f(u) }
// BlockAllWhitelist is a Whitelist implementation that blocks all URLs.
type BlockAllWhitelist struct{}
// NewBlockAllWhitelist creates a new BlockAllWhitelist instance. It is safe to
// use the zero value of this type; this constructor is provided for consistency.
func NewBlockAllWhitelist() BlockAllWhitelist { return BlockAllWhitelist{} }
func (BlockAllWhitelist) IsAllowed(_ string) bool { return false }
// InsecureWhitelist is a Whitelist implementation that allows all URLs. Be careful
// when using this in your production code: make sure you do not blindly register
// URLs from untrusted sources.
type InsecureWhitelist struct{}
// NewInsecureWhitelist creates a new InsecureWhitelist instance. It is safe to
// use the zero value of this type; this constructor is provided for consistency.
func NewInsecureWhitelist() InsecureWhitelist { return InsecureWhitelist{} }
func (InsecureWhitelist) IsAllowed(_ string) bool { return true }
// RegexpWhitelist is a jwk.Whitelist object comprised of a list of *regexp.Regexp
// objects. All entries in the list are tried until one matches. If none of the
// *regexp.Regexp objects match, then the URL is deemed unallowed.
type RegexpWhitelist struct {
mu sync.RWMutex
patterns []*regexp.Regexp
}
// NewRegexpWhitelist creates a new RegexpWhitelist instance. It is safe to use the
// zero value of this type; this constructor is provided for consistency.
func NewRegexpWhitelist() *RegexpWhitelist {
return &RegexpWhitelist{}
}
// Add adds a new regular expression to the list of expressions to match against.
func (w *RegexpWhitelist) Add(pat *regexp.Regexp) *RegexpWhitelist {
w.mu.Lock()
defer w.mu.Unlock()
w.patterns = append(w.patterns, pat)
return w
}
// IsAllowed returns true if any of the patterns in the whitelist
// returns true.
func (w *RegexpWhitelist) IsAllowed(u string) bool {
w.mu.RLock()
patterns := w.patterns
w.mu.RUnlock()
for _, pat := range patterns {
if pat.MatchString(u) {
return true
}
}
return false
}
// MapWhitelist is a jwk.Whitelist object comprised of a map of strings.
// If the URL exists in the map, then the URL is allowed to be fetched.
type MapWhitelist interface {
Whitelist
Add(string) MapWhitelist
}
type mapWhitelist struct {
mu sync.RWMutex
store map[string]struct{}
}
func NewMapWhitelist() MapWhitelist {
return &mapWhitelist{store: make(map[string]struct{})}
}
func (w *mapWhitelist) Add(pat string) MapWhitelist {
w.mu.Lock()
defer w.mu.Unlock()
w.store[pat] = struct{}{}
return w
}
func (w *mapWhitelist) IsAllowed(u string) bool {
w.mu.RLock()
_, b := w.store[u]
w.mu.RUnlock()
return b
}
+62
View File
@@ -0,0 +1,62 @@
package httprc
import (
"context"
"fmt"
"sync"
)
type worker struct {
httpcl HTTPClient
incoming chan any
next <-chan Resource
nextsync <-chan synchronousRequest
errSink ErrorSink
traceSink TraceSink
}
func (w worker) Run(ctx context.Context, readywg *sync.WaitGroup, donewg *sync.WaitGroup) {
w.traceSink.Put(ctx, "httprc worker: START worker loop")
defer w.traceSink.Put(ctx, "httprc worker: END worker loop")
defer donewg.Done()
ctx = withTraceSink(ctx, w.traceSink)
ctx = withHTTPClient(ctx, w.httpcl)
readywg.Done()
for {
select {
case <-ctx.Done():
w.traceSink.Put(ctx, "httprc worker: stopping worker loop")
return
case r := <-w.next:
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: syncing %q (async)", r.URL()))
if err := r.Sync(ctx); err != nil {
w.errSink.Put(ctx, err)
}
r.SetBusy(false)
w.sendAdjustIntervalRequest(ctx, r)
case sr := <-w.nextsync:
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: syncing %q (synchronous)", sr.resource.URL()))
if err := sr.resource.Sync(ctx); err != nil {
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: FAILED to sync %q (synchronous): %s", sr.resource.URL(), err))
sendReply(ctx, sr.reply, struct{}{}, err)
sr.resource.SetBusy(false)
return
}
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: SUCCESS syncing %q (synchronous)", sr.resource.URL()))
sr.resource.SetBusy(false)
sendReply(ctx, sr.reply, struct{}{}, nil)
w.sendAdjustIntervalRequest(ctx, sr.resource)
}
}
}
func (w worker) sendAdjustIntervalRequest(ctx context.Context, r Resource) {
w.traceSink.Put(ctx, "httprc worker: Sending interval adjustment request for "+r.URL())
select {
case <-ctx.Done():
case w.incoming <- adjustIntervalRequest{resource: r}:
}
w.traceSink.Put(ctx, "httprc worker: Sent interval adjustment request for "+r.URL())
}