Initial QSfera import
This commit is contained in:
@@ -0,0 +1,15 @@
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
version: "2"
|
||||
linters:
|
||||
default: all
|
||||
disable:
|
||||
- cyclop
|
||||
- depguard
|
||||
- dupl
|
||||
- errorlint
|
||||
- exhaustive
|
||||
- forbidigo
|
||||
- funcorder
|
||||
- funlen
|
||||
- gochecknoglobals
|
||||
- gochecknoinits
|
||||
- gocognit
|
||||
- gocritic
|
||||
- gocyclo
|
||||
- godot
|
||||
- godox
|
||||
- gosec
|
||||
- gosmopolitan
|
||||
- govet
|
||||
- inamedparam
|
||||
- ireturn
|
||||
- lll
|
||||
- maintidx
|
||||
- makezero
|
||||
- mnd
|
||||
- nakedret
|
||||
- nestif
|
||||
- nlreturn
|
||||
- noinlineerr
|
||||
- nonamedreturns
|
||||
- paralleltest
|
||||
- tagliatelle
|
||||
- testpackage
|
||||
- thelper
|
||||
- varnamelen
|
||||
- wrapcheck
|
||||
- wsl
|
||||
- wsl_v5
|
||||
settings:
|
||||
govet:
|
||||
disable:
|
||||
- shadow
|
||||
- fieldalignment
|
||||
enable-all: true
|
||||
exclusions:
|
||||
generated: lax
|
||||
presets:
|
||||
- comments
|
||||
- common-false-positives
|
||||
- legacy
|
||||
- std-error-handling
|
||||
rules:
|
||||
- linters:
|
||||
- staticcheck
|
||||
path: /*.go
|
||||
text: 'ST1003: should not use underscores in package names'
|
||||
- linters:
|
||||
- revive
|
||||
path: /*.go
|
||||
text: don't use an underscore in package name
|
||||
- linters:
|
||||
- contextcheck
|
||||
- exhaustruct
|
||||
path: /*.go
|
||||
- linters:
|
||||
- errcheck
|
||||
path: /main.go
|
||||
- linters:
|
||||
- errcheck
|
||||
- errchkjson
|
||||
- forcetypeassert
|
||||
path: /*_test.go
|
||||
- linters:
|
||||
- forbidigo
|
||||
path: /*_example_test.go
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
issues:
|
||||
max-issues-per-linter: 0
|
||||
max-same-issues: 0
|
||||
formatters:
|
||||
enable:
|
||||
- gofmt
|
||||
- goimports
|
||||
exclusions:
|
||||
generated: lax
|
||||
paths:
|
||||
- third_party$
|
||||
- builtin$
|
||||
- examples$
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
Changes
|
||||
=======
|
||||
|
||||
v3.0.2 05 Dev 2025
|
||||
* Code changes mainly due to upgraded linter.
|
||||
* github.com/lestrrat-go/option upgraded to v2
|
||||
|
||||
v3.0.1 18 Aug 2025
|
||||
* Refresh() no longer requires the resource to be ready.
|
||||
|
||||
v3.0.0 5 Jun 2025
|
||||
[Breaking Changes]
|
||||
* The entire API has been re-imagined for Go versions that allow typed parameters
|
||||
|
||||
v2.0.0 19 Feb 2024
|
||||
[Breaking Changes]
|
||||
* `Fetcher` type is no longer available. You probably want to provide
|
||||
a customg HTTP client instead via httprc.WithHTTPClient()).
|
||||
*
|
||||
|
||||
v1.0.4 19 Jul 2022
|
||||
* Fix sloppy API breakage
|
||||
|
||||
v1.0.3 19 Jul 2022
|
||||
* Fix queue insertion in the middle of the queue (#7)
|
||||
|
||||
v1.0.2 13 Jun 2022
|
||||
* Properly release a lock when the fetch fails (#5)
|
||||
|
||||
v1.0.1 29 Mar 2022
|
||||
* Bump dependency for github.com/lestrrat-go/httpcc to v1.0.1
|
||||
|
||||
v1.0.0 29 Mar 2022
|
||||
* Initial release, refactored out of github.com/lestrrat-go/jwx
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2022 lestrrat
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
+172
@@ -0,0 +1,172 @@
|
||||
# github.com/lestrrat-go/httprc/v3  [](https://pkg.go.dev/github.com/lestrrat-go/httprc/v3)
|
||||
|
||||
`httprc` is a HTTP "Refresh" Cache. Its aim is to cache a remote resource that
|
||||
can be fetched via HTTP, but keep the cached content up-to-date based on periodic
|
||||
refreshing.
|
||||
|
||||
# Client
|
||||
|
||||
A `httprc.Client` object is comprised of 3 parts: The user-facing controller API,
|
||||
the main controller loop, and set of workers that perform the actual fetching.
|
||||
|
||||
The user-facing controller API is the object returned when you call `(httprc.Client).Start`.
|
||||
|
||||
```go
|
||||
ctrl, _ := client.Start(ctx)
|
||||
```
|
||||
|
||||
# Controller API
|
||||
|
||||
The controller API gives you access to the controller backend that runs asynchronously.
|
||||
All methods take a `context.Context` object because they potentially block. You should
|
||||
be careful to use `context.WithTimeout` to properly set a timeout if you cannot tolerate
|
||||
a blocking operation.
|
||||
|
||||
# Main Controller Loop
|
||||
|
||||
The main controller loop is run asynchronously to the controller API. It is single threaded,
|
||||
and it has two reponsibilities.
|
||||
|
||||
The first is to receive commands from the controller API,
|
||||
and appropriately modify the state of the goroutine, i.e. modify the list of resources
|
||||
it is watching, performing forced refreshes, etc.
|
||||
|
||||
The other is to periodically wake up and go through the list of resources and re-fetch
|
||||
ones that are past their TTL (in reality, each resource carry a "next-check" time, not
|
||||
a TTL). The main controller loop itself does nothing more: it just kicks these checks periodically.
|
||||
|
||||
The interval between fetches is changed dynamically based on either the metadata carried
|
||||
with the HTTP responses, such as `Cache-Control` and `Expires` headers, or a constant
|
||||
interval set by the user for a given resource. Between these values, the main controller loop
|
||||
will pick the shortest interval (but no less than 1 second) and checks if resources
|
||||
need updating based on that value.
|
||||
|
||||
For example, if a resource A has an expiry of 10 minutes and if resource has an expiry of 5
|
||||
minutes, the main controller loop will attempt to wake up roughly every 5 minutes to check
|
||||
on the resources.
|
||||
|
||||
When the controller loop detects that a resource needs to be checked for freshness,
|
||||
it will send the resource to the worker pool to be synced.
|
||||
|
||||
# Interval calculation
|
||||
|
||||
After the resource is synced, the next fetch is scheduled. The interval to the next
|
||||
fetch is calculated either by using constant intervals, or by heuristics using values
|
||||
from the `http.Response` object.
|
||||
|
||||
If the constant interval is specified, no extra calculation is performed. If you specify
|
||||
a constant interval of 15 minutes, the resource will be checked every 15 minutes. This is
|
||||
predictable and reliable, but not necessarily efficient.
|
||||
|
||||
If you do not specify a constant interval, the HTTP response is analyzed for
|
||||
values in `Cache-Control` and `Expires` headers. These values will be compared against
|
||||
a maximum and minimum interval values, which default to 30 days and 15 minutes, respectively.
|
||||
If the values obtained from the headers fall within that range, the value from the header is
|
||||
used. If the value is larger than the maximum, the maximum is used. If the value is lower
|
||||
than the minimum, the minimum is used.
|
||||
|
||||
# SYNOPSIS
|
||||
|
||||
<!-- INCLUDE(client_example_test.go) -->
|
||||
```go
|
||||
package httprc_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/httprc/v3"
|
||||
)
|
||||
|
||||
func ExampleClient() {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
type HelloWorld struct {
|
||||
Hello string `json:"hello"`
|
||||
}
|
||||
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
json.NewEncoder(w).Encode(map[string]string{"hello": "world"})
|
||||
}))
|
||||
|
||||
options := []httprc.NewClientOption{
|
||||
// By default the client will allow all URLs (which is what the option
|
||||
// below is explicitly specifying). If you want to restrict what URLs
|
||||
// are allowed, you can specify another whitelist.
|
||||
//
|
||||
// httprc.WithWhitelist(httprc.NewInsecureWhitelist()),
|
||||
}
|
||||
// If you would like to handle errors from asynchronous workers, you can specify a error sink.
|
||||
// This is disabled in this example because the trace logs are dynamic
|
||||
// and thus would interfere with the runnable example test.
|
||||
// options = append(options, httprc.WithErrorSink(errsink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))))
|
||||
|
||||
// If you would like to see the trace logs, you can specify a trace sink.
|
||||
// This is disabled in this example because the trace logs are dynamic
|
||||
// and thus would interfere with the runnable example test.
|
||||
// options = append(options, httprc.WithTraceSink(tracesink.NewSlog(slog.New(slog.NewJSONHandler(os.Stdout, nil)))))
|
||||
|
||||
// Create a new client
|
||||
cl := httprc.NewClient(options...)
|
||||
|
||||
// Start the client, and obtain a Controller object
|
||||
ctrl, err := cl.Start(ctx)
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return
|
||||
}
|
||||
// The following is required if you want to make sure that there are no
|
||||
// dangling goroutines hanging around when you exit. For example, if you
|
||||
// are running tests to check for goroutine leaks, you should call this
|
||||
// function before the end of your test.
|
||||
defer ctrl.Shutdown(time.Second)
|
||||
|
||||
// Create a new resource that is synchronized every so often
|
||||
//
|
||||
// By default the client will attempt to fetch the resource once
|
||||
// as soon as it can, and then if no other metadata is provided,
|
||||
// it will fetch the resource every 15 minutes.
|
||||
//
|
||||
// If the resource responds with a Cache-Control/Expires header,
|
||||
// the client will attempt to respect that, and will try to fetch
|
||||
// the resource again based on the values obatained from the headers.
|
||||
r, err := httprc.NewResource[HelloWorld](srv.URL, httprc.JSONTransformer[HelloWorld]())
|
||||
if err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
// Add the resource to the controller, so that it starts fetching.
|
||||
// By default, a call to `Add()` will block until the first fetch
|
||||
// succeeds, via an implicit call to `r.Ready()`
|
||||
// You can change this behavior if you specify the `WithWaitReady(false)`
|
||||
// option.
|
||||
ctrl.Add(ctx, r)
|
||||
|
||||
// if you specified `httprc.WithWaitReady(false)` option, the fetch will happen
|
||||
// "soon", but you're not guaranteed that it will happen before the next
|
||||
// call to `Lookup()`. If you want to make sure that the resource is ready,
|
||||
// you can call `Ready()` like so:
|
||||
/*
|
||||
{
|
||||
tctx, tcancel := context.WithTimeout(ctx, time.Second)
|
||||
defer tcancel()
|
||||
if err := r.Ready(tctx); err != nil {
|
||||
fmt.Println(err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
*/
|
||||
m := r.Resource()
|
||||
fmt.Println(m.Hello)
|
||||
// OUTPUT:
|
||||
// world
|
||||
}
|
||||
```
|
||||
source: [client_example_test.go](https://github.com/lestrrat-go/httprc/blob/refs/heads/v3/client_example_test.go)
|
||||
<!-- END INCLUDE -->
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
func (c *ctrlBackend) adjustInterval(ctx context.Context, req adjustIntervalRequest) {
|
||||
interval := roundupToSeconds(time.Until(req.resource.Next()))
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got adjust request (current tick interval=%s, next for %q=%s)", c.tickInterval, req.resource.URL(), interval))
|
||||
|
||||
if interval < time.Second {
|
||||
interval = time.Second
|
||||
}
|
||||
|
||||
if c.tickInterval < interval {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: no adjusting required (time to next check %s > current tick interval %s)", interval, c.tickInterval))
|
||||
} else {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: adjusting tick interval to %s", interval))
|
||||
c.tickInterval = interval
|
||||
c.check.Reset(interval)
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) addResource(ctx context.Context, req addRequest) {
|
||||
r := req.resource
|
||||
if _, ok := c.items[r.URL()]; ok {
|
||||
// Already exists
|
||||
sendReply(ctx, req.reply, struct{}{}, errResourceAlreadyExists)
|
||||
return
|
||||
}
|
||||
c.items[r.URL()] = r
|
||||
|
||||
if r.MaxInterval() == 0 {
|
||||
r.SetMaxInterval(c.defaultMaxInterval)
|
||||
}
|
||||
|
||||
if r.MinInterval() == 0 {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: set minimum interval to %s", c.defaultMinInterval))
|
||||
r.SetMinInterval(c.defaultMinInterval)
|
||||
}
|
||||
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: added resource %q", r.URL()))
|
||||
sendReply(ctx, req.reply, struct{}{}, nil)
|
||||
c.SetTickInterval(time.Nanosecond)
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) rmResource(ctx context.Context, req rmRequest) {
|
||||
u := req.u
|
||||
if _, ok := c.items[u]; !ok {
|
||||
sendReply(ctx, req.reply, struct{}{}, errResourceNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
delete(c.items, u)
|
||||
|
||||
minInterval := oneDay
|
||||
for _, item := range c.items {
|
||||
if d := item.MinInterval(); d < minInterval {
|
||||
minInterval = d
|
||||
}
|
||||
}
|
||||
|
||||
close(req.reply)
|
||||
c.check.Reset(minInterval)
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) refreshResource(ctx context.Context, req refreshRequest) {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] START %q", req.u))
|
||||
defer c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] END %q", req.u))
|
||||
u := req.u
|
||||
|
||||
r, ok := c.items[u]
|
||||
if !ok {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] %s is not registered", req.u))
|
||||
sendReply(ctx, req.reply, struct{}{}, errResourceNotFound)
|
||||
return
|
||||
}
|
||||
|
||||
// Note: We don't wait for r.Ready() here because refresh should work
|
||||
// regardless of whether the resource has been fetched before. This allows
|
||||
// refresh to work with resources registered using WithWaitReady(false).
|
||||
|
||||
r.SetNext(time.Unix(0, 0))
|
||||
sendWorkerSynchronous(ctx, c.syncoutgoing, synchronousRequest{
|
||||
resource: r,
|
||||
reply: req.reply,
|
||||
})
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: [refresh] sync request for %s sent to worker pool", req.u))
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) lookupResource(ctx context.Context, req lookupRequest) {
|
||||
u := req.u
|
||||
r, ok := c.items[u]
|
||||
if !ok {
|
||||
sendReply(ctx, req.reply, nil, errResourceNotFound)
|
||||
return
|
||||
}
|
||||
sendReply(ctx, req.reply, r, nil)
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) handleRequest(ctx context.Context, req any) {
|
||||
switch req := req.(type) {
|
||||
case adjustIntervalRequest:
|
||||
c.adjustInterval(ctx, req)
|
||||
case addRequest:
|
||||
c.addResource(ctx, req)
|
||||
case rmRequest:
|
||||
c.rmResource(ctx, req)
|
||||
case refreshRequest:
|
||||
c.refreshResource(ctx, req)
|
||||
case lookupRequest:
|
||||
c.lookupResource(ctx, req)
|
||||
default:
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: unknown request type %T", req))
|
||||
}
|
||||
}
|
||||
|
||||
func sendWorker(ctx context.Context, ch chan Resource, r Resource) {
|
||||
r.SetBusy(true)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case ch <- r:
|
||||
}
|
||||
}
|
||||
|
||||
func sendWorkerSynchronous(ctx context.Context, ch chan synchronousRequest, r synchronousRequest) {
|
||||
r.resource.SetBusy(true)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case ch <- r:
|
||||
}
|
||||
}
|
||||
|
||||
func sendReply[T any](ctx context.Context, ch chan backendResponse[T], v T, err error) {
|
||||
defer close(ch)
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case ch <- backendResponse[T]{payload: v, err: err}:
|
||||
}
|
||||
}
|
||||
|
||||
type ctrlBackend struct {
|
||||
items map[string]Resource
|
||||
outgoing chan Resource
|
||||
syncoutgoing chan synchronousRequest
|
||||
incoming chan any // incoming requests to the controller
|
||||
traceSink TraceSink
|
||||
tickInterval time.Duration
|
||||
check *time.Ticker
|
||||
defaultMaxInterval time.Duration
|
||||
defaultMinInterval time.Duration
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) loop(ctx context.Context, readywg, donewg *sync.WaitGroup) {
|
||||
c.traceSink.Put(ctx, "httprc controller: starting main controller loop")
|
||||
readywg.Done()
|
||||
defer c.traceSink.Put(ctx, "httprc controller: stopping main controller loop")
|
||||
defer donewg.Done()
|
||||
for {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: waiting for request or tick (tick interval=%s)", c.tickInterval))
|
||||
select {
|
||||
case req := <-c.incoming:
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: got request %T", req))
|
||||
c.handleRequest(ctx, req)
|
||||
case t := <-c.check.C:
|
||||
c.periodicCheck(ctx, t)
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) periodicCheck(ctx context.Context, t time.Time) {
|
||||
c.traceSink.Put(ctx, "httprc controller: START periodic check")
|
||||
defer c.traceSink.Put(ctx, "httprc controller: END periodic check")
|
||||
var minNext time.Time
|
||||
var dispatched int
|
||||
minInterval := -1 * time.Second
|
||||
for _, item := range c.items {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: checking resource %q", item.URL()))
|
||||
|
||||
next := item.Next()
|
||||
if minNext.IsZero() || next.Before(minNext) {
|
||||
minNext = next
|
||||
}
|
||||
|
||||
if interval := item.MinInterval(); minInterval < 0 || interval < minInterval {
|
||||
minInterval = interval
|
||||
}
|
||||
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q isBusy=%t, next(%s).After(%s)=%t", item.URL(), item.IsBusy(), next, t, next.After(t)))
|
||||
if item.IsBusy() || next.After(t) {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q is busy or not ready yet, skipping", item.URL()))
|
||||
continue
|
||||
}
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resource %q is ready, dispatching to worker pool", item.URL()))
|
||||
|
||||
dispatched++
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: dispatching resource %q to worker pool", item.URL()))
|
||||
sendWorker(ctx, c.outgoing, item)
|
||||
}
|
||||
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: dispatched %d resources", dispatched))
|
||||
|
||||
// Next check is always at the earliest next check + 1 second.
|
||||
// The extra second makes sure that we are _past_ the actual next check time
|
||||
// so we can send the resource to the worker pool
|
||||
if interval := time.Until(minNext); interval > 0 {
|
||||
c.SetTickInterval(roundupToSeconds(interval) + time.Second)
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resetting check intervanl to %s", c.tickInterval))
|
||||
} else {
|
||||
// if we got here, either we have no resources, or all resources are busy.
|
||||
// In this state, it's possible that the interval is less than 1 second,
|
||||
// because we previously set it to a small value for an immediate refresh.
|
||||
// in this case, we want to reset it to a sane value
|
||||
if c.tickInterval < time.Second {
|
||||
c.SetTickInterval(minInterval)
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: resetting check intervanl to %s after forced refresh", c.tickInterval))
|
||||
}
|
||||
}
|
||||
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: next check in %s", c.tickInterval))
|
||||
}
|
||||
|
||||
func (c *ctrlBackend) SetTickInterval(d time.Duration) {
|
||||
// TODO synchronize
|
||||
if d <= 0 {
|
||||
d = time.Second // ensure positive interval
|
||||
}
|
||||
c.tickInterval = d
|
||||
c.check.Reset(d)
|
||||
}
|
||||
+185
@@ -0,0 +1,185 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/httprc/v3/errsink"
|
||||
"github.com/lestrrat-go/httprc/v3/proxysink"
|
||||
"github.com/lestrrat-go/httprc/v3/tracesink"
|
||||
)
|
||||
|
||||
// setupSink creates and starts a proxy for the given sink if it's not a Nop sink
|
||||
// Returns the sink to use and a cancel function that should be chained with the original cancel
|
||||
func setupSink[T any, S proxysink.Backend[T], NopType any](ctx context.Context, sink S, wg *sync.WaitGroup) (S, context.CancelFunc) {
|
||||
if _, ok := any(sink).(NopType); ok {
|
||||
return sink, func() {}
|
||||
}
|
||||
|
||||
proxy := proxysink.New[T](sink)
|
||||
wg.Add(1)
|
||||
go func(ctx context.Context, wg *sync.WaitGroup, proxy *proxysink.Proxy[T]) {
|
||||
defer wg.Done()
|
||||
proxy.Run(ctx)
|
||||
}(ctx, wg, proxy)
|
||||
|
||||
// proxy can be converted to one of the sink subtypes
|
||||
s, ok := any(proxy).(S)
|
||||
if !ok {
|
||||
panic("type assertion failed: proxy cannot be converted to type S")
|
||||
}
|
||||
return s, proxy.Close
|
||||
}
|
||||
|
||||
// Client is the main entry point for the httprc package.
|
||||
type Client struct {
|
||||
mu sync.Mutex
|
||||
httpcl HTTPClient
|
||||
numWorkers int
|
||||
running bool
|
||||
errSink ErrorSink
|
||||
traceSink TraceSink
|
||||
wl Whitelist
|
||||
defaultMaxInterval time.Duration
|
||||
defaultMinInterval time.Duration
|
||||
}
|
||||
|
||||
// NewClient creates a new `httprc.Client` object.
|
||||
//
|
||||
// By default ALL urls are allowed. This may not be suitable for you if
|
||||
// are using this in a production environment. You are encouraged to specify
|
||||
// a whitelist using the `WithWhitelist` option.
|
||||
//
|
||||
// NOTE: In future versions, this function signature should be changed to
|
||||
// return an error to properly handle option parsing failures.
|
||||
func NewClient(options ...NewClientOption) *Client {
|
||||
//nolint:staticcheck
|
||||
var errSink ErrorSink = errsink.NewNop()
|
||||
//nolint:staticcheck
|
||||
var traceSink TraceSink = tracesink.NewNop()
|
||||
var wl Whitelist = InsecureWhitelist{}
|
||||
var httpcl HTTPClient = http.DefaultClient
|
||||
|
||||
defaultMinInterval := DefaultMinInterval
|
||||
defaultMaxInterval := DefaultMaxInterval
|
||||
|
||||
numWorkers := DefaultWorkers
|
||||
for _, option := range options {
|
||||
switch option.Ident() {
|
||||
case identHTTPClient{}:
|
||||
_ = option.Value(&httpcl)
|
||||
case identWorkers{}:
|
||||
_ = option.Value(&numWorkers)
|
||||
case identErrorSink{}:
|
||||
_ = option.Value(&errSink)
|
||||
case identTraceSink{}:
|
||||
_ = option.Value(&traceSink)
|
||||
case identWhitelist{}:
|
||||
_ = option.Value(&wl)
|
||||
}
|
||||
}
|
||||
|
||||
if numWorkers <= 0 {
|
||||
numWorkers = 1
|
||||
}
|
||||
return &Client{
|
||||
httpcl: httpcl,
|
||||
numWorkers: numWorkers,
|
||||
errSink: errSink,
|
||||
traceSink: traceSink,
|
||||
wl: wl,
|
||||
|
||||
defaultMinInterval: defaultMinInterval,
|
||||
defaultMaxInterval: defaultMaxInterval,
|
||||
}
|
||||
}
|
||||
|
||||
// Start sets the client into motion. It will start a number of worker goroutines,
|
||||
// and return a Controller object that you can use to control the execution of
|
||||
// the client.
|
||||
//
|
||||
// If you attempt to call Start more than once, it will return an error.
|
||||
func (c *Client) Start(octx context.Context) (Controller, error) {
|
||||
c.mu.Lock()
|
||||
if c.running {
|
||||
c.mu.Unlock()
|
||||
return nil, errAlreadyRunning
|
||||
}
|
||||
c.running = true
|
||||
c.mu.Unlock()
|
||||
|
||||
// DON'T CANCEL THIS IN THIS METHOD! It's the responsibility of the
|
||||
// controller to cancel this context.
|
||||
ctx, cancel := context.WithCancel(octx)
|
||||
|
||||
var donewg sync.WaitGroup
|
||||
|
||||
// start proxy goroutines that will accept sink requests
|
||||
// and forward them to the appropriate sink
|
||||
errSink, errCancel := setupSink[error, ErrorSink, errsink.Nop](ctx, c.errSink, &donewg)
|
||||
traceSink, traceCancel := setupSink[string, TraceSink, tracesink.Nop](ctx, c.traceSink, &donewg)
|
||||
|
||||
// Chain the cancel functions
|
||||
ocancel := cancel
|
||||
cancel = func() {
|
||||
ocancel()
|
||||
errCancel()
|
||||
traceCancel()
|
||||
}
|
||||
|
||||
chbuf := c.numWorkers + 1
|
||||
incoming := make(chan any, chbuf)
|
||||
outgoing := make(chan Resource, chbuf)
|
||||
syncoutgoing := make(chan synchronousRequest, chbuf)
|
||||
|
||||
var readywg sync.WaitGroup
|
||||
readywg.Add(c.numWorkers)
|
||||
donewg.Add(c.numWorkers)
|
||||
for range c.numWorkers {
|
||||
wrk := worker{
|
||||
incoming: incoming,
|
||||
next: outgoing,
|
||||
nextsync: syncoutgoing,
|
||||
errSink: errSink,
|
||||
traceSink: traceSink,
|
||||
httpcl: c.httpcl,
|
||||
}
|
||||
go wrk.Run(ctx, &readywg, &donewg)
|
||||
}
|
||||
|
||||
tickInterval := oneDay
|
||||
ctrl := &controller{
|
||||
cancel: cancel,
|
||||
incoming: incoming,
|
||||
shutdown: make(chan struct{}),
|
||||
traceSink: traceSink,
|
||||
wl: c.wl,
|
||||
}
|
||||
|
||||
backend := &ctrlBackend{
|
||||
items: make(map[string]Resource),
|
||||
outgoing: outgoing,
|
||||
syncoutgoing: syncoutgoing,
|
||||
incoming: incoming,
|
||||
traceSink: traceSink,
|
||||
tickInterval: tickInterval,
|
||||
check: time.NewTicker(tickInterval),
|
||||
|
||||
defaultMinInterval: c.defaultMinInterval,
|
||||
defaultMaxInterval: c.defaultMaxInterval,
|
||||
}
|
||||
donewg.Add(1)
|
||||
readywg.Add(1)
|
||||
go backend.loop(ctx, &readywg, &donewg)
|
||||
|
||||
go func(wg *sync.WaitGroup, ch chan struct{}) {
|
||||
wg.Wait()
|
||||
close(ch)
|
||||
}(&donewg, ctrl.shutdown)
|
||||
|
||||
readywg.Wait()
|
||||
|
||||
return ctrl, nil
|
||||
}
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Controller interface {
|
||||
// Add adds a new `http.Resource` to the controller. If the resource already exists,
|
||||
// it will return an error.
|
||||
Add(context.Context, Resource, ...AddOption) error
|
||||
|
||||
// Lookup a `httprc.Resource` by its URL. If the resource does not exist, it
|
||||
// will return an error.
|
||||
Lookup(context.Context, string) (Resource, error)
|
||||
|
||||
// Remove a `httprc.Resource` from the controller by its URL. If the resource does
|
||||
// not exist, it will return an error.
|
||||
Remove(context.Context, string) error
|
||||
|
||||
// Refresh forces a resource to be refreshed immediately. If the resource does
|
||||
// not exist, or if the refresh fails, it will return an error.
|
||||
Refresh(context.Context, string) error
|
||||
|
||||
ShutdownContext(context.Context) error
|
||||
Shutdown(time.Duration) error
|
||||
}
|
||||
|
||||
type controller struct {
|
||||
cancel context.CancelFunc
|
||||
incoming chan any // incoming requests to the controller
|
||||
shutdown chan struct{}
|
||||
traceSink TraceSink
|
||||
wl Whitelist
|
||||
}
|
||||
|
||||
// Shutdown is a convenience function that calls ShutdownContext with a
|
||||
// context that has a timeout of `timeout`.
|
||||
func (c *controller) Shutdown(timeout time.Duration) error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), timeout)
|
||||
defer cancel()
|
||||
return c.ShutdownContext(ctx)
|
||||
}
|
||||
|
||||
// ShutdownContext stops the client and all associated goroutines, and waits for them
|
||||
// to finish. If the context is canceled, the function will return immediately:
|
||||
// there fore you should not use the context you used to start the client (because
|
||||
// presumably it's already canceled).
|
||||
//
|
||||
// Waiting for the client shutdown will also ensure that all sinks are properly
|
||||
// flushed.
|
||||
func (c *controller) ShutdownContext(ctx context.Context) error {
|
||||
c.cancel()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-c.shutdown:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
type ctrlRequest[T any] struct {
|
||||
reply chan T
|
||||
resource Resource
|
||||
u string
|
||||
}
|
||||
type addRequest ctrlRequest[backendResponse[struct{}]]
|
||||
type rmRequest ctrlRequest[backendResponse[struct{}]]
|
||||
type refreshRequest ctrlRequest[backendResponse[struct{}]]
|
||||
type lookupRequest ctrlRequest[backendResponse[Resource]]
|
||||
type synchronousRequest ctrlRequest[backendResponse[struct{}]]
|
||||
type adjustIntervalRequest struct {
|
||||
resource Resource
|
||||
}
|
||||
|
||||
type backendResponse[T any] struct {
|
||||
payload T
|
||||
err error
|
||||
}
|
||||
|
||||
func sendBackend[TReq any, TB any](ctx context.Context, backendCh chan any, v TReq, replyCh chan backendResponse[TB]) (TB, error) {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case backendCh <- v:
|
||||
}
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
var zero TB
|
||||
return zero, ctx.Err()
|
||||
case res := <-replyCh:
|
||||
return res.payload, res.err
|
||||
}
|
||||
}
|
||||
|
||||
// Lookup returns a resource by its URL. If the resource does not exist, it
|
||||
// will return an error.
|
||||
//
|
||||
// Unfortunately, due to the way typed parameters are handled in Go, we can only
|
||||
// return a Resource object (and not a ResourceBase[T] object). This means that
|
||||
// you will either need to use the `Resource.Get()` method or use a type
|
||||
// assertion to obtain a `ResourceBase[T]` to get to the actual object you are
|
||||
// looking for
|
||||
func (c *controller) Lookup(ctx context.Context, u string) (Resource, error) {
|
||||
reply := make(chan backendResponse[Resource], 1)
|
||||
req := lookupRequest{
|
||||
reply: reply,
|
||||
u: u,
|
||||
}
|
||||
return sendBackend[lookupRequest, Resource](ctx, c.incoming, req, reply)
|
||||
}
|
||||
|
||||
// Add adds a new resource to the controller. If the resource already
|
||||
// exists, it will return an error.
|
||||
//
|
||||
// By default this function will automatically wait for the resource to be
|
||||
// fetched once (by calling `r.Ready()`). Note that the `r.Ready()` call will NOT
|
||||
// timeout unless you configure your context object with `context.WithTimeout`.
|
||||
// To disable waiting, you can specify the `WithWaitReady(false)` option.
|
||||
func (c *controller) Add(ctx context.Context, r Resource, options ...AddOption) error {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: START Add(%q)", r.URL()))
|
||||
defer c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: END Add(%q)", r.URL()))
|
||||
waitReady := true
|
||||
for _, option := range options {
|
||||
switch option.Ident() {
|
||||
case identWaitReady{}:
|
||||
if err := option.Value(&waitReady); err != nil {
|
||||
return fmt.Errorf(`httprc.Controller.Add: failed to parse WaitReady option: %w`, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !c.wl.IsAllowed(r.URL()) {
|
||||
return fmt.Errorf(`httprc.Controller.AddResource: cannot add %q: %w`, r.URL(), errBlockedByWhitelist)
|
||||
}
|
||||
|
||||
reply := make(chan backendResponse[struct{}], 1)
|
||||
req := addRequest{
|
||||
reply: reply,
|
||||
resource: r,
|
||||
}
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: sending add request for %q to backend", r.URL()))
|
||||
if _, err := sendBackend[addRequest, struct{}](ctx, c.incoming, req, reply); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if waitReady {
|
||||
c.traceSink.Put(ctx, fmt.Sprintf("httprc controller: waiting for resource %q to be ready", r.URL()))
|
||||
if err := r.Ready(ctx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Remove removes a resource from the controller. If the resource does
|
||||
// not exist, it will return an error.
|
||||
func (c *controller) Remove(ctx context.Context, u string) error {
|
||||
reply := make(chan backendResponse[struct{}], 1)
|
||||
req := rmRequest{
|
||||
reply: reply,
|
||||
u: u,
|
||||
}
|
||||
if _, err := sendBackend[rmRequest, struct{}](ctx, c.incoming, req, reply); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Refresh forces a resource to be refreshed immediately. If the resource does
|
||||
// not exist, or if the refresh fails, it will return an error.
|
||||
//
|
||||
// This function is synchronous, and will block until the resource has been refreshed.
|
||||
func (c *controller) Refresh(ctx context.Context, u string) error {
|
||||
reply := make(chan backendResponse[struct{}], 1)
|
||||
req := refreshRequest{
|
||||
reply: reply,
|
||||
u: u,
|
||||
}
|
||||
|
||||
if _, err := sendBackend[refreshRequest, struct{}](ctx, c.incoming, req, reply); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
package httprc
|
||||
|
||||
import "errors"
|
||||
|
||||
var errResourceAlreadyExists = errors.New(`resource already exists`)
|
||||
|
||||
func ErrResourceAlreadyExists() error {
|
||||
return errResourceAlreadyExists
|
||||
}
|
||||
|
||||
var errAlreadyRunning = errors.New(`client is already running`)
|
||||
|
||||
func ErrAlreadyRunning() error {
|
||||
return errAlreadyRunning
|
||||
}
|
||||
|
||||
var errResourceNotFound = errors.New(`resource not found`)
|
||||
|
||||
func ErrResourceNotFound() error {
|
||||
return errResourceNotFound
|
||||
}
|
||||
|
||||
var errTransformerRequired = errors.New(`transformer is required`)
|
||||
|
||||
func ErrTransformerRequired() error {
|
||||
return errTransformerRequired
|
||||
}
|
||||
|
||||
var errURLCannotBeEmpty = errors.New(`URL cannot be empty`)
|
||||
|
||||
func ErrURLCannotBeEmpty() error {
|
||||
return errURLCannotBeEmpty
|
||||
}
|
||||
|
||||
var errUnexpectedStatusCode = errors.New(`unexpected status code`)
|
||||
|
||||
func ErrUnexpectedStatusCode() error {
|
||||
return errUnexpectedStatusCode
|
||||
}
|
||||
|
||||
var errTransformerFailed = errors.New(`failed to transform response body`)
|
||||
|
||||
func ErrTransformerFailed() error {
|
||||
return errTransformerFailed
|
||||
}
|
||||
|
||||
var errRecoveredFromPanic = errors.New(`recovered from panic`)
|
||||
|
||||
func ErrRecoveredFromPanic() error {
|
||||
return errRecoveredFromPanic
|
||||
}
|
||||
|
||||
var errBlockedByWhitelist = errors.New(`blocked by whitelist`)
|
||||
|
||||
func ErrBlockedByWhitelist() error {
|
||||
return errBlockedByWhitelist
|
||||
}
|
||||
+59
@@ -0,0 +1,59 @@
|
||||
package errsink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type Interface interface {
|
||||
Put(context.Context, error)
|
||||
}
|
||||
|
||||
// Nop is an ErrorSink that does nothing. It does not require
|
||||
// any initialization, so the zero value can be used.
|
||||
type Nop struct{}
|
||||
|
||||
// NewNop returns a new NopErrorSink object. The constructor
|
||||
// is provided for consistency.
|
||||
func NewNop() Interface {
|
||||
return Nop{}
|
||||
}
|
||||
|
||||
// Put for NopErrorSink does nothing.
|
||||
func (Nop) Put(context.Context, error) {}
|
||||
|
||||
type SlogLogger interface {
|
||||
Log(context.Context, slog.Level, string, ...any)
|
||||
}
|
||||
|
||||
type slogSink struct {
|
||||
logger SlogLogger
|
||||
}
|
||||
|
||||
// NewSlog returns a new ErrorSink that logs errors using the provided slog.Logger
|
||||
func NewSlog(l SlogLogger) Interface {
|
||||
return &slogSink{
|
||||
logger: l,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *slogSink) Put(ctx context.Context, v error) {
|
||||
s.logger.Log(ctx, slog.LevelError, v.Error())
|
||||
}
|
||||
|
||||
// FuncSink is an ErrorSink that calls a function with the error.
|
||||
type FuncSink struct {
|
||||
fn func(context.Context, error)
|
||||
}
|
||||
|
||||
// NewFunc returns a new FuncSink that calls the provided function with errors.
|
||||
func NewFunc(fn func(context.Context, error)) Interface {
|
||||
return &FuncSink{fn: fn}
|
||||
}
|
||||
|
||||
// Put calls the function with the error.
|
||||
func (f *FuncSink) Put(ctx context.Context, err error) {
|
||||
if f.fn != nil {
|
||||
f.fn(ctx, err)
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/httprc/v3/errsink"
|
||||
"github.com/lestrrat-go/httprc/v3/tracesink"
|
||||
)
|
||||
|
||||
// Buffer size constants
|
||||
const (
|
||||
// ReadBufferSize is the default buffer size for reading HTTP responses (10MB)
|
||||
ReadBufferSize = 1024 * 1024 * 10
|
||||
// MaxBufferSize is the maximum allowed buffer size (1GB)
|
||||
MaxBufferSize = 1024 * 1024 * 1000
|
||||
)
|
||||
|
||||
// Client worker constants
|
||||
const (
|
||||
// DefaultWorkers is the default number of worker goroutines
|
||||
DefaultWorkers = 5
|
||||
)
|
||||
|
||||
// Interval constants
|
||||
const (
|
||||
// DefaultMaxInterval is the default maximum interval between fetches (30 days)
|
||||
DefaultMaxInterval = 24 * time.Hour * 30
|
||||
// DefaultMinInterval is the default minimum interval between fetches (15 minutes)
|
||||
DefaultMinInterval = 15 * time.Minute
|
||||
// oneDay is used internally for time calculations
|
||||
oneDay = 24 * time.Hour
|
||||
)
|
||||
|
||||
// utility to round up intervals to the nearest second
|
||||
func roundupToSeconds(d time.Duration) time.Duration {
|
||||
if diff := d % time.Second; diff > 0 {
|
||||
return d + time.Second - diff
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
// ErrorSink is an interface that abstracts a sink for errors.
|
||||
type ErrorSink = errsink.Interface
|
||||
|
||||
type TraceSink = tracesink.Interface
|
||||
|
||||
// HTTPClient is an interface that abstracts a "net/http".Client, so that
|
||||
// users can provide their own implementation of the HTTP client, if need be.
|
||||
type HTTPClient interface {
|
||||
Do(*http.Request) (*http.Response, error)
|
||||
}
|
||||
|
||||
// Transformer is used to convert the body of an HTTP response into an appropriate
|
||||
// object of type T.
|
||||
type Transformer[T any] interface {
|
||||
Transform(context.Context, *http.Response) (T, error)
|
||||
}
|
||||
|
||||
// TransformFunc is a function type that implements the Transformer interface.
|
||||
type TransformFunc[T any] func(context.Context, *http.Response) (T, error)
|
||||
|
||||
func (f TransformFunc[T]) Transform(ctx context.Context, res *http.Response) (T, error) {
|
||||
return f(ctx, res)
|
||||
}
|
||||
|
||||
// Resource is a single resource that can be retrieved via HTTP, and (possibly) transformed
|
||||
// into an arbitrary object type.
|
||||
//
|
||||
// Realistically, there is no need for third-parties to implement this interface. This exists
|
||||
// to provide a way to aggregate `httprc.ResourceBase` objects with different specialized types
|
||||
// into a single collection.
|
||||
//
|
||||
// See ResourceBase for details
|
||||
type Resource interface { //nolint:interfacebloat
|
||||
Get(any) error
|
||||
Next() time.Time
|
||||
SetNext(time.Time)
|
||||
URL() string
|
||||
Sync(context.Context) error
|
||||
ConstantInterval() time.Duration
|
||||
MaxInterval() time.Duration
|
||||
SetMaxInterval(time.Duration)
|
||||
MinInterval() time.Duration
|
||||
SetMinInterval(time.Duration)
|
||||
IsBusy() bool
|
||||
SetBusy(bool)
|
||||
Ready(context.Context) error
|
||||
}
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/option/v2"
|
||||
)
|
||||
|
||||
type NewClientOption interface {
|
||||
option.Interface
|
||||
newClientOption()
|
||||
}
|
||||
|
||||
type newClientOption struct {
|
||||
option.Interface
|
||||
}
|
||||
|
||||
func (newClientOption) newClientOption() {}
|
||||
|
||||
type identWorkers struct{}
|
||||
|
||||
// WithWorkers specifies the number of concurrent workers to use for the client.
|
||||
// If n is less than or equal to 0, the client will use a single worker.
|
||||
func WithWorkers(n int) NewClientOption {
|
||||
return newClientOption{option.New(identWorkers{}, n)}
|
||||
}
|
||||
|
||||
type identErrorSink struct{}
|
||||
|
||||
// WithErrorSink specifies the error sink to use for the client.
|
||||
// If not specified, the client will use a NopErrorSink.
|
||||
func WithErrorSink(sink ErrorSink) NewClientOption {
|
||||
return newClientOption{option.New(identErrorSink{}, sink)}
|
||||
}
|
||||
|
||||
type identTraceSink struct{}
|
||||
|
||||
// WithTraceSink specifies the trace sink to use for the client.
|
||||
// If not specified, the client will use a NopTraceSink.
|
||||
func WithTraceSink(sink TraceSink) NewClientOption {
|
||||
return newClientOption{option.New(identTraceSink{}, sink)}
|
||||
}
|
||||
|
||||
type identWhitelist struct{}
|
||||
|
||||
// WithWhitelist specifies the whitelist to use for the client.
|
||||
// If not specified, the client will use a BlockAllWhitelist.
|
||||
func WithWhitelist(wl Whitelist) NewClientOption {
|
||||
return newClientOption{option.New(identWhitelist{}, wl)}
|
||||
}
|
||||
|
||||
type NewResourceOption interface {
|
||||
option.Interface
|
||||
newResourceOption()
|
||||
}
|
||||
|
||||
type newResourceOption struct {
|
||||
option.Interface
|
||||
}
|
||||
|
||||
func (newResourceOption) newResourceOption() {}
|
||||
|
||||
type NewClientResourceOption interface {
|
||||
option.Interface
|
||||
newResourceOption()
|
||||
newClientOption()
|
||||
}
|
||||
|
||||
type newClientResourceOption struct {
|
||||
option.Interface
|
||||
}
|
||||
|
||||
func (newClientResourceOption) newResourceOption() {}
|
||||
func (newClientResourceOption) newClientOption() {}
|
||||
|
||||
type identHTTPClient struct{}
|
||||
|
||||
// WithHTTPClient specifies the HTTP client to use for the client.
|
||||
// If not specified, the client will use http.DefaultClient.
|
||||
//
|
||||
// This option can be passed to NewClient or NewResource.
|
||||
func WithHTTPClient(cl HTTPClient) NewClientResourceOption {
|
||||
return newClientResourceOption{option.New(identHTTPClient{}, cl)}
|
||||
}
|
||||
|
||||
type identMinimumInterval struct{}
|
||||
|
||||
// WithMinInterval specifies the minimum interval between fetches.
|
||||
//
|
||||
// This option affects the dynamic calculation of the interval between fetches.
|
||||
// If the value calculated from the http.Response is less than the this value,
|
||||
// the client will use this value instead.
|
||||
func WithMinInterval(d time.Duration) NewResourceOption {
|
||||
return newResourceOption{option.New(identMinimumInterval{}, d)}
|
||||
}
|
||||
|
||||
type identMaximumInterval struct{}
|
||||
|
||||
// WithMaxInterval specifies the maximum interval between fetches.
|
||||
//
|
||||
// This option affects the dynamic calculation of the interval between fetches.
|
||||
// If the value calculated from the http.Response is greater than the this value,
|
||||
// the client will use this value instead.
|
||||
func WithMaxInterval(d time.Duration) NewResourceOption {
|
||||
return newResourceOption{option.New(identMaximumInterval{}, d)}
|
||||
}
|
||||
|
||||
type identConstantInterval struct{}
|
||||
|
||||
// WithConstantInterval specifies the interval between fetches. When you
|
||||
// specify this option, the client will fetch the resource at the specified
|
||||
// intervals, regardless of the response's Cache-Control or Expires headers.
|
||||
//
|
||||
// By default this option is disabled.
|
||||
func WithConstantInterval(d time.Duration) NewResourceOption {
|
||||
return newResourceOption{option.New(identConstantInterval{}, d)}
|
||||
}
|
||||
|
||||
type AddOption interface {
|
||||
option.Interface
|
||||
newAddOption()
|
||||
}
|
||||
|
||||
type addOption struct {
|
||||
option.Interface
|
||||
}
|
||||
|
||||
func (addOption) newAddOption() {}
|
||||
|
||||
type identWaitReady struct{}
|
||||
|
||||
// WithWaitReady specifies whether the client should wait for the resource to be
|
||||
// ready before returning from the Add method.
|
||||
//
|
||||
// By default, the client will wait for the resource to be ready before returning.
|
||||
// If you specify this option with a value of false, the client will not wait for
|
||||
// the resource to be fully registered, which is usually not what you want.
|
||||
// This option exists to accommodate for cases where you for some reason want to
|
||||
// add a resource to the controller, but want to do something else before
|
||||
// you wait for it. Make sure to call `r.Ready()` later on to ensure that
|
||||
// the resource is ready before you try to access it.
|
||||
func WithWaitReady(b bool) AddOption {
|
||||
return addOption{option.New(identWaitReady{}, b)}
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
package proxysink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type Backend[T any] interface {
|
||||
Put(context.Context, T)
|
||||
}
|
||||
|
||||
// Proxy is used to send values through a channel. This is used to
|
||||
// serialize calls to underlying sinks.
|
||||
type Proxy[T any] struct {
|
||||
mu *sync.Mutex
|
||||
cancel context.CancelFunc
|
||||
ch chan T
|
||||
cond *sync.Cond
|
||||
pending []T
|
||||
backend Backend[T]
|
||||
closed bool
|
||||
}
|
||||
|
||||
func New[T any](b Backend[T]) *Proxy[T] {
|
||||
mu := &sync.Mutex{}
|
||||
return &Proxy[T]{
|
||||
ch: make(chan T, 1),
|
||||
mu: mu,
|
||||
cond: sync.NewCond(mu),
|
||||
backend: b,
|
||||
cancel: func() {},
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy[T]) Run(ctx context.Context) {
|
||||
defer p.cond.Broadcast()
|
||||
|
||||
p.mu.Lock()
|
||||
ctx, cancel := context.WithCancel(ctx)
|
||||
p.cancel = cancel
|
||||
p.mu.Unlock()
|
||||
|
||||
go p.controlloop(ctx)
|
||||
go p.flushloop(ctx)
|
||||
|
||||
<-ctx.Done()
|
||||
}
|
||||
|
||||
func (p *Proxy[T]) controlloop(ctx context.Context) {
|
||||
defer p.cond.Broadcast()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case r := <-p.ch:
|
||||
p.mu.Lock()
|
||||
p.pending = append(p.pending, r)
|
||||
p.mu.Unlock()
|
||||
}
|
||||
p.cond.Broadcast()
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy[T]) flushloop(ctx context.Context) {
|
||||
const defaultPendingSize = 10
|
||||
pending := make([]T, defaultPendingSize)
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.mu.Lock()
|
||||
if len(p.pending) <= 0 {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
default:
|
||||
}
|
||||
|
||||
p.mu.Lock()
|
||||
for len(p.pending) <= 0 {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
p.mu.Unlock()
|
||||
return
|
||||
default:
|
||||
p.cond.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// extract all pending values, and clear the shared slice
|
||||
if cap(pending) < len(p.pending) {
|
||||
pending = make([]T, len(p.pending))
|
||||
} else {
|
||||
pending = pending[:len(p.pending)]
|
||||
}
|
||||
copy(pending, p.pending)
|
||||
if cap(p.pending) > defaultPendingSize {
|
||||
p.pending = make([]T, 0, defaultPendingSize)
|
||||
} else {
|
||||
p.pending = p.pending[:0]
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
for _, v := range pending {
|
||||
// send to sink serially
|
||||
p.backend.Put(ctx, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy[T]) Put(ctx context.Context, v T) {
|
||||
p.mu.Lock()
|
||||
if p.closed {
|
||||
p.mu.Unlock()
|
||||
return
|
||||
}
|
||||
p.mu.Unlock()
|
||||
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case p.ch <- v:
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func (p *Proxy[T]) Close() {
|
||||
p.mu.Lock()
|
||||
defer p.mu.Unlock()
|
||||
|
||||
if !p.closed {
|
||||
p.closed = true
|
||||
}
|
||||
p.cancel()
|
||||
p.cond.Broadcast()
|
||||
}
|
||||
+366
@@ -0,0 +1,366 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/lestrrat-go/blackmagic"
|
||||
"github.com/lestrrat-go/httpcc"
|
||||
"github.com/lestrrat-go/httprc/v3/tracesink"
|
||||
)
|
||||
|
||||
// ResourceBase is a generic Resource type
|
||||
type ResourceBase[T any] struct {
|
||||
u string
|
||||
ready chan struct{} // closed when the resource is ready (i.e. after first successful fetch)
|
||||
once sync.Once
|
||||
httpcl HTTPClient
|
||||
t Transformer[T]
|
||||
r atomic.Value
|
||||
next atomic.Value
|
||||
interval time.Duration
|
||||
minInterval atomic.Int64
|
||||
maxInterval atomic.Int64
|
||||
busy atomic.Bool
|
||||
}
|
||||
|
||||
// NewResource creates a new Resource object which after fetching the
|
||||
// resource from the URL, will transform the response body using the
|
||||
// provided Transformer to an object of type T.
|
||||
//
|
||||
// This function will return an error if the URL is not a valid URL
|
||||
// (i.e. it cannot be parsed by url.Parse), or if the transformer is nil.
|
||||
func NewResource[T any](s string, transformer Transformer[T], options ...NewResourceOption) (*ResourceBase[T], error) {
|
||||
var httpcl HTTPClient
|
||||
var interval time.Duration
|
||||
minInterval := DefaultMinInterval
|
||||
maxInterval := DefaultMaxInterval
|
||||
for _, option := range options {
|
||||
switch option.Ident() {
|
||||
case identHTTPClient{}:
|
||||
if err := option.Value(&httpcl); err != nil {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: failed to parse HTTPClient option: %w`, err)
|
||||
}
|
||||
case identMinimumInterval{}:
|
||||
if err := option.Value(&minInterval); err != nil {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: failed to parse MinimumInterval option: %w`, err)
|
||||
}
|
||||
case identMaximumInterval{}:
|
||||
if err := option.Value(&maxInterval); err != nil {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: failed to parse MaximumInterval option: %w`, err)
|
||||
}
|
||||
case identConstantInterval{}:
|
||||
if err := option.Value(&interval); err != nil {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: failed to parse ConstantInterval option: %w`, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if transformer == nil {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: %w`, errTransformerRequired)
|
||||
}
|
||||
|
||||
if s == "" {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: %w`, errURLCannotBeEmpty)
|
||||
}
|
||||
|
||||
if _, err := url.Parse(s); err != nil {
|
||||
return nil, fmt.Errorf(`httprc.NewResource: %w`, err)
|
||||
}
|
||||
r := &ResourceBase[T]{
|
||||
u: s,
|
||||
httpcl: httpcl,
|
||||
t: transformer,
|
||||
interval: interval,
|
||||
ready: make(chan struct{}),
|
||||
}
|
||||
if httpcl != nil {
|
||||
r.httpcl = httpcl
|
||||
}
|
||||
r.minInterval.Store(int64(minInterval))
|
||||
r.maxInterval.Store(int64(maxInterval))
|
||||
r.SetNext(time.Unix(0, 0)) // initially, it should be fetched immediately
|
||||
return r, nil
|
||||
}
|
||||
|
||||
// URL returns the URL of the resource.
|
||||
func (r *ResourceBase[T]) URL() string {
|
||||
return r.u
|
||||
}
|
||||
|
||||
// Ready returns an empty error when the resource is ready. If the context
|
||||
// is canceled before the resource is ready, it will return the error from
|
||||
// the context.
|
||||
func (r *ResourceBase[T]) Ready(ctx context.Context) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-r.ready:
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// Get assigns the value of the resource to the provided pointer.
|
||||
// If using the `httprc.ResourceBase[T]` type directly, you can use the `Resource()`
|
||||
// method to get the resource directly.
|
||||
//
|
||||
// This method exists because parametric types cannot be assigned to a single object type
|
||||
// that return different return values of the specialized type. i.e. for resources
|
||||
// `ResourceBase[A]` and `ResourceBase[B]`, we cannot have a single interface that can
|
||||
// be assigned to the same interface type `X` that expects a `Resource()` method that
|
||||
// returns `A` or `B` depending on the type of the resource. When accessing the
|
||||
// resource through the `httprc.Resource` interface, use this method to obtain the
|
||||
// stored value.
|
||||
func (r *ResourceBase[T]) Get(dst any) error {
|
||||
return blackmagic.AssignIfCompatible(dst, r.Resource())
|
||||
}
|
||||
|
||||
// Resource returns the last fetched resource. If the resource has not been
|
||||
// fetched yet, this will return the zero value of type T.
|
||||
//
|
||||
// If you would rather wait until the resource is fetched, you can use the
|
||||
// `Ready()` method to wait until the resource is ready (i.e. fetched at least once).
|
||||
func (r *ResourceBase[T]) Resource() T {
|
||||
v := r.r.Load()
|
||||
switch v := v.(type) {
|
||||
case T:
|
||||
return v
|
||||
default:
|
||||
var zero T
|
||||
return zero
|
||||
}
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) Next() time.Time {
|
||||
//nolint:forcetypeassert
|
||||
return r.next.Load().(time.Time)
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) SetNext(v time.Time) {
|
||||
r.next.Store(v)
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) ConstantInterval() time.Duration {
|
||||
return r.interval
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) MaxInterval() time.Duration {
|
||||
return time.Duration(r.maxInterval.Load())
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) MinInterval() time.Duration {
|
||||
return time.Duration(r.minInterval.Load())
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) SetMaxInterval(v time.Duration) {
|
||||
r.maxInterval.Store(int64(v))
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) SetMinInterval(v time.Duration) {
|
||||
r.minInterval.Store(int64(v))
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) SetBusy(v bool) {
|
||||
r.busy.Store(v)
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) IsBusy() bool {
|
||||
return r.busy.Load()
|
||||
}
|
||||
|
||||
// limitedBody is a wrapper around an io.Reader that will only read up to
|
||||
// MaxBufferSize bytes. This is provided to prevent the user from accidentally
|
||||
// reading a huge response body into memory
|
||||
type limitedBody struct {
|
||||
rdr io.Reader
|
||||
close func() error
|
||||
}
|
||||
|
||||
func (l *limitedBody) Read(p []byte) (n int, err error) {
|
||||
return l.rdr.Read(p)
|
||||
}
|
||||
|
||||
func (l *limitedBody) Close() error {
|
||||
return l.close()
|
||||
}
|
||||
|
||||
type traceSinkKey struct{}
|
||||
|
||||
func withTraceSink(ctx context.Context, sink TraceSink) context.Context {
|
||||
return context.WithValue(ctx, traceSinkKey{}, sink)
|
||||
}
|
||||
|
||||
func traceSinkFromContext(ctx context.Context) TraceSink {
|
||||
if v := ctx.Value(traceSinkKey{}); v != nil {
|
||||
//nolint:forcetypeassert
|
||||
return v.(TraceSink)
|
||||
}
|
||||
return tracesink.Nop{}
|
||||
}
|
||||
|
||||
type httpClientKey struct{}
|
||||
|
||||
func withHTTPClient(ctx context.Context, cl HTTPClient) context.Context {
|
||||
return context.WithValue(ctx, httpClientKey{}, cl)
|
||||
}
|
||||
|
||||
func httpClientFromContext(ctx context.Context) HTTPClient {
|
||||
if v := ctx.Value(httpClientKey{}); v != nil {
|
||||
//nolint:forcetypeassert
|
||||
return v.(HTTPClient)
|
||||
}
|
||||
return http.DefaultClient
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) Sync(ctx context.Context) error {
|
||||
traceSink := traceSinkFromContext(ctx)
|
||||
httpcl := r.httpcl
|
||||
if httpcl == nil {
|
||||
httpcl = httpClientFromContext(ctx)
|
||||
}
|
||||
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.u, nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`httprc.Resource.Sync: failed to create request: %w`, err)
|
||||
}
|
||||
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: fetching %q", r.u))
|
||||
res, err := httpcl.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`httprc.Resource.Sync: failed to execute HTTP request: %w`, err)
|
||||
}
|
||||
defer res.Body.Close()
|
||||
|
||||
next := r.calculateNextRefreshTime(ctx, res)
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: next refresh time for %q is %v", r.u, next))
|
||||
r.SetNext(next)
|
||||
|
||||
if res.StatusCode != http.StatusOK {
|
||||
return fmt.Errorf(`httprc.Resource.Sync: %w (status code=%d, url=%q)`, errUnexpectedStatusCode, res.StatusCode, r.u)
|
||||
}
|
||||
|
||||
// replace the body of the response with a limited reader that
|
||||
// will only read up to MaxBufferSize bytes
|
||||
res.Body = &limitedBody{
|
||||
rdr: &io.LimitedReader{R: res.Body, N: MaxBufferSize},
|
||||
close: res.Body.Close,
|
||||
}
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: transforming %q", r.u))
|
||||
v, err := r.transform(ctx, res)
|
||||
if err != nil {
|
||||
return fmt.Errorf(`httprc.Resource.Sync: %w: %w`, errTransformerFailed, err)
|
||||
}
|
||||
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: storing new value for %q", r.u))
|
||||
r.r.Store(v)
|
||||
r.once.Do(func() { close(r.ready) })
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: stored value for %q", r.u))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) transform(ctx context.Context, res *http.Response) (ret T, gerr error) {
|
||||
// Protect the call to Transform with a defer/recover block, so that even
|
||||
// if the Transform method panics, we can recover from it and return an error
|
||||
defer func() {
|
||||
if recovered := recover(); recovered != nil {
|
||||
gerr = fmt.Errorf(`httprc.Resource.transform: %w: %v`, errRecoveredFromPanic, recovered)
|
||||
}
|
||||
}()
|
||||
return r.t.Transform(ctx, res)
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) determineNextFetchInterval(ctx context.Context, name string, fromHeader, minValue, maxValue time.Duration) time.Duration {
|
||||
traceSink := traceSinkFromContext(ctx)
|
||||
|
||||
if fromHeader > maxValue {
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s %s > maximum interval, using maximum interval %s", r.URL(), name, maxValue))
|
||||
return maxValue
|
||||
}
|
||||
|
||||
if fromHeader < minValue {
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s %s < minimum interval, using minimum interval %s", r.URL(), name, minValue))
|
||||
return minValue
|
||||
}
|
||||
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Using %s (%s)", r.URL(), name, fromHeader))
|
||||
return fromHeader
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) calculateNextRefreshTime(ctx context.Context, res *http.Response) time.Time {
|
||||
traceSink := traceSinkFromContext(ctx)
|
||||
now := time.Now()
|
||||
|
||||
// If constant interval is set, use that regardless of what the
|
||||
// response headers say.
|
||||
if interval := r.ConstantInterval(); interval > 0 {
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Explicit interval set, using value %s", r.URL(), interval))
|
||||
return now.Add(interval)
|
||||
}
|
||||
|
||||
if interval := r.extractCacheControlMaxAge(ctx, res); interval > 0 {
|
||||
return now.Add(interval)
|
||||
}
|
||||
|
||||
if interval := r.extractExpiresInterval(ctx, res); interval > 0 {
|
||||
return now.Add(interval)
|
||||
}
|
||||
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s No cache-control/expires headers found, using minimum interval", r.URL()))
|
||||
return now.Add(r.MinInterval())
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) extractCacheControlMaxAge(ctx context.Context, res *http.Response) time.Duration {
|
||||
traceSink := traceSinkFromContext(ctx)
|
||||
|
||||
v := res.Header.Get(`Cache-Control`)
|
||||
if v == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
dir, err := httpcc.ParseResponse(v)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
maxAge, ok := dir.MaxAge()
|
||||
if !ok {
|
||||
return 0
|
||||
}
|
||||
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Cache-Control=max-age directive set (%d)", r.URL(), maxAge))
|
||||
return r.determineNextFetchInterval(
|
||||
ctx,
|
||||
"max-age",
|
||||
time.Duration(maxAge)*time.Second,
|
||||
r.MinInterval(),
|
||||
r.MaxInterval(),
|
||||
)
|
||||
}
|
||||
|
||||
func (r *ResourceBase[T]) extractExpiresInterval(ctx context.Context, res *http.Response) time.Duration {
|
||||
traceSink := traceSinkFromContext(ctx)
|
||||
|
||||
v := res.Header.Get(`Expires`)
|
||||
if v == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
expires, err := http.ParseTime(v)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
traceSink.Put(ctx, fmt.Sprintf("httprc.Resource.Sync: %s Expires header set (%s)", r.URL(), expires))
|
||||
return r.determineNextFetchInterval(
|
||||
ctx,
|
||||
"expires",
|
||||
time.Until(expires),
|
||||
r.MinInterval(),
|
||||
r.MaxInterval(),
|
||||
)
|
||||
}
|
||||
+52
@@ -0,0 +1,52 @@
|
||||
package tracesink
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
)
|
||||
|
||||
type Interface interface {
|
||||
Put(context.Context, string)
|
||||
}
|
||||
|
||||
// Nop is an ErrorSink that does nothing. It does not require
|
||||
// any initialization, so the zero value can be used.
|
||||
type Nop struct{}
|
||||
|
||||
// NewNop returns a new NopTraceSink object. The constructor
|
||||
// is provided for consistency.
|
||||
func NewNop() Interface {
|
||||
return Nop{}
|
||||
}
|
||||
|
||||
// Put for NopTraceSink does nothing.
|
||||
func (Nop) Put(context.Context, string) {}
|
||||
|
||||
type slogSink struct {
|
||||
level slog.Level
|
||||
logger SlogLogger
|
||||
}
|
||||
|
||||
type SlogLogger interface {
|
||||
Log(context.Context, slog.Level, string, ...any)
|
||||
}
|
||||
|
||||
// NewSlog returns a new ErrorSink that logs errors using the provided slog.Logger
|
||||
func NewSlog(l SlogLogger) Interface {
|
||||
return &slogSink{
|
||||
level: slog.LevelInfo,
|
||||
logger: l,
|
||||
}
|
||||
}
|
||||
|
||||
func (s *slogSink) Put(ctx context.Context, v string) {
|
||||
s.logger.Log(ctx, s.level, v)
|
||||
}
|
||||
|
||||
// Func is a TraceSink that calls a function with the trace message.
|
||||
type Func func(context.Context, string)
|
||||
|
||||
// Put calls the function with the trace message.
|
||||
func (f Func) Put(ctx context.Context, msg string) {
|
||||
f(ctx, msg)
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type bytesTransformer struct{}
|
||||
|
||||
// BytesTransformer returns a Transformer that reads the entire response body
|
||||
// as a byte slice. This is the default Transformer used by httprc.Client
|
||||
func BytesTransformer() Transformer[[]byte] {
|
||||
return bytesTransformer{}
|
||||
}
|
||||
|
||||
func (bytesTransformer) Transform(_ context.Context, res *http.Response) ([]byte, error) {
|
||||
return io.ReadAll(res.Body)
|
||||
}
|
||||
|
||||
type jsonTransformer[T any] struct{}
|
||||
|
||||
// JSONTransformer returns a Transformer that decodes the response body as JSON
|
||||
// into the provided type T.
|
||||
func JSONTransformer[T any]() Transformer[T] {
|
||||
return jsonTransformer[T]{}
|
||||
}
|
||||
|
||||
func (jsonTransformer[T]) Transform(_ context.Context, res *http.Response) (T, error) {
|
||||
var v T
|
||||
if err := json.NewDecoder(res.Body).Decode(&v); err != nil {
|
||||
var zero T
|
||||
return zero, err
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
+113
@@ -0,0 +1,113 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"regexp"
|
||||
"sync"
|
||||
)
|
||||
|
||||
// Whitelist is an interface that allows you to determine if a given URL is allowed
|
||||
// or not. Implementations of this interface can be used to restrict the URLs that
|
||||
// the client can access.
|
||||
//
|
||||
// By default all URLs are allowed, but this may not be ideal in production environments
|
||||
// for security reasons.
|
||||
//
|
||||
// This exists because you might use this module to store resources provided by
|
||||
// user of your application, in which case you cannot necessarily trust that the
|
||||
// URLs are safe.
|
||||
//
|
||||
// You will HAVE to provide some sort of whitelist.
|
||||
type Whitelist interface {
|
||||
IsAllowed(string) bool
|
||||
}
|
||||
|
||||
// WhitelistFunc is a function type that implements the Whitelist interface.
|
||||
type WhitelistFunc func(string) bool
|
||||
|
||||
func (f WhitelistFunc) IsAllowed(u string) bool { return f(u) }
|
||||
|
||||
// BlockAllWhitelist is a Whitelist implementation that blocks all URLs.
|
||||
type BlockAllWhitelist struct{}
|
||||
|
||||
// NewBlockAllWhitelist creates a new BlockAllWhitelist instance. It is safe to
|
||||
// use the zero value of this type; this constructor is provided for consistency.
|
||||
func NewBlockAllWhitelist() BlockAllWhitelist { return BlockAllWhitelist{} }
|
||||
|
||||
func (BlockAllWhitelist) IsAllowed(_ string) bool { return false }
|
||||
|
||||
// InsecureWhitelist is a Whitelist implementation that allows all URLs. Be careful
|
||||
// when using this in your production code: make sure you do not blindly register
|
||||
// URLs from untrusted sources.
|
||||
type InsecureWhitelist struct{}
|
||||
|
||||
// NewInsecureWhitelist creates a new InsecureWhitelist instance. It is safe to
|
||||
// use the zero value of this type; this constructor is provided for consistency.
|
||||
func NewInsecureWhitelist() InsecureWhitelist { return InsecureWhitelist{} }
|
||||
|
||||
func (InsecureWhitelist) IsAllowed(_ string) bool { return true }
|
||||
|
||||
// RegexpWhitelist is a jwk.Whitelist object comprised of a list of *regexp.Regexp
|
||||
// objects. All entries in the list are tried until one matches. If none of the
|
||||
// *regexp.Regexp objects match, then the URL is deemed unallowed.
|
||||
type RegexpWhitelist struct {
|
||||
mu sync.RWMutex
|
||||
patterns []*regexp.Regexp
|
||||
}
|
||||
|
||||
// NewRegexpWhitelist creates a new RegexpWhitelist instance. It is safe to use the
|
||||
// zero value of this type; this constructor is provided for consistency.
|
||||
func NewRegexpWhitelist() *RegexpWhitelist {
|
||||
return &RegexpWhitelist{}
|
||||
}
|
||||
|
||||
// Add adds a new regular expression to the list of expressions to match against.
|
||||
func (w *RegexpWhitelist) Add(pat *regexp.Regexp) *RegexpWhitelist {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.patterns = append(w.patterns, pat)
|
||||
return w
|
||||
}
|
||||
|
||||
// IsAllowed returns true if any of the patterns in the whitelist
|
||||
// returns true.
|
||||
func (w *RegexpWhitelist) IsAllowed(u string) bool {
|
||||
w.mu.RLock()
|
||||
patterns := w.patterns
|
||||
w.mu.RUnlock()
|
||||
for _, pat := range patterns {
|
||||
if pat.MatchString(u) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// MapWhitelist is a jwk.Whitelist object comprised of a map of strings.
|
||||
// If the URL exists in the map, then the URL is allowed to be fetched.
|
||||
type MapWhitelist interface {
|
||||
Whitelist
|
||||
Add(string) MapWhitelist
|
||||
}
|
||||
|
||||
type mapWhitelist struct {
|
||||
mu sync.RWMutex
|
||||
store map[string]struct{}
|
||||
}
|
||||
|
||||
func NewMapWhitelist() MapWhitelist {
|
||||
return &mapWhitelist{store: make(map[string]struct{})}
|
||||
}
|
||||
|
||||
func (w *mapWhitelist) Add(pat string) MapWhitelist {
|
||||
w.mu.Lock()
|
||||
defer w.mu.Unlock()
|
||||
w.store[pat] = struct{}{}
|
||||
return w
|
||||
}
|
||||
|
||||
func (w *mapWhitelist) IsAllowed(u string) bool {
|
||||
w.mu.RLock()
|
||||
_, b := w.store[u]
|
||||
w.mu.RUnlock()
|
||||
return b
|
||||
}
|
||||
+62
@@ -0,0 +1,62 @@
|
||||
package httprc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type worker struct {
|
||||
httpcl HTTPClient
|
||||
incoming chan any
|
||||
next <-chan Resource
|
||||
nextsync <-chan synchronousRequest
|
||||
errSink ErrorSink
|
||||
traceSink TraceSink
|
||||
}
|
||||
|
||||
func (w worker) Run(ctx context.Context, readywg *sync.WaitGroup, donewg *sync.WaitGroup) {
|
||||
w.traceSink.Put(ctx, "httprc worker: START worker loop")
|
||||
defer w.traceSink.Put(ctx, "httprc worker: END worker loop")
|
||||
defer donewg.Done()
|
||||
ctx = withTraceSink(ctx, w.traceSink)
|
||||
ctx = withHTTPClient(ctx, w.httpcl)
|
||||
|
||||
readywg.Done()
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
w.traceSink.Put(ctx, "httprc worker: stopping worker loop")
|
||||
return
|
||||
case r := <-w.next:
|
||||
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: syncing %q (async)", r.URL()))
|
||||
if err := r.Sync(ctx); err != nil {
|
||||
w.errSink.Put(ctx, err)
|
||||
}
|
||||
r.SetBusy(false)
|
||||
|
||||
w.sendAdjustIntervalRequest(ctx, r)
|
||||
case sr := <-w.nextsync:
|
||||
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: syncing %q (synchronous)", sr.resource.URL()))
|
||||
if err := sr.resource.Sync(ctx); err != nil {
|
||||
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: FAILED to sync %q (synchronous): %s", sr.resource.URL(), err))
|
||||
sendReply(ctx, sr.reply, struct{}{}, err)
|
||||
sr.resource.SetBusy(false)
|
||||
return
|
||||
}
|
||||
w.traceSink.Put(ctx, fmt.Sprintf("httprc worker: SUCCESS syncing %q (synchronous)", sr.resource.URL()))
|
||||
sr.resource.SetBusy(false)
|
||||
sendReply(ctx, sr.reply, struct{}{}, nil)
|
||||
w.sendAdjustIntervalRequest(ctx, sr.resource)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w worker) sendAdjustIntervalRequest(ctx context.Context, r Resource) {
|
||||
w.traceSink.Put(ctx, "httprc worker: Sending interval adjustment request for "+r.URL())
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
case w.incoming <- adjustIntervalRequest{resource: r}:
|
||||
}
|
||||
w.traceSink.Put(ctx, "httprc worker: Sent interval adjustment request for "+r.URL())
|
||||
}
|
||||
Reference in New Issue
Block a user