bump dependencies

Signed-off-by: Jörn Friedrich Dreyer <jfd@butonic.de>
This commit is contained in:
Jörn Friedrich Dreyer
2024-10-18 17:35:30 +02:00
parent 290477b494
commit ca113f5751
522 changed files with 93151 additions and 33036 deletions
+1 -1
View File
@@ -9,7 +9,7 @@ See the [releases page](https://github.com/BurntSushi/toml/releases) for a
changelog; this information is also in the git tag annotations (e.g. `git show
v0.4.0`).
This library requires Go 1.13 or newer; add it to your go.mod with:
This library requires Go 1.18 or newer; add it to your go.mod with:
% go get github.com/BurntSushi/toml@latest
+52 -41
View File
@@ -6,7 +6,7 @@ import (
"encoding/json"
"fmt"
"io"
"io/ioutil"
"io/fs"
"math"
"os"
"reflect"
@@ -18,13 +18,13 @@ import (
// Unmarshaler is the interface implemented by objects that can unmarshal a
// TOML description of themselves.
type Unmarshaler interface {
UnmarshalTOML(interface{}) error
UnmarshalTOML(any) error
}
// Unmarshal decodes the contents of data in TOML format into a pointer v.
//
// See [Decoder] for a description of the decoding process.
func Unmarshal(data []byte, v interface{}) error {
func Unmarshal(data []byte, v any) error {
_, err := NewDecoder(bytes.NewReader(data)).Decode(v)
return err
}
@@ -32,12 +32,12 @@ func Unmarshal(data []byte, v interface{}) error {
// Decode the TOML data in to the pointer v.
//
// See [Decoder] for a description of the decoding process.
func Decode(data string, v interface{}) (MetaData, error) {
func Decode(data string, v any) (MetaData, error) {
return NewDecoder(strings.NewReader(data)).Decode(v)
}
// DecodeFile reads the contents of a file and decodes it with [Decode].
func DecodeFile(path string, v interface{}) (MetaData, error) {
func DecodeFile(path string, v any) (MetaData, error) {
fp, err := os.Open(path)
if err != nil {
return MetaData{}, err
@@ -46,6 +46,17 @@ func DecodeFile(path string, v interface{}) (MetaData, error) {
return NewDecoder(fp).Decode(v)
}
// DecodeFS reads the contents of a file from [fs.FS] and decodes it with
// [Decode].
func DecodeFS(fsys fs.FS, path string, v any) (MetaData, error) {
fp, err := fsys.Open(path)
if err != nil {
return MetaData{}, err
}
defer fp.Close()
return NewDecoder(fp).Decode(v)
}
// Primitive is a TOML value that hasn't been decoded into a Go value.
//
// This type can be used for any value, which will cause decoding to be delayed.
@@ -58,7 +69,7 @@ func DecodeFile(path string, v interface{}) (MetaData, error) {
// overhead of reflection. They can be useful when you don't know the exact type
// of TOML data until runtime.
type Primitive struct {
undecoded interface{}
undecoded any
context Key
}
@@ -122,7 +133,7 @@ var (
)
// Decode TOML data in to the pointer `v`.
func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
func (dec *Decoder) Decode(v any) (MetaData, error) {
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr {
s := "%q"
@@ -136,8 +147,8 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
return MetaData{}, fmt.Errorf("toml: cannot decode to nil value of %q", reflect.TypeOf(v))
}
// Check if this is a supported type: struct, map, interface{}, or something
// that implements UnmarshalTOML or UnmarshalText.
// Check if this is a supported type: struct, map, any, or something that
// implements UnmarshalTOML or UnmarshalText.
rv = indirect(rv)
rt := rv.Type()
if rv.Kind() != reflect.Struct && rv.Kind() != reflect.Map &&
@@ -148,7 +159,7 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
// TODO: parser should read from io.Reader? Or at the very least, make it
// read from []byte rather than string
data, err := ioutil.ReadAll(dec.r)
data, err := io.ReadAll(dec.r)
if err != nil {
return MetaData{}, err
}
@@ -179,7 +190,7 @@ func (dec *Decoder) Decode(v interface{}) (MetaData, error) {
// will only reflect keys that were decoded. Namely, any keys hidden behind a
// Primitive will be considered undecoded. Executing this method will update the
// undecoded keys in the meta data. (See the example.)
func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error {
func (md *MetaData) PrimitiveDecode(primValue Primitive, v any) error {
md.context = primValue.context
defer func() { md.context = nil }()
return md.unify(primValue.undecoded, rvalue(v))
@@ -190,7 +201,7 @@ func (md *MetaData) PrimitiveDecode(primValue Primitive, v interface{}) error {
//
// Any type mismatch produces an error. Finding a type that we don't know
// how to handle produces an unsupported type error.
func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
func (md *MetaData) unify(data any, rv reflect.Value) error {
// Special case. Look for a `Primitive` value.
// TODO: #76 would make this superfluous after implemented.
if rv.Type() == primitiveType {
@@ -207,7 +218,11 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
rvi := rv.Interface()
if v, ok := rvi.(Unmarshaler); ok {
return v.UnmarshalTOML(data)
err := v.UnmarshalTOML(data)
if err != nil {
return md.parseErr(err)
}
return nil
}
if v, ok := rvi.(encoding.TextUnmarshaler); ok {
return md.unifyText(data, v)
@@ -227,14 +242,6 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
return md.unifyInt(data, rv)
}
switch k {
case reflect.Ptr:
elem := reflect.New(rv.Type().Elem())
err := md.unify(data, reflect.Indirect(elem))
if err != nil {
return err
}
rv.Set(elem)
return nil
case reflect.Struct:
return md.unifyStruct(data, rv)
case reflect.Map:
@@ -258,14 +265,13 @@ func (md *MetaData) unify(data interface{}, rv reflect.Value) error {
return md.e("unsupported type %s", rv.Kind())
}
func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error {
tmap, ok := mapping.(map[string]interface{})
func (md *MetaData) unifyStruct(mapping any, rv reflect.Value) error {
tmap, ok := mapping.(map[string]any)
if !ok {
if mapping == nil {
return nil
}
return md.e("type mismatch for %s: expected table but found %T",
rv.Type().String(), mapping)
return md.e("type mismatch for %s: expected table but found %s", rv.Type().String(), fmtType(mapping))
}
for key, datum := range tmap {
@@ -304,14 +310,14 @@ func (md *MetaData) unifyStruct(mapping interface{}, rv reflect.Value) error {
return nil
}
func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error {
func (md *MetaData) unifyMap(mapping any, rv reflect.Value) error {
keyType := rv.Type().Key().Kind()
if keyType != reflect.String && keyType != reflect.Interface {
return fmt.Errorf("toml: cannot decode to a map with non-string key type (%s in %q)",
keyType, rv.Type())
}
tmap, ok := mapping.(map[string]interface{})
tmap, ok := mapping.(map[string]any)
if !ok {
if tmap == nil {
return nil
@@ -347,7 +353,7 @@ func (md *MetaData) unifyMap(mapping interface{}, rv reflect.Value) error {
return nil
}
func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifyArray(data any, rv reflect.Value) error {
datav := reflect.ValueOf(data)
if datav.Kind() != reflect.Slice {
if !datav.IsValid() {
@@ -361,7 +367,7 @@ func (md *MetaData) unifyArray(data interface{}, rv reflect.Value) error {
return md.unifySliceArray(datav, rv)
}
func (md *MetaData) unifySlice(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifySlice(data any, rv reflect.Value) error {
datav := reflect.ValueOf(data)
if datav.Kind() != reflect.Slice {
if !datav.IsValid() {
@@ -388,7 +394,7 @@ func (md *MetaData) unifySliceArray(data, rv reflect.Value) error {
return nil
}
func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifyString(data any, rv reflect.Value) error {
_, ok := rv.Interface().(json.Number)
if ok {
if i, ok := data.(int64); ok {
@@ -408,7 +414,7 @@ func (md *MetaData) unifyString(data interface{}, rv reflect.Value) error {
return md.badtype("string", data)
}
func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifyFloat64(data any, rv reflect.Value) error {
rvk := rv.Kind()
if num, ok := data.(float64); ok {
@@ -429,7 +435,7 @@ func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error {
if num, ok := data.(int64); ok {
if (rvk == reflect.Float32 && (num < -maxSafeFloat32Int || num > maxSafeFloat32Int)) ||
(rvk == reflect.Float64 && (num < -maxSafeFloat64Int || num > maxSafeFloat64Int)) {
return md.parseErr(errParseRange{i: num, size: rvk.String()})
return md.parseErr(errUnsafeFloat{i: num, size: rvk.String()})
}
rv.SetFloat(float64(num))
return nil
@@ -438,7 +444,7 @@ func (md *MetaData) unifyFloat64(data interface{}, rv reflect.Value) error {
return md.badtype("float", data)
}
func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifyInt(data any, rv reflect.Value) error {
_, ok := rv.Interface().(time.Duration)
if ok {
// Parse as string duration, and fall back to regular integer parsing
@@ -481,7 +487,7 @@ func (md *MetaData) unifyInt(data interface{}, rv reflect.Value) error {
return nil
}
func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifyBool(data any, rv reflect.Value) error {
if b, ok := data.(bool); ok {
rv.SetBool(b)
return nil
@@ -489,12 +495,12 @@ func (md *MetaData) unifyBool(data interface{}, rv reflect.Value) error {
return md.badtype("boolean", data)
}
func (md *MetaData) unifyAnything(data interface{}, rv reflect.Value) error {
func (md *MetaData) unifyAnything(data any, rv reflect.Value) error {
rv.Set(reflect.ValueOf(data))
return nil
}
func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) error {
func (md *MetaData) unifyText(data any, v encoding.TextUnmarshaler) error {
var s string
switch sdata := data.(type) {
case Marshaler:
@@ -523,13 +529,13 @@ func (md *MetaData) unifyText(data interface{}, v encoding.TextUnmarshaler) erro
return md.badtype("primitive (string-like)", data)
}
if err := v.UnmarshalText([]byte(s)); err != nil {
return err
return md.parseErr(err)
}
return nil
}
func (md *MetaData) badtype(dst string, data interface{}) error {
return md.e("incompatible types: TOML value has type %T; destination has type %s", data, dst)
func (md *MetaData) badtype(dst string, data any) error {
return md.e("incompatible types: TOML value has type %s; destination has type %s", fmtType(data), dst)
}
func (md *MetaData) parseErr(err error) error {
@@ -543,7 +549,7 @@ func (md *MetaData) parseErr(err error) error {
}
}
func (md *MetaData) e(format string, args ...interface{}) error {
func (md *MetaData) e(format string, args ...any) error {
f := "toml: "
if len(md.context) > 0 {
f = fmt.Sprintf("toml: (last key %q): ", md.context)
@@ -556,7 +562,7 @@ func (md *MetaData) e(format string, args ...interface{}) error {
}
// rvalue returns a reflect.Value of `v`. All pointers are resolved.
func rvalue(v interface{}) reflect.Value {
func rvalue(v any) reflect.Value {
return indirect(reflect.ValueOf(v))
}
@@ -600,3 +606,8 @@ func isUnifiable(rv reflect.Value) bool {
}
return false
}
// fmt %T with "interface {}" replaced with "any", which is far more readable.
func fmtType(t any) string {
return strings.ReplaceAll(fmt.Sprintf("%T", t), "interface {}", "any")
}
-19
View File
@@ -1,19 +0,0 @@
//go:build go1.16
// +build go1.16
package toml
import (
"io/fs"
)
// DecodeFS reads the contents of a file from [fs.FS] and decodes it with
// [Decode].
func DecodeFS(fsys fs.FS, path string, v interface{}) (MetaData, error) {
fp, err := fsys.Open(path)
if err != nil {
return MetaData{}, err
}
defer fp.Close()
return NewDecoder(fp).Decode(v)
}
+9 -9
View File
@@ -15,15 +15,15 @@ type TextMarshaler encoding.TextMarshaler
// Deprecated: use encoding.TextUnmarshaler
type TextUnmarshaler encoding.TextUnmarshaler
// PrimitiveDecode is an alias for MetaData.PrimitiveDecode().
//
// Deprecated: use MetaData.PrimitiveDecode.
func PrimitiveDecode(primValue Primitive, v interface{}) error {
md := MetaData{decoded: make(map[string]struct{})}
return md.unify(primValue.undecoded, rvalue(v))
}
// DecodeReader is an alias for NewDecoder(r).Decode(v).
//
// Deprecated: use NewDecoder(reader).Decode(&value).
func DecodeReader(r io.Reader, v interface{}) (MetaData, error) { return NewDecoder(r).Decode(v) }
func DecodeReader(r io.Reader, v any) (MetaData, error) { return NewDecoder(r).Decode(v) }
// PrimitiveDecode is an alias for MetaData.PrimitiveDecode().
//
// Deprecated: use MetaData.PrimitiveDecode.
func PrimitiveDecode(primValue Primitive, v any) error {
md := MetaData{decoded: make(map[string]struct{})}
return md.unify(primValue.undecoded, rvalue(v))
}
-3
View File
@@ -2,9 +2,6 @@
//
// This package supports TOML v1.0.0, as specified at https://toml.io
//
// There is also support for delaying decoding with the Primitive type, and
// querying the set of keys in a TOML document with the MetaData type.
//
// The github.com/BurntSushi/toml/cmd/tomlv package implements a TOML validator,
// and can be used to verify if TOML document is valid. It can also be used to
// print the type of each key.
+32 -13
View File
@@ -2,6 +2,7 @@ package toml
import (
"bufio"
"bytes"
"encoding"
"encoding/json"
"errors"
@@ -76,6 +77,17 @@ type Marshaler interface {
MarshalTOML() ([]byte, error)
}
// Marshal returns a TOML representation of the Go value.
//
// See [Encoder] for a description of the encoding process.
func Marshal(v any) ([]byte, error) {
buff := new(bytes.Buffer)
if err := NewEncoder(buff).Encode(v); err != nil {
return nil, err
}
return buff.Bytes(), nil
}
// Encoder encodes a Go to a TOML document.
//
// The mapping between Go values and TOML values should be precisely the same as
@@ -115,26 +127,21 @@ type Marshaler interface {
// NOTE: only exported keys are encoded due to the use of reflection. Unexported
// keys are silently discarded.
type Encoder struct {
// String to use for a single indentation level; default is two spaces.
Indent string
Indent string // string for a single indentation level; default is two spaces.
hasWritten bool // written any output to w yet?
w *bufio.Writer
hasWritten bool // written any output to w yet?
}
// NewEncoder create a new Encoder.
func NewEncoder(w io.Writer) *Encoder {
return &Encoder{
w: bufio.NewWriter(w),
Indent: " ",
}
return &Encoder{w: bufio.NewWriter(w), Indent: " "}
}
// Encode writes a TOML representation of the Go value to the [Encoder]'s writer.
//
// An error is returned if the value given cannot be encoded to a valid TOML
// document.
func (enc *Encoder) Encode(v interface{}) error {
func (enc *Encoder) Encode(v any) error {
rv := eindirect(reflect.ValueOf(v))
err := enc.safeEncode(Key([]string{}), rv)
if err != nil {
@@ -280,18 +287,30 @@ func (enc *Encoder) eElement(rv reflect.Value) {
case reflect.Float32:
f := rv.Float()
if math.IsNaN(f) {
if math.Signbit(f) {
enc.wf("-")
}
enc.wf("nan")
} else if math.IsInf(f, 0) {
enc.wf("%cinf", map[bool]byte{true: '-', false: '+'}[math.Signbit(f)])
if math.Signbit(f) {
enc.wf("-")
}
enc.wf("inf")
} else {
enc.wf(floatAddDecimal(strconv.FormatFloat(f, 'f', -1, 32)))
}
case reflect.Float64:
f := rv.Float()
if math.IsNaN(f) {
if math.Signbit(f) {
enc.wf("-")
}
enc.wf("nan")
} else if math.IsInf(f, 0) {
enc.wf("%cinf", map[bool]byte{true: '-', false: '+'}[math.Signbit(f)])
if math.Signbit(f) {
enc.wf("-")
}
enc.wf("inf")
} else {
enc.wf(floatAddDecimal(strconv.FormatFloat(f, 'f', -1, 64)))
}
@@ -304,7 +323,7 @@ func (enc *Encoder) eElement(rv reflect.Value) {
case reflect.Interface:
enc.eElement(rv.Elem())
default:
encPanic(fmt.Errorf("unexpected type: %T", rv.Interface()))
encPanic(fmt.Errorf("unexpected type: %s", fmtType(rv.Interface())))
}
}
@@ -712,7 +731,7 @@ func (enc *Encoder) writeKeyValue(key Key, val reflect.Value, inline bool) {
}
}
func (enc *Encoder) wf(format string, v ...interface{}) {
func (enc *Encoder) wf(format string, v ...any) {
_, err := fmt.Fprintf(enc.w, format, v...)
if err != nil {
encPanic(err)
+94 -17
View File
@@ -114,13 +114,22 @@ func (pe ParseError) ErrorWithPosition() string {
msg, pe.Position.Line, col, col+pe.Position.Len)
}
if pe.Position.Line > 2 {
fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-2, lines[pe.Position.Line-3])
fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-2, expandTab(lines[pe.Position.Line-3]))
}
if pe.Position.Line > 1 {
fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-1, lines[pe.Position.Line-2])
fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line-1, expandTab(lines[pe.Position.Line-2]))
}
fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line, lines[pe.Position.Line-1])
fmt.Fprintf(b, "% 10s%s%s\n", "", strings.Repeat(" ", col), strings.Repeat("^", pe.Position.Len))
/// Expand tabs, so that the ^^^s are at the correct position, but leave
/// "column 10-13" intact. Adjusting this to the visual column would be
/// better, but we don't know the tabsize of the user in their editor, which
/// can be 8, 4, 2, or something else. We can't know. So leaving it as the
/// character index is probably the "most correct".
expanded := expandTab(lines[pe.Position.Line-1])
diff := len(expanded) - len(lines[pe.Position.Line-1])
fmt.Fprintf(b, "% 7d | %s\n", pe.Position.Line, expanded)
fmt.Fprintf(b, "% 10s%s%s\n", "", strings.Repeat(" ", col+diff), strings.Repeat("^", pe.Position.Len))
return b.String()
}
@@ -159,17 +168,47 @@ func (pe ParseError) column(lines []string) int {
return col
}
func expandTab(s string) string {
var (
b strings.Builder
l int
fill = func(n int) string {
b := make([]byte, n)
for i := range b {
b[i] = ' '
}
return string(b)
}
)
b.Grow(len(s))
for _, r := range s {
switch r {
case '\t':
tw := 8 - l%8
b.WriteString(fill(tw))
l += tw
default:
b.WriteRune(r)
l += 1
}
}
return b.String()
}
type (
errLexControl struct{ r rune }
errLexEscape struct{ r rune }
errLexUTF8 struct{ b byte }
errLexInvalidNum struct{ v string }
errLexInvalidDate struct{ v string }
errParseDate struct{ v string }
errLexInlineTableNL struct{}
errLexStringNL struct{}
errParseRange struct {
i interface{} // int or float
size string // "int64", "uint16", etc.
i any // int or float
size string // "int64", "uint16", etc.
}
errUnsafeFloat struct {
i interface{} // float32 or float64
size string // "float32" or "float64"
}
errParseDuration struct{ d string }
)
@@ -183,18 +222,20 @@ func (e errLexEscape) Error() string { return fmt.Sprintf(`invalid escape
func (e errLexEscape) Usage() string { return usageEscape }
func (e errLexUTF8) Error() string { return fmt.Sprintf("invalid UTF-8 byte: 0x%02x", e.b) }
func (e errLexUTF8) Usage() string { return "" }
func (e errLexInvalidNum) Error() string { return fmt.Sprintf("invalid number: %q", e.v) }
func (e errLexInvalidNum) Usage() string { return "" }
func (e errLexInvalidDate) Error() string { return fmt.Sprintf("invalid date: %q", e.v) }
func (e errLexInvalidDate) Usage() string { return "" }
func (e errParseDate) Error() string { return fmt.Sprintf("invalid datetime: %q", e.v) }
func (e errParseDate) Usage() string { return usageDate }
func (e errLexInlineTableNL) Error() string { return "newlines not allowed within inline tables" }
func (e errLexInlineTableNL) Usage() string { return usageInlineNewline }
func (e errLexStringNL) Error() string { return "strings cannot contain newlines" }
func (e errLexStringNL) Usage() string { return usageStringNewline }
func (e errParseRange) Error() string { return fmt.Sprintf("%v is out of range for %s", e.i, e.size) }
func (e errParseRange) Usage() string { return usageIntOverflow }
func (e errParseDuration) Error() string { return fmt.Sprintf("invalid duration: %q", e.d) }
func (e errParseDuration) Usage() string { return usageDuration }
func (e errUnsafeFloat) Error() string {
return fmt.Sprintf("%v is out of the safe %s range", e.i, e.size)
}
func (e errUnsafeFloat) Usage() string { return usageUnsafeFloat }
func (e errParseDuration) Error() string { return fmt.Sprintf("invalid duration: %q", e.d) }
func (e errParseDuration) Usage() string { return usageDuration }
const usageEscape = `
A '\' inside a "-delimited string is interpreted as an escape character.
@@ -251,19 +292,35 @@ bug in the program that uses too small of an integer.
The maximum and minimum values are:
size │ lowest │ highest
───────┼────────────────┼──────────
───────┼────────────────┼──────────────
int8 │ -128 │ 127
int16 │ -32,768 │ 32,767
int32 │ -2,147,483,648 │ 2,147,483,647
int64 │ -9.2 × 10¹⁷ │ 9.2 × 10¹⁷
uint8 │ 0 │ 255
uint16 │ 0 │ 65535
uint32 │ 0 │ 4294967295
uint16 │ 0 │ 65,535
uint32 │ 0 │ 4,294,967,295
uint64 │ 0 │ 1.8 × 10¹⁸
int refers to int32 on 32-bit systems and int64 on 64-bit systems.
`
const usageUnsafeFloat = `
This number is outside of the "safe" range for floating point numbers; whole
(non-fractional) numbers outside the below range can not always be represented
accurately in a float, leading to some loss of accuracy.
Explicitly mark a number as a fractional unit by adding ".0", which will incur
some loss of accuracy; for example:
f = 2_000_000_000.0
Accuracy ranges:
float32 = 16,777,215
float64 = 9,007,199,254,740,991
`
const usageDuration = `
A duration must be as "number<unit>", without any spaces. Valid units are:
@@ -277,3 +334,23 @@ A duration must be as "number<unit>", without any spaces. Valid units are:
You can combine multiple units; for example "5m10s" for 5 minutes and 10
seconds.
`
const usageDate = `
A TOML datetime must be in one of the following formats:
2006-01-02T15:04:05Z07:00 Date and time, with timezone.
2006-01-02T15:04:05 Date and time, but without timezone.
2006-01-02 Date without a time or timezone.
15:04:05 Just a time, without any timezone.
Seconds may optionally have a fraction, up to nanosecond precision:
15:04:05.123
15:04:05.856018510
`
// TOML 1.1:
// The seconds part in times is optional, and may be omitted:
// 2006-01-02T15:04Z07:00
// 2006-01-02T15:04
// 15:04
+24 -26
View File
@@ -17,6 +17,7 @@ const (
itemEOF
itemText
itemString
itemStringEsc
itemRawString
itemMultilineString
itemRawMultilineString
@@ -53,6 +54,7 @@ type lexer struct {
state stateFn
items chan item
tomlNext bool
esc bool
// Allow for backing up up to 4 runes. This is necessary because TOML
// contains 3-rune tokens (""" and ''').
@@ -164,7 +166,7 @@ func (lx *lexer) next() (r rune) {
}
r, w := utf8.DecodeRuneInString(lx.input[lx.pos:])
if r == utf8.RuneError {
if r == utf8.RuneError && w == 1 {
lx.error(errLexUTF8{lx.input[lx.pos]})
return utf8.RuneError
}
@@ -270,7 +272,7 @@ func (lx *lexer) errorPos(start, length int, err error) stateFn {
}
// errorf is like error, and creates a new error.
func (lx *lexer) errorf(format string, values ...interface{}) stateFn {
func (lx *lexer) errorf(format string, values ...any) stateFn {
if lx.atEOF {
pos := lx.getPos()
pos.Line--
@@ -333,9 +335,7 @@ func lexTopEnd(lx *lexer) stateFn {
lx.emit(itemEOF)
return nil
}
return lx.errorf(
"expected a top-level item to end with a newline, comment, or EOF, but got %q instead",
r)
return lx.errorf("expected a top-level item to end with a newline, comment, or EOF, but got %q instead", r)
}
// lexTable lexes the beginning of a table. Namely, it makes sure that
@@ -698,7 +698,12 @@ func lexString(lx *lexer) stateFn {
return lexStringEscape
case r == '"':
lx.backup()
lx.emit(itemString)
if lx.esc {
lx.esc = false
lx.emit(itemStringEsc)
} else {
lx.emit(itemString)
}
lx.next()
lx.ignore()
return lx.pop()
@@ -748,6 +753,7 @@ func lexMultilineString(lx *lexer) stateFn {
lx.backup() /// backup: don't include the """ in the item.
lx.backup()
lx.backup()
lx.esc = false
lx.emit(itemMultilineString)
lx.next() /// Read over ''' again and discard it.
lx.next()
@@ -837,6 +843,7 @@ func lexMultilineStringEscape(lx *lexer) stateFn {
}
func lexStringEscape(lx *lexer) stateFn {
lx.esc = true
r := lx.next()
switch r {
case 'e':
@@ -879,10 +886,8 @@ func lexHexEscape(lx *lexer) stateFn {
var r rune
for i := 0; i < 2; i++ {
r = lx.next()
if !isHexadecimal(r) {
return lx.errorf(
`expected two hexadecimal digits after '\x', but got %q instead`,
lx.current())
if !isHex(r) {
return lx.errorf(`expected two hexadecimal digits after '\x', but got %q instead`, lx.current())
}
}
return lx.pop()
@@ -892,10 +897,8 @@ func lexShortUnicodeEscape(lx *lexer) stateFn {
var r rune
for i := 0; i < 4; i++ {
r = lx.next()
if !isHexadecimal(r) {
return lx.errorf(
`expected four hexadecimal digits after '\u', but got %q instead`,
lx.current())
if !isHex(r) {
return lx.errorf(`expected four hexadecimal digits after '\u', but got %q instead`, lx.current())
}
}
return lx.pop()
@@ -905,10 +908,8 @@ func lexLongUnicodeEscape(lx *lexer) stateFn {
var r rune
for i := 0; i < 8; i++ {
r = lx.next()
if !isHexadecimal(r) {
return lx.errorf(
`expected eight hexadecimal digits after '\U', but got %q instead`,
lx.current())
if !isHex(r) {
return lx.errorf(`expected eight hexadecimal digits after '\U', but got %q instead`, lx.current())
}
}
return lx.pop()
@@ -975,7 +976,7 @@ func lexDatetime(lx *lexer) stateFn {
// lexHexInteger consumes a hexadecimal integer after seeing the '0x' prefix.
func lexHexInteger(lx *lexer) stateFn {
r := lx.next()
if isHexadecimal(r) {
if isHex(r) {
return lexHexInteger
}
switch r {
@@ -1109,7 +1110,7 @@ func lexBaseNumberOrDate(lx *lexer) stateFn {
return lexOctalInteger
case 'x':
r = lx.peek()
if !isHexadecimal(r) {
if !isHex(r) {
lx.errorf("not a hexidecimal number: '%s%c'", lx.current(), r)
}
return lexHexInteger
@@ -1207,7 +1208,7 @@ func (itype itemType) String() string {
return "EOF"
case itemText:
return "Text"
case itemString, itemRawString, itemMultilineString, itemRawMultilineString:
case itemString, itemStringEsc, itemRawString, itemMultilineString, itemRawMultilineString:
return "String"
case itemBool:
return "Bool"
@@ -1240,7 +1241,7 @@ func (itype itemType) String() string {
}
func (item item) String() string {
return fmt.Sprintf("(%s, %s)", item.typ.String(), item.val)
return fmt.Sprintf("(%s, %s)", item.typ, item.val)
}
func isWhitespace(r rune) bool { return r == '\t' || r == ' ' }
@@ -1256,10 +1257,7 @@ func isControl(r rune) bool { // Control characters except \t, \r, \n
func isDigit(r rune) bool { return r >= '0' && r <= '9' }
func isBinary(r rune) bool { return r == '0' || r == '1' }
func isOctal(r rune) bool { return r >= '0' && r <= '7' }
func isHexadecimal(r rune) bool {
return (r >= '0' && r <= '9') || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F')
}
func isHex(r rune) bool { return (r >= '0' && r <= '9') || (r|0x20 >= 'a' && r|0x20 <= 'f') }
func isBareKeyChar(r rune, tomlNext bool) bool {
if tomlNext {
return (r >= 'A' && r <= 'Z') ||
+38 -11
View File
@@ -13,7 +13,7 @@ type MetaData struct {
context Key // Used only during decoding.
keyInfo map[string]keyInfo
mapping map[string]interface{}
mapping map[string]any
keys []Key
decoded map[string]struct{}
data []byte // Input file; for errors.
@@ -31,12 +31,12 @@ func (md *MetaData) IsDefined(key ...string) bool {
}
var (
hash map[string]interface{}
hash map[string]any
ok bool
hashOrVal interface{} = md.mapping
hashOrVal any = md.mapping
)
for _, k := range key {
if hash, ok = hashOrVal.(map[string]interface{}); !ok {
if hash, ok = hashOrVal.(map[string]any); !ok {
return false
}
if hashOrVal, ok = hash[k]; !ok {
@@ -94,28 +94,55 @@ func (md *MetaData) Undecoded() []Key {
type Key []string
func (k Key) String() string {
ss := make([]string, len(k))
for i := range k {
ss[i] = k.maybeQuoted(i)
// This is called quite often, so it's a bit funky to make it faster.
var b strings.Builder
b.Grow(len(k) * 25)
outer:
for i, kk := range k {
if i > 0 {
b.WriteByte('.')
}
if kk == "" {
b.WriteString(`""`)
} else {
for _, r := range kk {
// "Inline" isBareKeyChar
if !((r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-') {
b.WriteByte('"')
b.WriteString(dblQuotedReplacer.Replace(kk))
b.WriteByte('"')
continue outer
}
}
b.WriteString(kk)
}
}
return strings.Join(ss, ".")
return b.String()
}
func (k Key) maybeQuoted(i int) string {
if k[i] == "" {
return `""`
}
for _, c := range k[i] {
if !isBareKeyChar(c, false) {
return `"` + dblQuotedReplacer.Replace(k[i]) + `"`
for _, r := range k[i] {
if (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') || r == '_' || r == '-' {
continue
}
return `"` + dblQuotedReplacer.Replace(k[i]) + `"`
}
return k[i]
}
// Like append(), but only increase the cap by 1.
func (k Key) add(piece string) Key {
if cap(k) > len(k) {
return append(k, piece)
}
newKey := make(Key, len(k)+1)
copy(newKey, k)
newKey[len(k)] = piece
return newKey
}
func (k Key) parent() Key { return k[:len(k)-1] } // all except the last piece.
func (k Key) last() string { return k[len(k)-1] } // last piece of this key.
+151 -118
View File
@@ -2,6 +2,7 @@ package toml
import (
"fmt"
"math"
"os"
"strconv"
"strings"
@@ -20,9 +21,9 @@ type parser struct {
ordered []Key // List of keys in the order that they appear in the TOML data.
keyInfo map[string]keyInfo // Map keyname → info about the TOML key.
mapping map[string]interface{} // Map keyname → key value.
implicits map[string]struct{} // Record implicit keys (e.g. "key.group.names").
keyInfo map[string]keyInfo // Map keyname → info about the TOML key.
mapping map[string]any // Map keyname → key value.
implicits map[string]struct{} // Record implicit keys (e.g. "key.group.names").
}
type keyInfo struct {
@@ -49,6 +50,7 @@ func parse(data string) (p *parser, err error) {
// it anyway.
if strings.HasPrefix(data, "\xff\xfe") || strings.HasPrefix(data, "\xfe\xff") { // UTF-16
data = data[2:]
//lint:ignore S1017 https://github.com/dominikh/go-tools/issues/1447
} else if strings.HasPrefix(data, "\xef\xbb\xbf") { // UTF-8
data = data[3:]
}
@@ -71,7 +73,7 @@ func parse(data string) (p *parser, err error) {
p = &parser{
keyInfo: make(map[string]keyInfo),
mapping: make(map[string]interface{}),
mapping: make(map[string]any),
lx: lex(data, tomlNext),
ordered: make([]Key, 0),
implicits: make(map[string]struct{}),
@@ -97,7 +99,7 @@ func (p *parser) panicErr(it item, err error) {
})
}
func (p *parser) panicItemf(it item, format string, v ...interface{}) {
func (p *parser) panicItemf(it item, format string, v ...any) {
panic(ParseError{
Message: fmt.Sprintf(format, v...),
Position: it.pos,
@@ -106,7 +108,7 @@ func (p *parser) panicItemf(it item, format string, v ...interface{}) {
})
}
func (p *parser) panicf(format string, v ...interface{}) {
func (p *parser) panicf(format string, v ...any) {
panic(ParseError{
Message: fmt.Sprintf(format, v...),
Position: p.pos,
@@ -139,7 +141,7 @@ func (p *parser) nextPos() item {
return it
}
func (p *parser) bug(format string, v ...interface{}) {
func (p *parser) bug(format string, v ...any) {
panic(fmt.Sprintf("BUG: "+format+"\n\n", v...))
}
@@ -194,11 +196,11 @@ func (p *parser) topLevel(item item) {
p.assertEqual(itemKeyEnd, k.typ)
/// The current key is the last part.
p.currentKey = key[len(key)-1]
p.currentKey = key.last()
/// All the other parts (if any) are the context; need to set each part
/// as implicit.
context := key[:len(key)-1]
context := key.parent()
for i := range context {
p.addImplicitContext(append(p.context, context[i:i+1]...))
}
@@ -207,7 +209,8 @@ func (p *parser) topLevel(item item) {
/// Set value.
vItem := p.next()
val, typ := p.value(vItem, false)
p.set(p.currentKey, val, typ, vItem.pos)
p.setValue(p.currentKey, val)
p.setType(p.currentKey, typ, vItem.pos)
/// Remove the context we added (preserving any context from [tbl] lines).
p.context = outerContext
@@ -222,7 +225,7 @@ func (p *parser) keyString(it item) string {
switch it.typ {
case itemText:
return it.val
case itemString, itemMultilineString,
case itemString, itemStringEsc, itemMultilineString,
itemRawString, itemRawMultilineString:
s, _ := p.value(it, false)
return s.(string)
@@ -239,9 +242,11 @@ var datetimeRepl = strings.NewReplacer(
// value translates an expected value from the lexer into a Go value wrapped
// as an empty interface.
func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) {
func (p *parser) value(it item, parentIsArray bool) (any, tomlType) {
switch it.typ {
case itemString:
return it.val, p.typeOfPrimitive(it)
case itemStringEsc:
return p.replaceEscapes(it, it.val), p.typeOfPrimitive(it)
case itemMultilineString:
return p.replaceEscapes(it, p.stripEscapedNewlines(stripFirstNewline(it.val))), p.typeOfPrimitive(it)
@@ -274,7 +279,7 @@ func (p *parser) value(it item, parentIsArray bool) (interface{}, tomlType) {
panic("unreachable")
}
func (p *parser) valueInteger(it item) (interface{}, tomlType) {
func (p *parser) valueInteger(it item) (any, tomlType) {
if !numUnderscoresOK(it.val) {
p.panicItemf(it, "Invalid integer %q: underscores must be surrounded by digits", it.val)
}
@@ -298,7 +303,7 @@ func (p *parser) valueInteger(it item) (interface{}, tomlType) {
return num, p.typeOfPrimitive(it)
}
func (p *parser) valueFloat(it item) (interface{}, tomlType) {
func (p *parser) valueFloat(it item) (any, tomlType) {
parts := strings.FieldsFunc(it.val, func(r rune) bool {
switch r {
case '.', 'e', 'E':
@@ -322,7 +327,9 @@ func (p *parser) valueFloat(it item) (interface{}, tomlType) {
p.panicItemf(it, "Invalid float %q: '.' must be followed by one or more digits", it.val)
}
val := strings.Replace(it.val, "_", "", -1)
if val == "+nan" || val == "-nan" { // Go doesn't support this, but TOML spec does.
signbit := false
if val == "+nan" || val == "-nan" {
signbit = val == "-nan"
val = "nan"
}
num, err := strconv.ParseFloat(val, 64)
@@ -333,6 +340,9 @@ func (p *parser) valueFloat(it item) (interface{}, tomlType) {
p.panicItemf(it, "Invalid float value: %q", it.val)
}
}
if signbit {
num = math.Copysign(num, -1)
}
return num, p.typeOfPrimitive(it)
}
@@ -352,7 +362,7 @@ var dtTypes = []struct {
{"15:04", internal.LocalTime, true},
}
func (p *parser) valueDatetime(it item) (interface{}, tomlType) {
func (p *parser) valueDatetime(it item) (any, tomlType) {
it.val = datetimeRepl.Replace(it.val)
var (
t time.Time
@@ -365,26 +375,44 @@ func (p *parser) valueDatetime(it item) (interface{}, tomlType) {
}
t, err = time.ParseInLocation(dt.fmt, it.val, dt.zone)
if err == nil {
if missingLeadingZero(it.val, dt.fmt) {
p.panicErr(it, errParseDate{it.val})
}
ok = true
break
}
}
if !ok {
p.panicItemf(it, "Invalid TOML Datetime: %q.", it.val)
p.panicErr(it, errParseDate{it.val})
}
return t, p.typeOfPrimitive(it)
}
func (p *parser) valueArray(it item) (interface{}, tomlType) {
// Go's time.Parse() will accept numbers without a leading zero; there isn't any
// way to require it. https://github.com/golang/go/issues/29911
//
// Depend on the fact that the separators (- and :) should always be at the same
// location.
func missingLeadingZero(d, l string) bool {
for i, c := range []byte(l) {
if c == '.' || c == 'Z' {
return false
}
if (c < '0' || c > '9') && d[i] != c {
return true
}
}
return false
}
func (p *parser) valueArray(it item) (any, tomlType) {
p.setType(p.currentKey, tomlArray, it.pos)
var (
types []tomlType
// Initialize to a non-nil empty slice. This makes it consistent with
// how S = [] decodes into a non-nil slice inside something like struct
// { S []string }. See #338
array = []interface{}{}
// Initialize to a non-nil slice to make it consistent with how S = []
// decodes into a non-nil slice inside something like struct { S
// []string }. See #338
array = make([]any, 0, 2)
)
for it = p.next(); it.typ != itemArrayEnd; it = p.next() {
if it.typ == itemCommentStart {
@@ -394,21 +422,20 @@ func (p *parser) valueArray(it item) (interface{}, tomlType) {
val, typ := p.value(it, true)
array = append(array, val)
types = append(types, typ)
// XXX: types isn't used here, we need it to record the accurate type
// XXX: type isn't used here, we need it to record the accurate type
// information.
//
// Not entirely sure how to best store this; could use "key[0]",
// "key[1]" notation, or maybe store it on the Array type?
_ = types
_ = typ
}
return array, tomlArray
}
func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tomlType) {
func (p *parser) valueInlineTable(it item, parentIsArray bool) (any, tomlType) {
var (
hash = make(map[string]interface{})
topHash = make(map[string]any)
outerContext = p.context
outerKey = p.currentKey
)
@@ -436,11 +463,11 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom
p.assertEqual(itemKeyEnd, k.typ)
/// The current key is the last part.
p.currentKey = key[len(key)-1]
p.currentKey = key.last()
/// All the other parts (if any) are the context; need to set each part
/// as implicit.
context := key[:len(key)-1]
context := key.parent()
for i := range context {
p.addImplicitContext(append(p.context, context[i:i+1]...))
}
@@ -448,7 +475,21 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom
/// Set the value.
val, typ := p.value(p.next(), false)
p.set(p.currentKey, val, typ, it.pos)
p.setValue(p.currentKey, val)
p.setType(p.currentKey, typ, it.pos)
hash := topHash
for _, c := range context {
h, ok := hash[c]
if !ok {
h = make(map[string]any)
hash[c] = h
}
hash, ok = h.(map[string]any)
if !ok {
p.panicf("%q is not a table", p.context)
}
}
hash[p.currentKey] = val
/// Restore context.
@@ -456,7 +497,7 @@ func (p *parser) valueInlineTable(it item, parentIsArray bool) (interface{}, tom
}
p.context = outerContext
p.currentKey = outerKey
return hash, tomlHash
return topHash, tomlHash
}
// numHasLeadingZero checks if this number has leading zeroes, allowing for '0',
@@ -486,9 +527,9 @@ func numUnderscoresOK(s string) bool {
}
}
// isHexadecimal is a superset of all the permissable characters
// surrounding an underscore.
accept = isHexadecimal(r)
// isHexis a superset of all the permissable characters surrounding an
// underscore.
accept = isHex(r)
}
return accept
}
@@ -511,21 +552,19 @@ func numPeriodsOK(s string) bool {
// Establishing the context also makes sure that the key isn't a duplicate, and
// will create implicit hashes automatically.
func (p *parser) addContext(key Key, array bool) {
var ok bool
// Always start at the top level and drill down for our context.
/// Always start at the top level and drill down for our context.
hashContext := p.mapping
keyContext := make(Key, 0)
keyContext := make(Key, 0, len(key)-1)
// We only need implicit hashes for key[0:-1]
for _, k := range key[0 : len(key)-1] {
_, ok = hashContext[k]
/// We only need implicit hashes for the parents.
for _, k := range key.parent() {
_, ok := hashContext[k]
keyContext = append(keyContext, k)
// No key? Make an implicit hash and move on.
if !ok {
p.addImplicit(keyContext)
hashContext[k] = make(map[string]interface{})
hashContext[k] = make(map[string]any)
}
// If the hash context is actually an array of tables, then set
@@ -534,9 +573,9 @@ func (p *parser) addContext(key Key, array bool) {
// Otherwise, it better be a table, since this MUST be a key group (by
// virtue of it not being the last element in a key).
switch t := hashContext[k].(type) {
case []map[string]interface{}:
case []map[string]any:
hashContext = t[len(t)-1]
case map[string]interface{}:
case map[string]any:
hashContext = t
default:
p.panicf("Key '%s' was already created as a hash.", keyContext)
@@ -547,39 +586,33 @@ func (p *parser) addContext(key Key, array bool) {
if array {
// If this is the first element for this array, then allocate a new
// list of tables for it.
k := key[len(key)-1]
k := key.last()
if _, ok := hashContext[k]; !ok {
hashContext[k] = make([]map[string]interface{}, 0, 4)
hashContext[k] = make([]map[string]any, 0, 4)
}
// Add a new table. But make sure the key hasn't already been used
// for something else.
if hash, ok := hashContext[k].([]map[string]interface{}); ok {
hashContext[k] = append(hash, make(map[string]interface{}))
if hash, ok := hashContext[k].([]map[string]any); ok {
hashContext[k] = append(hash, make(map[string]any))
} else {
p.panicf("Key '%s' was already created and cannot be used as an array.", key)
}
} else {
p.setValue(key[len(key)-1], make(map[string]interface{}))
p.setValue(key.last(), make(map[string]any))
}
p.context = append(p.context, key[len(key)-1])
}
// set calls setValue and setType.
func (p *parser) set(key string, val interface{}, typ tomlType, pos Position) {
p.setValue(key, val)
p.setType(key, typ, pos)
p.context = append(p.context, key.last())
}
// setValue sets the given key to the given value in the current context.
// It will make sure that the key hasn't already been defined, account for
// implicit key groups.
func (p *parser) setValue(key string, value interface{}) {
func (p *parser) setValue(key string, value any) {
var (
tmpHash interface{}
tmpHash any
ok bool
hash = p.mapping
keyContext Key
keyContext = make(Key, 0, len(p.context)+1)
)
for _, k := range p.context {
keyContext = append(keyContext, k)
@@ -587,11 +620,11 @@ func (p *parser) setValue(key string, value interface{}) {
p.bug("Context for key '%s' has not been established.", keyContext)
}
switch t := tmpHash.(type) {
case []map[string]interface{}:
case []map[string]any:
// The context is a table of hashes. Pick the most recent table
// defined as the current hash.
hash = t[len(t)-1]
case map[string]interface{}:
case map[string]any:
hash = t
default:
p.panicf("Key '%s' has already been defined.", keyContext)
@@ -618,9 +651,8 @@ func (p *parser) setValue(key string, value interface{}) {
p.removeImplicit(keyContext)
return
}
// Otherwise, we have a concrete key trying to override a previous
// key, which is *always* wrong.
// Otherwise, we have a concrete key trying to override a previous key,
// which is *always* wrong.
p.panicf("Key '%s' has already been defined.", keyContext)
}
@@ -683,8 +715,11 @@ func stripFirstNewline(s string) string {
// the next newline. After a line-ending backslash, all whitespace is removed
// until the next non-whitespace character.
func (p *parser) stripEscapedNewlines(s string) string {
var b strings.Builder
var i int
var (
b strings.Builder
i int
)
b.Grow(len(s))
for {
ix := strings.Index(s[i:], `\`)
if ix < 0 {
@@ -714,9 +749,8 @@ func (p *parser) stripEscapedNewlines(s string) string {
continue
}
if !strings.Contains(s[i:j], "\n") {
// This is not a line-ending backslash.
// (It's a bad escape sequence, but we can let
// replaceEscapes catch it.)
// This is not a line-ending backslash. (It's a bad escape sequence,
// but we can let replaceEscapes catch it.)
i++
continue
}
@@ -727,79 +761,78 @@ func (p *parser) stripEscapedNewlines(s string) string {
}
func (p *parser) replaceEscapes(it item, str string) string {
replaced := make([]rune, 0, len(str))
s := []byte(str)
r := 0
for r < len(s) {
if s[r] != '\\' {
c, size := utf8.DecodeRune(s[r:])
r += size
replaced = append(replaced, c)
var (
b strings.Builder
skip = 0
)
b.Grow(len(str))
for i, c := range str {
if skip > 0 {
skip--
continue
}
r += 1
if r >= len(s) {
if c != '\\' {
b.WriteRune(c)
continue
}
if i >= len(str) {
p.bug("Escape sequence at end of string.")
return ""
}
switch s[r] {
switch str[i+1] {
default:
p.bug("Expected valid escape code after \\, but got %q.", s[r])
p.bug("Expected valid escape code after \\, but got %q.", str[i+1])
case ' ', '\t':
p.panicItemf(it, "invalid escape: '\\%c'", s[r])
p.panicItemf(it, "invalid escape: '\\%c'", str[i+1])
case 'b':
replaced = append(replaced, rune(0x0008))
r += 1
b.WriteByte(0x08)
skip = 1
case 't':
replaced = append(replaced, rune(0x0009))
r += 1
b.WriteByte(0x09)
skip = 1
case 'n':
replaced = append(replaced, rune(0x000A))
r += 1
b.WriteByte(0x0a)
skip = 1
case 'f':
replaced = append(replaced, rune(0x000C))
r += 1
b.WriteByte(0x0c)
skip = 1
case 'r':
replaced = append(replaced, rune(0x000D))
r += 1
b.WriteByte(0x0d)
skip = 1
case 'e':
if p.tomlNext {
replaced = append(replaced, rune(0x001B))
r += 1
b.WriteByte(0x1b)
skip = 1
}
case '"':
replaced = append(replaced, rune(0x0022))
r += 1
b.WriteByte(0x22)
skip = 1
case '\\':
replaced = append(replaced, rune(0x005C))
r += 1
b.WriteByte(0x5c)
skip = 1
// The lexer guarantees the correct number of characters are present;
// don't need to check here.
case 'x':
if p.tomlNext {
escaped := p.asciiEscapeToUnicode(it, s[r+1:r+3])
replaced = append(replaced, escaped)
r += 3
escaped := p.asciiEscapeToUnicode(it, str[i+2:i+4])
b.WriteRune(escaped)
skip = 3
}
case 'u':
// At this point, we know we have a Unicode escape of the form
// `uXXXX` at [r, r+5). (Because the lexer guarantees this
// for us.)
escaped := p.asciiEscapeToUnicode(it, s[r+1:r+5])
replaced = append(replaced, escaped)
r += 5
escaped := p.asciiEscapeToUnicode(it, str[i+2:i+6])
b.WriteRune(escaped)
skip = 5
case 'U':
// At this point, we know we have a Unicode escape of the form
// `uXXXX` at [r, r+9). (Because the lexer guarantees this
// for us.)
escaped := p.asciiEscapeToUnicode(it, s[r+1:r+9])
replaced = append(replaced, escaped)
r += 9
escaped := p.asciiEscapeToUnicode(it, str[i+2:i+10])
b.WriteRune(escaped)
skip = 9
}
}
return string(replaced)
return b.String()
}
func (p *parser) asciiEscapeToUnicode(it item, bs []byte) rune {
s := string(bs)
func (p *parser) asciiEscapeToUnicode(it item, s string) rune {
hex, err := strconv.ParseUint(strings.ToLower(s), 16, 32)
if err != nil {
p.bug("Could not parse '%s' as a hexadecimal number, but the lexer claims it's OK: %s", s, err)
+2 -6
View File
@@ -25,10 +25,8 @@ type field struct {
// breaking ties with index sequence.
type byName []field
func (x byName) Len() int { return len(x) }
func (x byName) Len() int { return len(x) }
func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x byName) Less(i, j int) bool {
if x[i].name != x[j].name {
return x[i].name < x[j].name
@@ -45,10 +43,8 @@ func (x byName) Less(i, j int) bool {
// byIndex sorts field by index sequence.
type byIndex []field
func (x byIndex) Len() int { return len(x) }
func (x byIndex) Len() int { return len(x) }
func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] }
func (x byIndex) Less(i, j int) bool {
for k, xik := range x[i].index {
if k >= len(x[j].index) {
+3 -8
View File
@@ -22,13 +22,8 @@ func typeIsTable(t tomlType) bool {
type tomlBaseType string
func (btype tomlBaseType) typeString() string {
return string(btype)
}
func (btype tomlBaseType) String() string {
return btype.typeString()
}
func (btype tomlBaseType) typeString() string { return string(btype) }
func (btype tomlBaseType) String() string { return btype.typeString() }
var (
tomlInteger tomlBaseType = "Integer"
@@ -54,7 +49,7 @@ func (p *parser) typeOfPrimitive(lexItem item) tomlType {
return tomlFloat
case itemDatetime:
return tomlDatetime
case itemString:
case itemString, itemStringEsc:
return tomlString
case itemMultilineString:
return tomlString
-23
View File
@@ -1,23 +0,0 @@
language: go
# See https://travis-ci.community/t/goos-js-goarch-wasm-go-run-fails-panic-newosproc-not-implemented/1651
#addons:
# chrome: stable
before_install:
- export GO111MODULE=on
#install:
#- go get github.com/agnivade/wasmbrowsertest
#- mv $GOPATH/bin/wasmbrowsertest $GOPATH/bin/go_js_wasm_exec
#- export PATH=$GOPATH/bin:$PATH
go:
- 1.13.x
- 1.14.x
- 1.15.x
- tip
script:
#- GOOS=js GOARCH=wasm go test -v
- go test -v
+3 -5
View File
@@ -4,12 +4,10 @@ install:
go install
lint:
gofmt -l -s -w . && go vet . && golint -set_exit_status=1 .
gofmt -l -s -w . && go vet .
test: # The first 2 go gets are to support older Go versions
go get github.com/arbovm/levenshtein
go get github.com/dgryski/trifles/leven
GO111MODULE=on go test -race -v -coverprofile=coverage.txt -covermode=atomic
test:
go test -race -v -coverprofile=coverage.txt -covermode=atomic
bench:
go test -run=XXX -bench=. -benchmem -count=5
+1 -1
View File
@@ -1,4 +1,4 @@
levenshtein [![Build Status](https://travis-ci.org/agnivade/levenshtein.svg?branch=master)](https://travis-ci.org/agnivade/levenshtein) [![Go Report Card](https://goreportcard.com/badge/github.com/agnivade/levenshtein)](https://goreportcard.com/report/github.com/agnivade/levenshtein) [![PkgGoDev](https://pkg.go.dev/badge/github.com/agnivade/levenshtein)](https://pkg.go.dev/github.com/agnivade/levenshtein)
levenshtein ![Build Status](https://github.com/agnivade/levenshtein/actions/workflows/ci.yml/badge.svg) [![Go Report Card](https://goreportcard.com/badge/github.com/agnivade/levenshtein)](https://goreportcard.com/report/github.com/agnivade/levenshtein) [![PkgGoDev](https://pkg.go.dev/badge/github.com/agnivade/levenshtein)](https://pkg.go.dev/github.com/agnivade/levenshtein)
===========
[Go](http://golang.org) package to calculate the [Levenshtein Distance](http://en.wikipedia.org/wiki/Levenshtein_distance)
+20 -8
View File
@@ -41,6 +41,25 @@ func ComputeDistance(a, b string) int {
if len(s1) > len(s2) {
s1, s2 = s2, s1
}
// remove trailing identical runes.
for i := 0; i < len(s1); i++ {
if s1[len(s1)-1-i] != s2[len(s2)-1-i] {
s1 = s1[:len(s1)-i]
s2 = s2[:len(s2)-i]
break
}
}
// Remove leading identical runes.
for i := 0; i < len(s1); i++ {
if s1[i] != s2[i] {
s1 = s1[i:]
s2 = s2[i:]
break
}
}
lenS1 := len(s1)
lenS2 := len(s2)
@@ -71,7 +90,7 @@ func ComputeDistance(a, b string) int {
for j := 1; j <= lenS1; j++ {
current := x[j-1] // match
if s2[i-1] != s1[j-1] {
current = min(min(x[j-1]+1, prev+1), x[j]+1)
current = min(x[j-1]+1, prev+1, x[j]+1)
}
x[j-1] = prev
prev = current
@@ -80,10 +99,3 @@ func ComputeDistance(a, b string) int {
}
return int(x[lenS1])
}
func min(a, b uint16) uint16 {
if a < b {
return a
}
return b
}
+11
View File
@@ -442,6 +442,17 @@ func (c *Config) WithUseDualStack(enable bool) *Config {
return c
}
// WithUseFIPSEndpoint sets a config UseFIPSEndpoint value returning a Config
// pointer for chaining.
func (c *Config) WithUseFIPSEndpoint(enable bool) *Config {
if enable {
c.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled
} else {
c.UseFIPSEndpoint = endpoints.FIPSEndpointStateDisabled
}
return c
}
// WithEC2MetadataDisableTimeoutOverride sets a config EC2MetadataDisableTimeoutOverride value
// returning a Config pointer for chaining.
func (c *Config) WithEC2MetadataDisableTimeoutOverride(enable bool) *Config {
+46 -1
View File
@@ -31,6 +31,8 @@ package endpointcreds
import (
"encoding/json"
"fmt"
"strings"
"time"
"github.com/aws/aws-sdk-go/aws"
@@ -69,7 +71,37 @@ type Provider struct {
// Optional authorization token value if set will be used as the value of
// the Authorization header of the endpoint credential request.
//
// When constructed from environment, the provider will use the value of
// AWS_CONTAINER_AUTHORIZATION_TOKEN environment variable as the token
//
// Will be overridden if AuthorizationTokenProvider is configured
AuthorizationToken string
// Optional auth provider func to dynamically load the auth token from a file
// everytime a credential is retrieved
//
// When constructed from environment, the provider will read and use the content
// of the file pointed to by AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE environment variable
// as the auth token everytime credentials are retrieved
//
// Will override AuthorizationToken if configured
AuthorizationTokenProvider AuthTokenProvider
}
// AuthTokenProvider defines an interface to dynamically load a value to be passed
// for the Authorization header of a credentials request.
type AuthTokenProvider interface {
GetToken() (string, error)
}
// TokenProviderFunc is a func type implementing AuthTokenProvider interface
// and enables customizing token provider behavior
type TokenProviderFunc func() (string, error)
// GetToken func retrieves auth token according to TokenProviderFunc implementation
func (p TokenProviderFunc) GetToken() (string, error) {
return p()
}
// NewProviderClient returns a credentials Provider for retrieving AWS credentials
@@ -164,7 +196,20 @@ func (p *Provider) getCredentials(ctx aws.Context) (*getCredentialsOutput, error
req := p.Client.NewRequest(op, nil, out)
req.SetContext(ctx)
req.HTTPRequest.Header.Set("Accept", "application/json")
if authToken := p.AuthorizationToken; len(authToken) != 0 {
authToken := p.AuthorizationToken
var err error
if p.AuthorizationTokenProvider != nil {
authToken, err = p.AuthorizationTokenProvider.GetToken()
if err != nil {
return nil, fmt.Errorf("get authorization token: %v", err)
}
}
if strings.ContainsAny(authToken, "\r\n") {
return nil, fmt.Errorf("authorization token contains invalid newline sequence")
}
if len(authToken) != 0 {
req.HTTPRequest.Header.Set("Authorization", authToken)
}
+54 -10
View File
@@ -9,6 +9,7 @@ package defaults
import (
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
@@ -115,9 +116,31 @@ func CredProviders(cfg *aws.Config, handlers request.Handlers) []credentials.Pro
const (
httpProviderAuthorizationEnvVar = "AWS_CONTAINER_AUTHORIZATION_TOKEN"
httpProviderAuthFileEnvVar = "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE"
httpProviderEnvVar = "AWS_CONTAINER_CREDENTIALS_FULL_URI"
)
// direct representation of the IPv4 address for the ECS container
// "169.254.170.2"
var ecsContainerIPv4 net.IP = []byte{
169, 254, 170, 2,
}
// direct representation of the IPv4 address for the EKS container
// "169.254.170.23"
var eksContainerIPv4 net.IP = []byte{
169, 254, 170, 23,
}
// direct representation of the IPv6 address for the EKS container
// "fd00:ec2::23"
var eksContainerIPv6 net.IP = []byte{
0xFD, 0, 0xE, 0xC2,
0, 0, 0, 0,
0, 0, 0, 0,
0, 0, 0, 0x23,
}
// RemoteCredProvider returns a credentials provider for the default remote
// endpoints such as EC2 or ECS Roles.
func RemoteCredProvider(cfg aws.Config, handlers request.Handlers) credentials.Provider {
@@ -135,19 +158,22 @@ func RemoteCredProvider(cfg aws.Config, handlers request.Handlers) credentials.P
var lookupHostFn = net.LookupHost
func isLoopbackHost(host string) (bool, error) {
ip := net.ParseIP(host)
if ip != nil {
return ip.IsLoopback(), nil
// isAllowedHost allows host to be loopback or known ECS/EKS container IPs
//
// host can either be an IP address OR an unresolved hostname - resolution will
// be automatically performed in the latter case
func isAllowedHost(host string) (bool, error) {
if ip := net.ParseIP(host); ip != nil {
return isIPAllowed(ip), nil
}
// Host is not an ip, perform lookup
addrs, err := lookupHostFn(host)
if err != nil {
return false, err
}
for _, addr := range addrs {
if !net.ParseIP(addr).IsLoopback() {
if ip := net.ParseIP(addr); ip == nil || !isIPAllowed(ip) {
return false, nil
}
}
@@ -155,6 +181,13 @@ func isLoopbackHost(host string) (bool, error) {
return true, nil
}
func isIPAllowed(ip net.IP) bool {
return ip.IsLoopback() ||
ip.Equal(ecsContainerIPv4) ||
ip.Equal(eksContainerIPv4) ||
ip.Equal(eksContainerIPv6)
}
func localHTTPCredProvider(cfg aws.Config, handlers request.Handlers, u string) credentials.Provider {
var errMsg string
@@ -165,10 +198,12 @@ func localHTTPCredProvider(cfg aws.Config, handlers request.Handlers, u string)
host := aws.URLHostname(parsed)
if len(host) == 0 {
errMsg = "unable to parse host from local HTTP cred provider URL"
} else if isLoopback, loopbackErr := isLoopbackHost(host); loopbackErr != nil {
errMsg = fmt.Sprintf("failed to resolve host %q, %v", host, loopbackErr)
} else if !isLoopback {
errMsg = fmt.Sprintf("invalid endpoint host, %q, only loopback hosts are allowed.", host)
} else if parsed.Scheme == "http" {
if isAllowedHost, allowHostErr := isAllowedHost(host); allowHostErr != nil {
errMsg = fmt.Sprintf("failed to resolve host %q, %v", host, allowHostErr)
} else if !isAllowedHost {
errMsg = fmt.Sprintf("invalid endpoint host, %q, only loopback/ecs/eks hosts are allowed.", host)
}
}
}
@@ -190,6 +225,15 @@ func httpCredProvider(cfg aws.Config, handlers request.Handlers, u string) crede
func(p *endpointcreds.Provider) {
p.ExpiryWindow = 5 * time.Minute
p.AuthorizationToken = os.Getenv(httpProviderAuthorizationEnvVar)
if authFilePath := os.Getenv(httpProviderAuthFileEnvVar); authFilePath != "" {
p.AuthorizationTokenProvider = endpointcreds.TokenProviderFunc(func() (string, error) {
if contents, err := ioutil.ReadFile(authFilePath); err != nil {
return "", fmt.Errorf("failed to read authorization token from %v: %v", authFilePath, err)
} else {
return string(contents), nil
}
})
}
},
)
}
+4 -1
View File
@@ -2,6 +2,7 @@ package ec2metadata
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"net/http"
"sync/atomic"
"time"
@@ -65,7 +66,9 @@ func (t *tokenProvider) fetchTokenHandler(r *request.Request) {
switch requestFailureError.StatusCode() {
case http.StatusForbidden, http.StatusNotFound, http.StatusMethodNotAllowed:
atomic.StoreUint32(&t.disabled, 1)
t.client.Config.Logger.Log(fmt.Sprintf("WARN: failed to get session token, falling back to IMDSv1: %v", requestFailureError))
if t.client.Config.LogLevel.Matches(aws.LogDebugWithDeprecated) {
t.client.Config.Logger.Log(fmt.Sprintf("WARN: failed to get session token, falling back to IMDSv1: %v", requestFailureError))
}
case http.StatusBadRequest:
r.Error = requestFailureError
}
File diff suppressed because it is too large Load Diff
+11 -2
View File
@@ -256,8 +256,17 @@ func (a *WaiterAcceptor) match(name string, l aws.Logger, req *Request, err erro
s := a.Expected.(int)
result = s == req.HTTPResponse.StatusCode
case ErrorWaiterMatch:
if aerr, ok := err.(awserr.Error); ok {
result = aerr.Code() == a.Expected.(string)
switch ex := a.Expected.(type) {
case string:
if aerr, ok := err.(awserr.Error); ok {
result = aerr.Code() == ex
}
case bool:
if ex {
result = err != nil
} else {
result = err == nil
}
}
default:
waiterLogf(l, "WARNING: Waiter %s encountered unexpected matcher: %s",
+28
View File
@@ -171,6 +171,12 @@ type envConfig struct {
// AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE=IPv6
EC2IMDSEndpointMode endpoints.EC2IMDSEndpointModeState
// Specifies that IMDS clients should not fallback to IMDSv1 if token
// requests fail.
//
// AWS_EC2_METADATA_V1_DISABLED=true
EC2IMDSv1Disabled *bool
// Specifies that SDK clients must resolve a dual-stack endpoint for
// services.
//
@@ -251,6 +257,9 @@ var (
ec2IMDSEndpointModeEnvKey = []string{
"AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE",
}
ec2MetadataV1DisabledEnvKey = []string{
"AWS_EC2_METADATA_V1_DISABLED",
}
useCABundleKey = []string{
"AWS_CA_BUNDLE",
}
@@ -393,6 +402,7 @@ func envConfigLoad(enableSharedConfig bool) (envConfig, error) {
if err := setEC2IMDSEndpointMode(&cfg.EC2IMDSEndpointMode, ec2IMDSEndpointModeEnvKey); err != nil {
return envConfig{}, err
}
setBoolPtrFromEnvVal(&cfg.EC2IMDSv1Disabled, ec2MetadataV1DisabledEnvKey)
if err := setUseDualStackEndpointFromEnvVal(&cfg.UseDualStackEndpoint, awsUseDualStackEndpoint); err != nil {
return cfg, err
@@ -414,6 +424,24 @@ func setFromEnvVal(dst *string, keys []string) {
}
}
func setBoolPtrFromEnvVal(dst **bool, keys []string) {
for _, k := range keys {
value := os.Getenv(k)
if len(value) == 0 {
continue
}
switch {
case strings.EqualFold(value, "false"):
*dst = new(bool)
**dst = false
case strings.EqualFold(value, "true"):
*dst = new(bool)
**dst = true
}
}
}
func setEC2IMDSEndpointMode(mode *endpoints.EC2IMDSEndpointModeState, keys []string) error {
for _, k := range keys {
value := os.Getenv(k)
+8
View File
@@ -779,6 +779,14 @@ func mergeConfigSrcs(cfg, userCfg *aws.Config,
cfg.EndpointResolver = wrapEC2IMDSEndpoint(cfg.EndpointResolver, ec2IMDSEndpoint, endpointMode)
}
cfg.EC2MetadataEnableFallback = userCfg.EC2MetadataEnableFallback
if cfg.EC2MetadataEnableFallback == nil && envCfg.EC2IMDSv1Disabled != nil {
cfg.EC2MetadataEnableFallback = aws.Bool(!*envCfg.EC2IMDSv1Disabled)
}
if cfg.EC2MetadataEnableFallback == nil && sharedCfg.EC2IMDSv1Disabled != nil {
cfg.EC2MetadataEnableFallback = aws.Bool(!*sharedCfg.EC2IMDSv1Disabled)
}
cfg.S3UseARNRegion = userCfg.S3UseARNRegion
if cfg.S3UseARNRegion == nil {
cfg.S3UseARNRegion = &envCfg.S3UseARNRegion
+30 -5
View File
@@ -80,6 +80,9 @@ const (
// EC2 IMDS Endpoint
ec2MetadataServiceEndpointKey = "ec2_metadata_service_endpoint"
// ECS IMDSv1 disable fallback
ec2MetadataV1DisabledKey = "ec2_metadata_v1_disabled"
// Use DualStack Endpoint Resolution
useDualStackEndpoint = "use_dualstack_endpoint"
@@ -179,6 +182,12 @@ type sharedConfig struct {
// ec2_metadata_service_endpoint=http://fd00:ec2::254
EC2IMDSEndpoint string
// Specifies that IMDS clients should not fallback to IMDSv1 if token
// requests fail.
//
// ec2_metadata_v1_disabled=true
EC2IMDSv1Disabled *bool
// Specifies that SDK clients must resolve a dual-stack endpoint for
// services.
//
@@ -389,8 +398,15 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
updateString(&cfg.Region, section, regionKey)
updateString(&cfg.CustomCABundle, section, customCABundleKey)
// we're retaining a behavioral quirk with this field that existed before
// the removal of literal parsing for (aws-sdk-go-v2/#2276):
// - if the key is missing, the config field will not be set
// - if the key is set to a non-numeric, the config field will be set to 0
if section.Has(roleDurationSecondsKey) {
d := time.Duration(section.Int(roleDurationSecondsKey)) * time.Second
var d time.Duration
if v, ok := section.Int(roleDurationSecondsKey); ok {
d = time.Duration(v) * time.Second
}
cfg.AssumeRoleDuration = &d
}
@@ -427,6 +443,7 @@ func (cfg *sharedConfig) setFromIniFile(profile string, file sharedConfigFile, e
ec2MetadataServiceEndpointModeKey, file.Filename, err)
}
updateString(&cfg.EC2IMDSEndpoint, section, ec2MetadataServiceEndpointKey)
updateBoolPtr(&cfg.EC2IMDSv1Disabled, section, ec2MetadataV1DisabledKey)
updateUseDualStackEndpoint(&cfg.UseDualStackEndpoint, section, useDualStackEndpoint)
@@ -668,7 +685,10 @@ func updateBool(dst *bool, section ini.Section, key string) {
if !section.Has(key) {
return
}
*dst = section.Bool(key)
// retains pre-(aws-sdk-go-v2#2276) behavior where non-bool value would resolve to false
v, _ := section.Bool(key)
*dst = v
}
// updateBoolPtr will only update the dst with the value in the section key,
@@ -677,8 +697,11 @@ func updateBoolPtr(dst **bool, section ini.Section, key string) {
if !section.Has(key) {
return
}
// retains pre-(aws-sdk-go-v2#2276) behavior where non-bool value would resolve to false
v, _ := section.Bool(key)
*dst = new(bool)
**dst = section.Bool(key)
**dst = v
}
// SharedConfigLoadError is an error for the shared config file failed to load.
@@ -805,7 +828,8 @@ func updateUseDualStackEndpoint(dst *endpoints.DualStackEndpointState, section i
return
}
if section.Bool(key) {
// retains pre-(aws-sdk-go-v2/#2276) behavior where non-bool value would resolve to false
if v, _ := section.Bool(key); v {
*dst = endpoints.DualStackEndpointStateEnabled
} else {
*dst = endpoints.DualStackEndpointStateDisabled
@@ -821,7 +845,8 @@ func updateUseFIPSEndpoint(dst *endpoints.FIPSEndpointState, section ini.Section
return
}
if section.Bool(key) {
// retains pre-(aws-sdk-go-v2/#2276) behavior where non-bool value would resolve to false
if v, _ := section.Bool(key); v {
*dst = endpoints.FIPSEndpointStateEnabled
} else {
*dst = endpoints.FIPSEndpointStateDisabled
+1
View File
@@ -125,6 +125,7 @@ var requiredSignedHeaders = rules{
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": struct{}{},
"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": struct{}{},
"X-Amz-Expected-Bucket-Owner": struct{}{},
"X-Amz-Grant-Full-control": struct{}{},
"X-Amz-Grant-Read": struct{}{},
"X-Amz-Grant-Read-Acp": struct{}{},
+1 -1
View File
@@ -5,4 +5,4 @@ package aws
const SDKName = "aws-sdk-go"
// SDKVersion is the version of this SDK
const SDKVersion = "1.45.1"
const SDKVersion = "1.55.5"
+27 -30
View File
@@ -154,11 +154,11 @@ func (v ValueType) String() string {
// ValueType enums
const (
NoneType = ValueType(iota)
DecimalType
IntegerType
DecimalType // deprecated
IntegerType // deprecated
StringType
QuotedStringType
BoolType
BoolType // deprecated
)
// Value is a union container
@@ -166,9 +166,9 @@ type Value struct {
Type ValueType
raw []rune
integer int64
decimal float64
boolean bool
integer int64 // deprecated
decimal float64 // deprecated
boolean bool // deprecated
str string
}
@@ -253,24 +253,6 @@ func newLitToken(b []rune) (Token, int, error) {
}
token = newToken(TokenLit, b[:n], QuotedStringType)
} else if isNumberValue(b) {
var base int
base, n, err = getNumericalValue(b)
if err != nil {
return token, 0, err
}
value := b[:n]
vType := IntegerType
if contains(value, '.') || hasExponent(value) {
vType = DecimalType
}
token = newToken(TokenLit, value, vType)
token.base = base
} else if isBoolValue(b) {
n, err = getBoolValue(b)
token = newToken(TokenLit, b[:n], BoolType)
} else {
n, err = getValue(b)
token = newToken(TokenLit, b[:n], StringType)
@@ -280,18 +262,33 @@ func newLitToken(b []rune) (Token, int, error) {
}
// IntValue returns an integer value
func (v Value) IntValue() int64 {
return v.integer
func (v Value) IntValue() (int64, bool) {
i, err := strconv.ParseInt(string(v.raw), 0, 64)
if err != nil {
return 0, false
}
return i, true
}
// FloatValue returns a float value
func (v Value) FloatValue() float64 {
return v.decimal
func (v Value) FloatValue() (float64, bool) {
f, err := strconv.ParseFloat(string(v.raw), 64)
if err != nil {
return 0, false
}
return f, true
}
// BoolValue returns a bool value
func (v Value) BoolValue() bool {
return v.boolean
func (v Value) BoolValue() (bool, bool) {
// we don't use ParseBool as it recognizes more than what we've
// historically supported
if isCaselessLitValue(runesTrue, v.raw) {
return true, true
} else if isCaselessLitValue(runesFalse, v.raw) {
return false, true
}
return false, false
}
func isTrimmable(r rune) bool {
+3 -3
View File
@@ -145,17 +145,17 @@ func (t Section) ValueType(k string) (ValueType, bool) {
}
// Bool returns a bool value at k
func (t Section) Bool(k string) bool {
func (t Section) Bool(k string) (bool, bool) {
return t.values[k].BoolValue()
}
// Int returns an integer value at k
func (t Section) Int(k string) int64 {
func (t Section) Int(k string) (int64, bool) {
return t.values[k].IntValue()
}
// Float64 returns a float value at k
func (t Section) Float64(k string) float64 {
func (t Section) Float64(k string) (float64, bool) {
return t.values[k].FloatValue()
}
@@ -122,8 +122,8 @@ func (q *queryParser) parseStruct(v url.Values, value reflect.Value, prefix stri
}
func (q *queryParser) parseList(v url.Values, value reflect.Value, prefix string, tag reflect.StructTag) error {
// If it's empty, generate an empty value
if !value.IsNil() && value.Len() == 0 {
// If it's empty, and not ec2, generate an empty value
if !value.IsNil() && value.Len() == 0 && !q.isEC2 {
v.Set(prefix, "")
return nil
}
+5591 -2187
View File
File diff suppressed because it is too large Load Diff
+9
View File
@@ -25,6 +25,15 @@ const (
// "InvalidObjectState".
//
// Object is archived and inaccessible until restored.
//
// If the object you are retrieving is stored in the S3 Glacier Flexible Retrieval
// storage class, the S3 Glacier Deep Archive storage class, the S3 Intelligent-Tiering
// Archive Access tier, or the S3 Intelligent-Tiering Deep Archive Access tier,
// before you can retrieve the object you must first restore a copy using RestoreObject
// (https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html).
// Otherwise, this operation returns an InvalidObjectState error. For information
// about restoring archived objects, see Restoring Archived Objects (https://docs.aws.amazon.com/AmazonS3/latest/dev/restoring-objects.html)
// in the Amazon S3 User Guide.
ErrCodeInvalidObjectState = "InvalidObjectState"
// ErrCodeNoSuchBucket for service response error code
+11
View File
@@ -80,6 +80,10 @@ type S3API interface {
CreateMultipartUploadWithContext(aws.Context, *s3.CreateMultipartUploadInput, ...request.Option) (*s3.CreateMultipartUploadOutput, error)
CreateMultipartUploadRequest(*s3.CreateMultipartUploadInput) (*request.Request, *s3.CreateMultipartUploadOutput)
CreateSession(*s3.CreateSessionInput) (*s3.CreateSessionOutput, error)
CreateSessionWithContext(aws.Context, *s3.CreateSessionInput, ...request.Option) (*s3.CreateSessionOutput, error)
CreateSessionRequest(*s3.CreateSessionInput) (*request.Request, *s3.CreateSessionOutput)
DeleteBucket(*s3.DeleteBucketInput) (*s3.DeleteBucketOutput, error)
DeleteBucketWithContext(aws.Context, *s3.DeleteBucketInput, ...request.Option) (*s3.DeleteBucketOutput, error)
DeleteBucketRequest(*s3.DeleteBucketInput) (*request.Request, *s3.DeleteBucketOutput)
@@ -300,6 +304,13 @@ type S3API interface {
ListBucketsWithContext(aws.Context, *s3.ListBucketsInput, ...request.Option) (*s3.ListBucketsOutput, error)
ListBucketsRequest(*s3.ListBucketsInput) (*request.Request, *s3.ListBucketsOutput)
ListDirectoryBuckets(*s3.ListDirectoryBucketsInput) (*s3.ListDirectoryBucketsOutput, error)
ListDirectoryBucketsWithContext(aws.Context, *s3.ListDirectoryBucketsInput, ...request.Option) (*s3.ListDirectoryBucketsOutput, error)
ListDirectoryBucketsRequest(*s3.ListDirectoryBucketsInput) (*request.Request, *s3.ListDirectoryBucketsOutput)
ListDirectoryBucketsPages(*s3.ListDirectoryBucketsInput, func(*s3.ListDirectoryBucketsOutput, bool) bool) error
ListDirectoryBucketsPagesWithContext(aws.Context, *s3.ListDirectoryBucketsInput, func(*s3.ListDirectoryBucketsOutput, bool) bool, ...request.Option) error
ListMultipartUploads(*s3.ListMultipartUploadsInput) (*s3.ListMultipartUploadsOutput, error)
ListMultipartUploadsWithContext(aws.Context, *s3.ListMultipartUploadsInput, ...request.Option) (*s3.ListMultipartUploadsOutput, error)
ListMultipartUploadsRequest(*s3.ListMultipartUploadsInput) (*request.Request, *s3.ListMultipartUploadsOutput)
+156 -40
View File
@@ -23,9 +23,32 @@ type UploadInput struct {
_ struct{} `locationName:"PutObjectRequest" type:"structure" payload:"Body"`
// The canned ACL to apply to the object. For more information, see Canned ACL
// (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL).
// (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html#CannedACL)
// in the Amazon S3 User Guide.
//
// This action is not supported by Amazon S3 on Outposts.
// When adding a new object, you can use headers to grant ACL-based permissions
// to individual Amazon Web Services accounts or to predefined groups defined
// by Amazon S3. These permissions are then added to the ACL on the object.
// By default, all objects are private. Only the owner has full access control.
// For more information, see Access Control List (ACL) Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-overview.html)
// and Managing ACLs Using the REST API (https://docs.aws.amazon.com/AmazonS3/latest/dev/acl-using-rest-api.html)
// in the Amazon S3 User Guide.
//
// If the bucket that you're uploading objects to uses the bucket owner enforced
// setting for S3 Object Ownership, ACLs are disabled and no longer affect permissions.
// Buckets that use this setting only accept PUT requests that don't specify
// an ACL or PUT requests that specify bucket owner full control ACLs, such
// as the bucket-owner-full-control canned ACL or an equivalent form of this
// ACL expressed in the XML format. PUT requests that contain other ACLs (for
// example, custom grants to certain Amazon Web Services accounts) fail and
// return a 400 error with the error code AccessControlListNotSupported. For
// more information, see Controlling ownership of objects and disabling ACLs
// (https://docs.aws.amazon.com/AmazonS3/latest/userguide/about-object-ownership.html)
// in the Amazon S3 User Guide.
//
// * This functionality is not supported for directory buckets.
//
// * This functionality is not supported for Amazon S3 on Outposts.
ACL *string `location:"header" locationName:"x-amz-acl" type:"string" enum:"ObjectCannedACL"`
// The readable body payload to send to S3.
@@ -33,19 +56,33 @@ type UploadInput struct {
// The bucket name to which the PUT action was initiated.
//
// When using this action with an access point, you must direct requests to
// the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
// Directory buckets - When you use this operation with a directory bucket,
// you must use virtual-hosted-style requests in the format Bucket_name.s3express-az_id.region.amazonaws.com.
// Path-style requests are not supported. Directory bucket names must be unique
// in the chosen Availability Zone. Bucket names must follow the format bucket_base_name--az-id--x-s3
// (for example, DOC-EXAMPLE-BUCKET--usw2-az1--x-s3). For information about
// bucket naming restrictions, see Directory bucket naming rules (https://docs.aws.amazon.com/AmazonS3/latest/userguide/directory-bucket-naming-rules.html)
// in the Amazon S3 User Guide.
//
// Access points - When you use this action with an access point, you must provide
// the alias of the access point in place of the bucket name or specify the
// access point ARN. When using the access point ARN, you must direct requests
// to the access point hostname. The access point hostname takes the form AccessPointName-AccountId.s3-accesspoint.Region.amazonaws.com.
// When using this action with an access point through the Amazon Web Services
// SDKs, you provide the access point ARN in place of the bucket name. For more
// information about access point ARNs, see Using access points (https://docs.aws.amazon.com/AmazonS3/latest/userguide/using-access-points.html)
// in the Amazon S3 User Guide.
//
// When you use this action with Amazon S3 on Outposts, you must direct requests
// to the S3 on Outposts hostname. The S3 on Outposts hostname takes the form
// AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com. When
// you use this action with S3 on Outposts through the Amazon Web Services SDKs,
// you provide the Outposts access point ARN in place of the bucket name. For
// more information about S3 on Outposts ARNs, see What is S3 on Outposts? (https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html)
// Access points and Object Lambda access points are not supported by directory
// buckets.
//
// S3 on Outposts - When you use this action with Amazon S3 on Outposts, you
// must direct requests to the S3 on Outposts hostname. The S3 on Outposts hostname
// takes the form AccessPointName-AccountId.outpostID.s3-outposts.Region.amazonaws.com.
// When you use this action with S3 on Outposts through the Amazon Web Services
// SDKs, you provide the Outposts access point ARN in place of the bucket name.
// For more information about S3 on Outposts ARNs, see What is S3 on Outposts?
// (https://docs.aws.amazon.com/AmazonS3/latest/userguide/S3onOutposts.html)
// in the Amazon S3 User Guide.
//
// Bucket is a required field
@@ -58,6 +95,8 @@ type UploadInput struct {
//
// Specifying this header with a PUT action doesnt affect bucket-level settings
// for S3 Bucket Key.
//
// This functionality is not supported for directory buckets.
BucketKeyEnabled *bool `location:"header" locationName:"x-amz-server-side-encryption-bucket-key-enabled" type:"boolean"`
// Can be used to specify caching behavior along the request/reply chain. For
@@ -65,16 +104,33 @@ type UploadInput struct {
// (http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9).
CacheControl *string `location:"header" locationName:"Cache-Control" type:"string"`
// Indicates the algorithm used to create the checksum for the object when using
// the SDK. This header will not provide any additional functionality if not
// using the SDK. When sending this header, there must be a corresponding x-amz-checksum
// or x-amz-trailer header sent. Otherwise, Amazon S3 fails the request with
// the HTTP status code 400 Bad Request. For more information, see Checking
// object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// Indicates the algorithm used to create the checksum for the object when you
// use the SDK. This header will not provide any additional functionality if
// you don't use the SDK. When you send this header, there must be a corresponding
// x-amz-checksum-algorithm or x-amz-trailer header sent. Otherwise, Amazon
// S3 fails the request with the HTTP status code 400 Bad Request.
//
// For the x-amz-checksum-algorithm header, replace algorithm with the supported
// algorithm from the following list:
//
// * CRC32
//
// * CRC32C
//
// * SHA1
//
// * SHA256
//
// For more information, see Checking object integrity (https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html)
// in the Amazon S3 User Guide.
//
// If you provide an individual checksum, Amazon S3 ignores any provided ChecksumAlgorithm
// parameter.
// If the individual checksum value you provide through x-amz-checksum-algorithm
// doesn't match the checksum algorithm you set through x-amz-sdk-checksum-algorithm,
// Amazon S3 ignores any provided ChecksumAlgorithm parameter and uses the checksum
// algorithm that matches the provided value in x-amz-checksum-algorithm .
//
// For directory buckets, when you use Amazon Web Services SDKs, CRC32 is the
// default checksum algorithm that's used for performance.
//
// The AWS SDK for Go v1 does not support automatic computing request payload
// checksum. This feature is available in the AWS SDK for Go v2. If a value
@@ -130,6 +186,13 @@ type UploadInput struct {
// integrity check. For more information about REST request authentication,
// see REST Authentication (https://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html).
//
// The Content-MD5 header is required for any request to upload an object with
// a retention period configured using Amazon S3 Object Lock. For more information
// about Amazon S3 Object Lock, see Amazon S3 Object Lock Overview (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html)
// in the Amazon S3 User Guide.
//
// This functionality is not supported for directory buckets.
//
// If the ContentMD5 is provided for a multipart upload, it will be ignored.
// Objects that will be uploaded in a single part, the ContentMD5 will be used.
ContentMD5 *string `location:"header" locationName:"Content-MD5" type:"string"`
@@ -138,9 +201,9 @@ type UploadInput struct {
// see https://www.rfc-editor.org/rfc/rfc9110.html#name-content-type (https://www.rfc-editor.org/rfc/rfc9110.html#name-content-type).
ContentType *string `location:"header" locationName:"Content-Type" type:"string"`
// The account ID of the expected bucket owner. If the bucket is owned by a
// different account, the request fails with the HTTP status code 403 Forbidden
// (access denied).
// The account ID of the expected bucket owner. If the account ID that you provide
// does not match the actual owner of the bucket, the request fails with the
// HTTP status code 403 Forbidden (access denied).
ExpectedBucketOwner *string `location:"header" locationName:"x-amz-expected-bucket-owner" type:"string"`
// The date and time at which the object is no longer cacheable. For more information,
@@ -149,22 +212,30 @@ type UploadInput struct {
// Gives the grantee READ, READ_ACP, and WRITE_ACP permissions on the object.
//
// This action is not supported by Amazon S3 on Outposts.
// * This functionality is not supported for directory buckets.
//
// * This functionality is not supported for Amazon S3 on Outposts.
GrantFullControl *string `location:"header" locationName:"x-amz-grant-full-control" type:"string"`
// Allows grantee to read the object data and its metadata.
//
// This action is not supported by Amazon S3 on Outposts.
// * This functionality is not supported for directory buckets.
//
// * This functionality is not supported for Amazon S3 on Outposts.
GrantRead *string `location:"header" locationName:"x-amz-grant-read" type:"string"`
// Allows grantee to read the object ACL.
//
// This action is not supported by Amazon S3 on Outposts.
// * This functionality is not supported for directory buckets.
//
// * This functionality is not supported for Amazon S3 on Outposts.
GrantReadACP *string `location:"header" locationName:"x-amz-grant-read-acp" type:"string"`
// Allows grantee to write the ACL for the applicable object.
//
// This action is not supported by Amazon S3 on Outposts.
// * This functionality is not supported for directory buckets.
//
// * This functionality is not supported for Amazon S3 on Outposts.
GrantWriteACP *string `location:"header" locationName:"x-amz-grant-write-acp" type:"string"`
// Object key for which the PUT action was initiated.
@@ -176,25 +247,37 @@ type UploadInput struct {
Metadata map[string]*string `location:"headers" locationName:"x-amz-meta-" type:"map"`
// Specifies whether a legal hold will be applied to this object. For more information
// about S3 Object Lock, see Object Lock (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html).
// about S3 Object Lock, see Object Lock (https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock.html)
// in the Amazon S3 User Guide.
//
// This functionality is not supported for directory buckets.
ObjectLockLegalHoldStatus *string `location:"header" locationName:"x-amz-object-lock-legal-hold" type:"string" enum:"ObjectLockLegalHoldStatus"`
// The Object Lock mode that you want to apply to this object.
//
// This functionality is not supported for directory buckets.
ObjectLockMode *string `location:"header" locationName:"x-amz-object-lock-mode" type:"string" enum:"ObjectLockMode"`
// The date and time when you want this object's Object Lock to expire. Must
// be formatted as a timestamp parameter.
//
// This functionality is not supported for directory buckets.
ObjectLockRetainUntilDate *time.Time `location:"header" locationName:"x-amz-object-lock-retain-until-date" type:"timestamp" timestampFormat:"iso8601"`
// Confirms that the requester knows that they will be charged for the request.
// Bucket owners need not specify this parameter in their requests. For information
// about downloading objects from Requester Pays buckets, see Downloading Objects
// Bucket owners need not specify this parameter in their requests. If either
// the source or destination S3 bucket has Requester Pays enabled, the requester
// will pay for corresponding charges to copy the object. For information about
// downloading objects from Requester Pays buckets, see Downloading Objects
// in Requester Pays Buckets (https://docs.aws.amazon.com/AmazonS3/latest/dev/ObjectsinRequesterPaysBuckets.html)
// in the Amazon S3 User Guide.
//
// This functionality is not supported for directory buckets.
RequestPayer *string `location:"header" locationName:"x-amz-request-payer" type:"string" enum:"RequestPayer"`
// Specifies the algorithm to use to when encrypting the object (for example,
// AES256).
// Specifies the algorithm to use when encrypting the object (for example, AES256).
//
// This functionality is not supported for directory buckets.
SSECustomerAlgorithm *string `location:"header" locationName:"x-amz-server-side-encryption-customer-algorithm" type:"string"`
// Specifies the customer-provided encryption key for Amazon S3 to use in encrypting
@@ -202,50 +285,80 @@ type UploadInput struct {
// S3 does not store the encryption key. The key must be appropriate for use
// with the algorithm specified in the x-amz-server-side-encryption-customer-algorithm
// header.
//
// This functionality is not supported for directory buckets.
SSECustomerKey *string `marshal-as:"blob" location:"header" locationName:"x-amz-server-side-encryption-customer-key" type:"string" sensitive:"true"`
// Specifies the 128-bit MD5 digest of the encryption key according to RFC 1321.
// Amazon S3 uses this header for a message integrity check to ensure that the
// encryption key was transmitted without error.
//
// This functionality is not supported for directory buckets.
SSECustomerKeyMD5 *string `location:"header" locationName:"x-amz-server-side-encryption-customer-key-MD5" type:"string"`
// Specifies the Amazon Web Services KMS Encryption Context to use for object
// encryption. The value of this header is a base64-encoded UTF-8 string holding
// JSON with the encryption context key-value pairs. This value is stored as
// object metadata and automatically gets passed on to Amazon Web Services KMS
// for future GetObject or CopyObject operations on this object.
// for future GetObject or CopyObject operations on this object. This value
// must be explicitly added during CopyObject operations.
//
// This functionality is not supported for directory buckets.
SSEKMSEncryptionContext *string `location:"header" locationName:"x-amz-server-side-encryption-context" type:"string" sensitive:"true"`
// If x-amz-server-side-encryption has a valid value of aws:kms or aws:kms:dsse,
// this header specifies the ID of the Key Management Service (KMS) symmetric
// encryption customer managed key that was used for the object. If you specify
// x-amz-server-side-encryption:aws:kms or x-amz-server-side-encryption:aws:kms:dsse,
// this header specifies the ID (Key ID, Key ARN, or Key Alias) of the Key Management
// Service (KMS) symmetric encryption customer managed key that was used for
// the object. If you specify x-amz-server-side-encryption:aws:kms or x-amz-server-side-encryption:aws:kms:dsse,
// but do not providex-amz-server-side-encryption-aws-kms-key-id, Amazon S3
// uses the Amazon Web Services managed key (aws/s3) to protect the data. If
// the KMS key does not exist in the same account that's issuing the command,
// you must use the full ARN and not just the ID.
//
// This functionality is not supported for directory buckets.
SSEKMSKeyId *string `location:"header" locationName:"x-amz-server-side-encryption-aws-kms-key-id" type:"string" sensitive:"true"`
// The server-side encryption algorithm used when storing this object in Amazon
// S3 (for example, AES256, aws:kms, aws:kms:dsse).
// The server-side encryption algorithm that was used when you store this object
// in Amazon S3 (for example, AES256, aws:kms, aws:kms:dsse).
//
// General purpose buckets - You have four mutually exclusive options to protect
// data using server-side encryption in Amazon S3, depending on how you choose
// to manage the encryption keys. Specifically, the encryption key options are
// Amazon S3 managed keys (SSE-S3), Amazon Web Services KMS keys (SSE-KMS or
// DSSE-KMS), and customer-provided keys (SSE-C). Amazon S3 encrypts data with
// server-side encryption by using Amazon S3 managed keys (SSE-S3) by default.
// You can optionally tell Amazon S3 to encrypt data at rest by using server-side
// encryption with other key options. For more information, see Using Server-Side
// Encryption (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingServerSideEncryption.html)
// in the Amazon S3 User Guide.
//
// Directory buckets - For directory buckets, only the server-side encryption
// with Amazon S3 managed keys (SSE-S3) (AES256) value is supported.
ServerSideEncryption *string `location:"header" locationName:"x-amz-server-side-encryption" type:"string" enum:"ServerSideEncryption"`
// By default, Amazon S3 uses the STANDARD Storage Class to store newly created
// objects. The STANDARD storage class provides high durability and high availability.
// Depending on performance needs, you can specify a different Storage Class.
// Amazon S3 on Outposts only uses the OUTPOSTS Storage Class. For more information,
// see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html)
// For more information, see Storage Classes (https://docs.aws.amazon.com/AmazonS3/latest/dev/storage-class-intro.html)
// in the Amazon S3 User Guide.
//
// * For directory buckets, only the S3 Express One Zone storage class is
// supported to store newly created objects.
//
// * Amazon S3 on Outposts only uses the OUTPOSTS Storage Class.
StorageClass *string `location:"header" locationName:"x-amz-storage-class" type:"string" enum:"StorageClass"`
// The tag-set for the object. The tag-set must be encoded as URL Query parameters.
// (For example, "Key1=Value1")
//
// This functionality is not supported for directory buckets.
Tagging *string `location:"header" locationName:"x-amz-tagging" type:"string"`
// If the bucket is configured as a website, redirects requests for this object
// to another object in the same bucket or to an external URL. Amazon S3 stores
// the value of this header in the object metadata. For information about object
// metadata, see Object Key and Metadata (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html).
// metadata, see Object Key and Metadata (https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html)
// in the Amazon S3 User Guide.
//
// In the following example, the request header sets the redirect to an object
// (anotherPage.html) in the same bucket:
@@ -259,6 +372,9 @@ type UploadInput struct {
//
// For more information about website hosting in Amazon S3, see Hosting Websites
// on Amazon S3 (https://docs.aws.amazon.com/AmazonS3/latest/dev/WebsiteHosting.html)
// and How to Configure Website Page Redirects (https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html).
// and How to Configure Website Page Redirects (https://docs.aws.amazon.com/AmazonS3/latest/dev/how-to-page-redirect.html)
// in the Amazon S3 User Guide.
//
// This functionality is not supported for directory buckets.
WebsiteRedirectLocation *string `location:"header" locationName:"x-amz-website-redirect-location" type:"string"`
}
File diff suppressed because it is too large Load Diff
+20 -19
View File
@@ -3,15 +3,13 @@
// Package ssooidc provides the client and types for making API
// requests to AWS SSO OIDC.
//
// AWS IAM Identity Center (successor to AWS Single Sign-On) OpenID Connect
// (OIDC) is a web service that enables a client (such as AWS CLI or a native
// application) to register with IAM Identity Center. The service also enables
// the client to fetch the users access token upon successful authentication
// and authorization with IAM Identity Center.
// IAM Identity Center OpenID Connect (OIDC) is a web service that enables a
// client (such as CLI or a native application) to register with IAM Identity
// Center. The service also enables the client to fetch the users access
// token upon successful authentication and authorization with IAM Identity
// Center.
//
// Although AWS Single Sign-On was renamed, the sso and identitystore API namespaces
// will continue to retain their original name for backward compatibility purposes.
// For more information, see IAM Identity Center rename (https://docs.aws.amazon.com/singlesignon/latest/userguide/what-is.html#renamed).
// IAM Identity Center uses the sso and identitystore API namespaces.
//
// # Considerations for Using This Guide
//
@@ -22,21 +20,24 @@
// - The IAM Identity Center OIDC service currently implements only the portions
// of the OAuth 2.0 Device Authorization Grant standard (https://tools.ietf.org/html/rfc8628
// (https://tools.ietf.org/html/rfc8628)) that are necessary to enable single
// sign-on authentication with the AWS CLI. Support for other OIDC flows
// frequently needed for native applications, such as Authorization Code
// Flow (+ PKCE), will be addressed in future releases.
// sign-on authentication with the CLI.
//
// - The service emits only OIDC access tokens, such that obtaining a new
// token (For example, token refresh) requires explicit user re-authentication.
// - With older versions of the CLI, the service only emits OIDC access tokens,
// so to obtain a new token, users must explicitly re-authenticate. To access
// the OIDC flow that supports token refresh and doesnt require re-authentication,
// update to the latest CLI version (1.27.10 for CLI V1 and 2.9.0 for CLI
// V2) with support for OIDC token refresh and configurable IAM Identity
// Center session durations. For more information, see Configure Amazon Web
// Services access portal session duration (https://docs.aws.amazon.com/singlesignon/latest/userguide/configure-user-session.html).
//
// - The access tokens provided by this service grant access to all AWS account
// entitlements assigned to an IAM Identity Center user, not just a particular
// application.
// - The access tokens provided by this service grant access to all Amazon
// Web Services account entitlements assigned to an IAM Identity Center user,
// not just a particular application.
//
// - The documentation in this guide does not describe the mechanism to convert
// the access token into AWS Auth (“sigv4”) credentials for use with
// IAM-protected AWS service endpoints. For more information, see GetRoleCredentials
// (https://docs.aws.amazon.com/singlesignon/latest/PortalAPIReference/API_GetRoleCredentials.html)
// the access token into Amazon Web Services Auth (“sigv4”) credentials
// for use with IAM-protected Amazon Web Services service endpoints. For
// more information, see GetRoleCredentials (https://docs.aws.amazon.com/singlesignon/latest/PortalAPIReference/API_GetRoleCredentials.html)
// in the IAM Identity Center Portal API Reference Guide.
//
// For general information about IAM Identity Center, see What is IAM Identity
+16
View File
@@ -57,6 +57,13 @@ const (
// makes a CreateToken request with an invalid grant type.
ErrCodeInvalidGrantException = "InvalidGrantException"
// ErrCodeInvalidRedirectUriException for service response error code
// "InvalidRedirectUriException".
//
// Indicates that one or more redirect URI in the request is not supported for
// this operation.
ErrCodeInvalidRedirectUriException = "InvalidRedirectUriException"
// ErrCodeInvalidRequestException for service response error code
// "InvalidRequestException".
//
@@ -64,6 +71,13 @@ const (
// a required parameter might be missing or out of range.
ErrCodeInvalidRequestException = "InvalidRequestException"
// ErrCodeInvalidRequestRegionException for service response error code
// "InvalidRequestRegionException".
//
// Indicates that a token provided as input to the request was issued by and
// is only usable by calling IAM Identity Center endpoints in another region.
ErrCodeInvalidRequestRegionException = "InvalidRequestRegionException"
// ErrCodeInvalidScopeException for service response error code
// "InvalidScopeException".
//
@@ -99,7 +113,9 @@ var exceptionFromCode = map[string]func(protocol.ResponseMetadata) error{
"InvalidClientException": newErrorInvalidClientException,
"InvalidClientMetadataException": newErrorInvalidClientMetadataException,
"InvalidGrantException": newErrorInvalidGrantException,
"InvalidRedirectUriException": newErrorInvalidRedirectUriException,
"InvalidRequestException": newErrorInvalidRequestException,
"InvalidRequestRegionException": newErrorInvalidRequestRegionException,
"InvalidScopeException": newErrorInvalidScopeException,
"SlowDownException": newErrorSlowDownException,
"UnauthorizedClientException": newErrorUnauthorizedClientException,
+1 -1
View File
@@ -51,7 +51,7 @@ const (
func New(p client.ConfigProvider, cfgs ...*aws.Config) *SSOOIDC {
c := p.ClientConfig(EndpointsID, cfgs...)
if c.SigningNameDerived || len(c.SigningName) == 0 {
c.SigningName = "awsssooidc"
c.SigningName = "sso-oauth"
}
return newClient(*c.Config, c.Handlers, c.PartitionID, c.Endpoint, c.SigningRegion, c.SigningName, c.ResolvedRegion)
}
+16 -4
View File
@@ -1460,7 +1460,15 @@ type AssumeRoleInput struct {
// in the IAM User Guide.
PolicyArns []*PolicyDescriptorType `type:"list"`
// Reserved for future use.
// A list of previously acquired trusted context assertions in the format of
// a JSON array. The trusted context assertion is signed and encrypted by Amazon
// Web Services STS.
//
// The following is an example of a ProvidedContext value that includes a single
// trusted context assertion and the ARN of the context provider from which
// the trusted context assertion was generated.
//
// [{"ProviderArn":"arn:aws:iam::aws:contextProvider/IdentityCenter","ContextAssertion":"trusted-context-assertion"}]
ProvidedContexts []*ProvidedContext `type:"list"`
// The Amazon Resource Name (ARN) of the role to assume.
@@ -3405,14 +3413,18 @@ func (s *PolicyDescriptorType) SetArn(v string) *PolicyDescriptorType {
return s
}
// Reserved for future use.
// Contains information about the provided context. This includes the signed
// and encrypted trusted context assertion and the context provider ARN from
// which the trusted context assertion was generated.
type ProvidedContext struct {
_ struct{} `type:"structure"`
// Reserved for future use.
// The signed and encrypted trusted context assertion generated by the context
// provider. The trusted context assertion is signed and encrypted by Amazon
// Web Services STS.
ContextAssertion *string `min:"4" type:"string"`
// Reserved for future use.
// The context provider ARN from which the trusted context assertion was generated.
ProviderArn *string `min:"20" type:"string"`
}
+6 -3
View File
@@ -28,7 +28,8 @@ type CloneOptions struct {
// specified using the clone options parameter.
//
// Similar To:
// ceph fs subvolume snapshot clone <volume> --group_name=<group> <subvolume> <snapshot> <name> [...]
//
// ceph fs subvolume snapshot clone <volume> --group_name=<group> <subvolume> <snapshot> <name> [...]
func (fsa *FSAdmin) CloneSubVolumeSnapshot(volume, group, subvolume, snapshot, name string, o *CloneOptions) error {
m := map[string]string{
"prefix": "fs subvolume snapshot clone",
@@ -114,7 +115,8 @@ func parseCloneStatus(res response) (*CloneStatus, error) {
// CloneStatus returns data reporting the status of a subvolume clone.
//
// Similar To:
// ceph fs clone status <volume> --group_name=<group> <clone>
//
// ceph fs clone status <volume> --group_name=<group> <clone>
func (fsa *FSAdmin) CloneStatus(volume, group, clone string) (*CloneStatus, error) {
m := map[string]string{
"prefix": "fs clone status",
@@ -132,7 +134,8 @@ func (fsa *FSAdmin) CloneStatus(volume, group, clone string) (*CloneStatus, erro
// CancelClone does not delete the clone.
//
// Similar To:
// ceph fs clone cancel <volume> --group_name=<group> <clone>
//
// ceph fs clone cancel <volume> --group_name=<group> <clone>
func (fsa *FSAdmin) CancelClone(volume, group, clone string) error {
m := map[string]string{
"prefix": "fs clone cancel",
+3 -2
View File
@@ -3,8 +3,9 @@ package admin
// GetFailure returns details about the CloneStatus when in CloneFailed state.
//
// Similar To:
// Reading the .failure object from the JSON returned by "ceph fs subvolume
// snapshot clone"
//
// Reading the .failure object from the JSON returned by "ceph fs subvolume
// snapshot clone"
func (cs *CloneStatus) GetFailure() *CloneFailure {
return cs.failure
}
+133
View File
@@ -0,0 +1,133 @@
package admin
import "fmt"
// fixedPointFloat is a custom type that implements the MarshalJSON interface.
// This is used to format float64 values to two decimal places.
// By default these get converted to integers in the JSON output and
// fail the command.
type fixedPointFloat float64
// MarshalJSON provides a custom implementation for the JSON marshalling
// of fixedPointFloat. It formats the float to two decimal places.
func (fpf fixedPointFloat) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf("%.2f", float64(fpf))), nil
}
// fSQuiesceFields is the internal type used to create JSON for ceph.
// See FSQuiesceOptions for the type that users of the library
// interact with.
type fSQuiesceFields struct {
Prefix string `json:"prefix"`
VolName string `json:"vol_name"`
GroupName string `json:"group_name,omitempty"`
Members []string `json:"members,omitempty"`
SetId string `json:"set_id,omitempty"`
Timeout fixedPointFloat `json:"timeout,omitempty"`
Expiration fixedPointFloat `json:"expiration,omitempty"`
AwaitFor fixedPointFloat `json:"await_for,omitempty"`
Await bool `json:"await,omitempty"`
IfVersion int `json:"if_version,omitempty"`
Include bool `json:"include,omitempty"`
Exclude bool `json:"exclude,omitempty"`
Reset bool `json:"reset,omitempty"`
Release bool `json:"release,omitempty"`
Query bool `json:"query,omitempty"`
All bool `json:"all,omitempty"`
Cancel bool `json:"cancel,omitempty"`
}
// FSQuiesceOptions are used to specify optional, non-identifying, values
// to be used when quiescing a cephfs volume.
type FSQuiesceOptions struct {
Timeout float64
Expiration float64
AwaitFor float64
Await bool
IfVersion int
Include bool
Exclude bool
Reset bool
Release bool
Query bool
All bool
Cancel bool
}
// toFields is used to convert the FSQuiesceOptions to the internal
// fSQuiesceFields type.
func (o *FSQuiesceOptions) toFields(volume, group string, subvolumes []string, setId string) *fSQuiesceFields {
return &fSQuiesceFields{
Prefix: "fs quiesce",
VolName: volume,
GroupName: group,
Members: subvolumes,
SetId: setId,
Timeout: fixedPointFloat(o.Timeout),
Expiration: fixedPointFloat(o.Expiration),
AwaitFor: fixedPointFloat(o.AwaitFor),
Await: o.Await,
IfVersion: o.IfVersion,
Include: o.Include,
Exclude: o.Exclude,
Reset: o.Reset,
Release: o.Release,
Query: o.Query,
All: o.All,
Cancel: o.Cancel,
}
}
// QuiesceState is used to report the state of a quiesced fs volume.
type QuiesceState struct {
Name string `json:"name"`
Age float64 `json:"age"`
}
// QuiesceInfoMember is used to report the state of a quiesced fs volume.
// This is part of sets members object array in the json.
type QuiesceInfoMember struct {
Excluded bool `json:"excluded"`
State QuiesceState `json:"state"`
}
// QuiesceInfo reports various informational values about a quiesced volume.
// This is returned as sets object array in the json.
type QuiesceInfo struct {
Version int `json:"version"`
AgeRef float64 `json:"age_ref"`
State QuiesceState `json:"state"`
Timeout float64 `json:"timeout"`
Expiration float64 `json:"expiration"`
Members map[string]QuiesceInfoMember `json:"members"`
}
// FSQuiesceInfo reports various informational values about quiesced volumes.
type FSQuiesceInfo struct {
Epoch int `json:"epoch"`
SetVersion int `json:"set_version"`
Sets map[string]QuiesceInfo `json:"sets"`
}
// parseFSQuiesceInfo is used to parse the response from the quiesce command. It returns a FSQuiesceInfo object.
func parseFSQuiesceInfo(res response) (*FSQuiesceInfo, error) {
var info FSQuiesceInfo
if err := res.NoStatus().Unmarshal(&info).End(); err != nil {
return nil, err
}
return &info, nil
}
// FSQuiesce will quiesce the specified subvolumes in a volume.
// Quiescing a fs will prevent new writes to the subvolumes.
// Similar To:
//
// ceph fs quiesce <volume>
func (fsa *FSAdmin) FSQuiesce(volume, group string, subvolumes []string, setId string, o *FSQuiesceOptions) (*FSQuiesceInfo, error) {
if o == nil {
o = &FSQuiesceOptions{}
}
f := o.toFields(volume, group, subvolumes, setId)
return parseFSQuiesceInfo(fsa.marshalMgrCommand(f))
}
-20
View File
@@ -17,26 +17,6 @@ type FSAdmin struct {
conn RadosCommander
}
// New creates an FSAdmin automatically based on the default ceph
// configuration file. If more customization is needed, create a
// *rados.Conn as you see fit and use NewFromConn to use that
// connection with these administrative functions.
func New() (*FSAdmin, error) {
conn, err := rados.NewConn()
if err != nil {
return nil, err
}
err = conn.ReadDefaultConfigFile()
if err != nil {
return nil, err
}
err = conn.Connect()
if err != nil {
return nil, err
}
return NewFromConn(conn), nil
}
// NewFromConn creates an FSAdmin management object from a preexisting
// rados connection. The existing connection can be rados.Conn or any
// type implementing the RadosCommander interface. This may be useful
+12 -7
View File
@@ -1,5 +1,5 @@
//go:build !(nautilus || octopus) && ceph_preview && ceph_pre_quincy
// +build !nautilus,!octopus,ceph_preview,ceph_pre_quincy
//go:build !(nautilus || octopus || pacific)
// +build !nautilus,!octopus,!pacific
package admin
@@ -7,7 +7,8 @@ package admin
// an optional subvolume group based on provided key name.
//
// Similar To:
// ceph fs subvolume metadata get <vol_name> <sub_name> <key_name> [--group_name <subvol_group_name>]
//
// ceph fs subvolume metadata get <vol_name> <sub_name> <key_name> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) GetMetadata(volume, group, subvolume, key string) (string, error) {
m := map[string]string{
"prefix": "fs subvolume metadata get",
@@ -28,7 +29,8 @@ func (fsa *FSAdmin) GetMetadata(volume, group, subvolume, key string) (string, e
// an optional subvolume group as a key-value pair.
//
// Similar To:
// ceph fs subvolume metadata set <vol_name> <sub_name> <key_name> <value> [--group_name <subvol_group_name>]
//
// ceph fs subvolume metadata set <vol_name> <sub_name> <key_name> <value> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) SetMetadata(volume, group, subvolume, key, value string) error {
m := map[string]string{
"prefix": "fs subvolume metadata set",
@@ -50,7 +52,8 @@ func (fsa *FSAdmin) SetMetadata(volume, group, subvolume, key, value string) err
// belonging to an optional subvolume group using the metadata key.
//
// Similar To:
// ceph fs subvolume metadata rm <vol_name> <sub_name> <key_name> [--group_name <subvol_group_name>]
//
// ceph fs subvolume metadata rm <vol_name> <sub_name> <key_name> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) RemoveMetadata(volume, group, subvolume, key string) error {
return fsa.rmSubVolumeMetadata(volume, group, subvolume, key, commonRmFlags{})
}
@@ -60,7 +63,8 @@ func (fsa *FSAdmin) RemoveMetadata(volume, group, subvolume, key string) error {
// the metadata key.
//
// Similar To:
// ceph fs subvolume metadata rm <vol_name> <sub_name> <key_name> [--group_name <subvol_group_name>] --force
//
// ceph fs subvolume metadata rm <vol_name> <sub_name> <key_name> [--group_name <subvol_group_name>] --force
func (fsa *FSAdmin) ForceRemoveMetadata(volume, group, subvolume, key string) error {
return fsa.rmSubVolumeMetadata(volume, group, subvolume, key, commonRmFlags{force: true})
}
@@ -85,7 +89,8 @@ func (fsa *FSAdmin) rmSubVolumeMetadata(volume, group, subvolume, key string, o
// in a volume belonging to an optional subvolume group.
//
// Similar To:
// ceph fs subvolume metadata ls <vol_name> <sub_name> [--group_name <subvol_group_name>]
//
// ceph fs subvolume metadata ls <vol_name> <sub_name> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) ListMetadata(volume, group, subvolume string) (map[string]string, error) {
m := map[string]string{
"prefix": "fs subvolume metadata ls",
+4 -2
View File
@@ -9,7 +9,8 @@ const mirroring = "mirroring"
// EnableMirroringModule will enable the mirroring module for cephfs.
//
// Similar To:
// ceph mgr module enable mirroring [--force]
//
// ceph mgr module enable mirroring [--force]
func (fsa *FSAdmin) EnableMirroringModule(force bool) error {
mgradmin := manager.NewFromConn(fsa.conn)
return mgradmin.EnableModule(mirroring, force)
@@ -18,7 +19,8 @@ func (fsa *FSAdmin) EnableMirroringModule(force bool) error {
// DisableMirroringModule will disable the mirroring module for cephfs.
//
// Similar To:
// ceph mgr module disable mirroring
//
// ceph mgr module disable mirroring
func (fsa *FSAdmin) DisableMirroringModule() error {
mgradmin := manager.NewFromConn(fsa.conn)
return mgradmin.DisableModule(mirroring)
+16 -8
View File
@@ -20,7 +20,8 @@ func (fsa *FSAdmin) SnapshotMirror() *SnapshotMirrorAdmin {
// Enable snapshot mirroring for the given file system.
//
// Similar To:
// ceph fs snapshot mirror enable <fs_name>
//
// ceph fs snapshot mirror enable <fs_name>
func (sma *SnapshotMirrorAdmin) Enable(fsname string) error {
m := map[string]string{
"prefix": "fs snapshot mirror enable",
@@ -33,7 +34,8 @@ func (sma *SnapshotMirrorAdmin) Enable(fsname string) error {
// Disable snapshot mirroring for the given file system.
//
// Similar To:
// ceph fs snapshot mirror disable <fs_name>
//
// ceph fs snapshot mirror disable <fs_name>
func (sma *SnapshotMirrorAdmin) Disable(fsname string) error {
m := map[string]string{
"prefix": "fs snapshot mirror disable",
@@ -46,7 +48,8 @@ func (sma *SnapshotMirrorAdmin) Disable(fsname string) error {
// Add a path in the file system to be mirrored.
//
// Similar To:
// ceph fs snapshot mirror add <fs_name> <path>
//
// ceph fs snapshot mirror add <fs_name> <path>
func (sma *SnapshotMirrorAdmin) Add(fsname, path string) error {
m := map[string]string{
"prefix": "fs snapshot mirror add",
@@ -60,7 +63,8 @@ func (sma *SnapshotMirrorAdmin) Add(fsname, path string) error {
// Remove a path in the file system from mirroring.
//
// Similar To:
// ceph fs snapshot mirror remove <fs_name> <path>
//
// ceph fs snapshot mirror remove <fs_name> <path>
func (sma *SnapshotMirrorAdmin) Remove(fsname, path string) error {
m := map[string]string{
"prefix": "fs snapshot mirror remove",
@@ -79,7 +83,8 @@ type bootstrapTokenResponse struct {
// a peering association between this site an another site.
//
// Similar To:
// ceph fs snapshot mirror peer_bootstrap create <fs_name> <client_entity> <site-name>
//
// ceph fs snapshot mirror peer_bootstrap create <fs_name> <client_entity> <site-name>
func (sma *SnapshotMirrorAdmin) CreatePeerBootstrapToken(
fsname, client, site string) (string, error) {
m := map[string]string{
@@ -100,7 +105,8 @@ func (sma *SnapshotMirrorAdmin) CreatePeerBootstrapToken(
// that has provided a token, with the current site.
//
// Similar To:
// ceph fs snapshot mirror peer_bootstrap import <fs_name> <token>
//
// ceph fs snapshot mirror peer_bootstrap import <fs_name> <token>
func (sma *SnapshotMirrorAdmin) ImportPeerBoostrapToken(fsname, token string) error {
m := map[string]string{
"prefix": "fs snapshot mirror peer_bootstrap import",
@@ -170,7 +176,8 @@ func parseDaemonStatus(res response) (DaemonStatusResults, error) {
// associated with the given file system.
//
// Similar To:
// ceph fs snapshot mirror daemon status <fs_name>
//
// ceph fs snapshot mirror daemon status <fs_name>
func (sma *SnapshotMirrorAdmin) DaemonStatus(fsname string) (
DaemonStatusResults, error) {
// ---
@@ -204,7 +211,8 @@ func parsePeerList(res response) (PeerListResults, error) {
// PeerList returns information about peers associated with the given file system.
//
// Similar To:
// ceph fs snapshot mirror peer_list <fs_name>
//
// ceph fs snapshot mirror peer_list <fs_name>
func (sma *SnapshotMirrorAdmin) PeerList(fsname string) (
PeerListResults, error) {
// ---
+46
View File
@@ -0,0 +1,46 @@
//go:build !nautilus
// +build !nautilus
package admin
// PinSubVolume pins subvolume to ranks according to policies. A valid pin
// setting value depends on the type of pin as described in the docs from
// https://docs.ceph.com/en/latest/cephfs/multimds/#cephfs-pinning and
// https://docs.ceph.com/en/latest/cephfs/multimds/#setting-subtree-partitioning-policies
//
// Similar To:
//
// ceph fs subvolume pin <vol_name> <sub_name> <pin_type> <pin_setting>
func (fsa *FSAdmin) PinSubVolume(volume, subvolume, pintype, pinsetting string) (string, error) {
m := map[string]string{
"prefix": "fs subvolume pin",
"format": "json",
"vol_name": volume,
"sub_name": subvolume,
"pin_type": pintype,
"pin_setting": pinsetting,
}
return parsePathResponse(fsa.marshalMgrCommand(m))
}
// PinSubVolumeGroup pins subvolume to ranks according to policies. A valid pin
// setting value depends on the type of pin as described in the docs from
// https://docs.ceph.com/en/latest/cephfs/multimds/#cephfs-pinning and
// https://docs.ceph.com/en/latest/cephfs/multimds/#setting-subtree-partitioning-policies
//
// Similar To:
//
// ceph fs subvolumegroup pin <vol_name> <group_name> <pin_type> <pin_setting>
func (fsa *FSAdmin) PinSubVolumeGroup(volume, group, pintype, pinsetting string) (string, error) {
m := map[string]string{
"prefix": "fs subvolumegroup pin",
"format": "json",
"vol_name": volume,
"group_name": group,
"pin_type": pintype,
"pin_setting": pinsetting,
}
return parsePathResponse(fsa.marshalMgrCommand(m))
}
+12 -7
View File
@@ -1,5 +1,5 @@
//go:build !(nautilus || octopus) && ceph_preview && ceph_pre_quincy
// +build !nautilus,!octopus,ceph_preview,ceph_pre_quincy
//go:build !(nautilus || octopus || pacific)
// +build !nautilus,!octopus,!pacific
package admin
@@ -7,7 +7,8 @@ package admin
// volume belonging to an optional subvolume group based on provided key name.
//
// Similar To:
// ceph fs subvolume snapshot metadata get <vol_name> <sub_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
//
// ceph fs subvolume snapshot metadata get <vol_name> <sub_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) GetSnapshotMetadata(volume, group, subvolume, snapname, key string) (string, error) {
m := map[string]string{
"prefix": "fs subvolume snapshot metadata get",
@@ -29,7 +30,8 @@ func (fsa *FSAdmin) GetSnapshotMetadata(volume, group, subvolume, snapname, key
// volume belonging to an optional subvolume group as a key-value pair.
//
// Similar To:
// ceph fs subvolume snapshot metadata set <vol_name> <sub_name> <snap_name> <key_name> <value> [--group_name <subvol_group_name>]
//
// ceph fs subvolume snapshot metadata set <vol_name> <sub_name> <snap_name> <key_name> <value> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) SetSnapshotMetadata(volume, group, subvolume, snapname, key, value string) error {
m := map[string]string{
"prefix": "fs subvolume snapshot metadata set",
@@ -53,7 +55,8 @@ func (fsa *FSAdmin) SetSnapshotMetadata(volume, group, subvolume, snapname, key,
// metadata key.
//
// Similar To:
// ceph fs subvolume snapshot metadata rm <vol_name> <sub_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
//
// ceph fs subvolume snapshot metadata rm <vol_name> <sub_name> <snap_name> <key_name> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) RemoveSnapshotMetadata(volume, group, subvolume, snapname, key string) error {
return fsa.rmSubVolumeSnapShotMetadata(volume, group, subvolume, snapname, key, commonRmFlags{})
}
@@ -63,7 +66,8 @@ func (fsa *FSAdmin) RemoveSnapshotMetadata(volume, group, subvolume, snapname, k
// subvolume group using the metadata key.
//
// Similar To:
// ceph fs subvolume snapshot metadata rm <vol_name> <sub_name> <snap_name> <key_name> [--group_name <subvol_group_name>] --force
//
// ceph fs subvolume snapshot metadata rm <vol_name> <sub_name> <snap_name> <key_name> [--group_name <subvol_group_name>] --force
func (fsa *FSAdmin) ForceRemoveSnapshotMetadata(volume, group, subvolume, snapname, key string) error {
return fsa.rmSubVolumeSnapShotMetadata(volume, group, subvolume, snapname, key, commonRmFlags{force: true})
}
@@ -89,7 +93,8 @@ func (fsa *FSAdmin) rmSubVolumeSnapShotMetadata(volume, group, subvolume, snapna
// snapshot in a volume belonging to an optional subvolume group.
//
// Similar To:
// ceph fs subvolume snapshot metadata ls <vol_name> <sub_name> <snap_name> [--group_name <subvol_group_name>]
//
// ceph fs subvolume snapshot metadata ls <vol_name> <sub_name> <snap_name> [--group_name <subvol_group_name>]
func (fsa *FSAdmin) ListSnapshotMetadata(volume, group, subvolume, snapname string) (map[string]string, error) {
m := map[string]string{
"prefix": "fs subvolume snapshot metadata ls",
+30 -15
View File
@@ -53,7 +53,8 @@ const NoGroup = ""
// belonging to an optional subvolume group.
//
// Similar To:
// ceph fs subvolume create <volume> --group-name=<group> <name> ...
//
// ceph fs subvolume create <volume> --group-name=<group> <name> ...
func (fsa *FSAdmin) CreateSubVolume(volume, group, name string, o *SubVolumeOptions) error {
if o == nil {
o = &SubVolumeOptions{}
@@ -66,7 +67,8 @@ func (fsa *FSAdmin) CreateSubVolume(volume, group, name string, o *SubVolumeOpti
// optional subvolume group.
//
// Similar To:
// ceph fs subvolume ls <volume> --group-name=<group>
//
// ceph fs subvolume ls <volume> --group-name=<group>
func (fsa *FSAdmin) ListSubVolumes(volume, group string) ([]string, error) {
m := map[string]string{
"prefix": "fs subvolume ls",
@@ -83,7 +85,8 @@ func (fsa *FSAdmin) ListSubVolumes(volume, group string) ([]string, error) {
// subvolume group.
//
// Similar To:
// ceph fs subvolume rm <volume> --group-name=<group> <name>
//
// ceph fs subvolume rm <volume> --group-name=<group> <name>
func (fsa *FSAdmin) RemoveSubVolume(volume, group, name string) error {
return fsa.RemoveSubVolumeWithFlags(volume, group, name, SubVolRmFlags{})
}
@@ -92,7 +95,8 @@ func (fsa *FSAdmin) RemoveSubVolume(volume, group, name string) error {
// subvolume group.
//
// Similar To:
// ceph fs subvolume rm <volume> --group-name=<group> <name> --force
//
// ceph fs subvolume rm <volume> --group-name=<group> <name> --force
func (fsa *FSAdmin) ForceRemoveSubVolume(volume, group, name string) error {
return fsa.RemoveSubVolumeWithFlags(volume, group, name, SubVolRmFlags{Force: true})
}
@@ -104,7 +108,8 @@ func (fsa *FSAdmin) ForceRemoveSubVolume(volume, group, name string) error {
// Equivalent to ForceRemoveSubVolume if only the "Force" flag is set.
//
// Similar To:
// ceph fs subvolume rm <volume> --group-name=<group> <name> [...flags...]
//
// ceph fs subvolume rm <volume> --group-name=<group> <name> [...flags...]
func (fsa *FSAdmin) RemoveSubVolumeWithFlags(volume, group, name string, o SubVolRmFlags) error {
m := map[string]string{
"prefix": "fs subvolume rm",
@@ -141,7 +146,8 @@ type SubVolumeResizeResult struct {
// prevent reducing the size of the volume below the current used size.
//
// Similar To:
// ceph fs subvolume resize <volume> --group-name=<group> <name> ...
//
// ceph fs subvolume resize <volume> --group-name=<group> <name> ...
func (fsa *FSAdmin) ResizeSubVolume(
volume, group, name string,
newSize QuotaSize, noShrink bool) (*SubVolumeResizeResult, error) {
@@ -166,7 +172,8 @@ func (fsa *FSAdmin) ResizeSubVolume(
// SubVolumePath returns the path to the subvolume from the root of the file system.
//
// Similar To:
// ceph fs subvolume getpath <volume> --group-name=<group> <name>
//
// ceph fs subvolume getpath <volume> --group-name=<group> <name>
func (fsa *FSAdmin) SubVolumePath(volume, group, name string) (string, error) {
m := map[string]string{
"prefix": "fs subvolume getpath",
@@ -258,7 +265,8 @@ func parseSubVolumeInfo(res response) (*SubVolumeInfo, error) {
// SubVolumeInfo returns information about the specified subvolume.
//
// Similar To:
// ceph fs subvolume info <volume> --group-name=<group> <name>
//
// ceph fs subvolume info <volume> --group-name=<group> <name>
func (fsa *FSAdmin) SubVolumeInfo(volume, group, name string) (*SubVolumeInfo, error) {
m := map[string]string{
"prefix": "fs subvolume info",
@@ -275,7 +283,8 @@ func (fsa *FSAdmin) SubVolumeInfo(volume, group, name string) (*SubVolumeInfo, e
// CreateSubVolumeSnapshot creates a new snapshot from the source subvolume.
//
// Similar To:
// ceph fs subvolume snapshot create <volume> --group-name=<group> <source> <name>
//
// ceph fs subvolume snapshot create <volume> --group-name=<group> <source> <name>
func (fsa *FSAdmin) CreateSubVolumeSnapshot(volume, group, source, name string) error {
m := map[string]string{
"prefix": "fs subvolume snapshot create",
@@ -293,7 +302,8 @@ func (fsa *FSAdmin) CreateSubVolumeSnapshot(volume, group, source, name string)
// RemoveSubVolumeSnapshot removes the specified snapshot from the subvolume.
//
// Similar To:
// ceph fs subvolume snapshot rm <volume> --group-name=<group> <subvolume> <name>
//
// ceph fs subvolume snapshot rm <volume> --group-name=<group> <subvolume> <name>
func (fsa *FSAdmin) RemoveSubVolumeSnapshot(volume, group, subvolume, name string) error {
return fsa.rmSubVolumeSnapshot(volume, group, subvolume, name, commonRmFlags{})
}
@@ -301,7 +311,8 @@ func (fsa *FSAdmin) RemoveSubVolumeSnapshot(volume, group, subvolume, name strin
// ForceRemoveSubVolumeSnapshot removes the specified snapshot from the subvolume.
//
// Similar To:
// ceph fs subvolume snapshot rm <volume> --group-name=<group> <subvolume> <name> --force
//
// ceph fs subvolume snapshot rm <volume> --group-name=<group> <subvolume> <name> --force
func (fsa *FSAdmin) ForceRemoveSubVolumeSnapshot(volume, group, subvolume, name string) error {
return fsa.rmSubVolumeSnapshot(volume, group, subvolume, name, commonRmFlags{force: true})
}
@@ -324,7 +335,8 @@ func (fsa *FSAdmin) rmSubVolumeSnapshot(volume, group, subvolume, name string, o
// ListSubVolumeSnapshots returns a listing of snapshots for a given subvolume.
//
// Similar To:
// ceph fs subvolume snapshot ls <volume> --group-name=<group> <name>
//
// ceph fs subvolume snapshot ls <volume> --group-name=<group> <name>
func (fsa *FSAdmin) ListSubVolumeSnapshots(volume, group, name string) ([]string, error) {
m := map[string]string{
"prefix": "fs subvolume snapshot ls",
@@ -358,7 +370,8 @@ func parseSubVolumeSnapshotInfo(res response) (*SubVolumeSnapshotInfo, error) {
// SubVolumeSnapshotInfo returns information about the specified subvolume snapshot.
//
// Similar To:
// ceph fs subvolume snapshot info <volume> --group-name=<group> <subvolume> <name>
//
// ceph fs subvolume snapshot info <volume> --group-name=<group> <subvolume> <name>
func (fsa *FSAdmin) SubVolumeSnapshotInfo(volume, group, subvolume, name string) (*SubVolumeSnapshotInfo, error) {
m := map[string]string{
"prefix": "fs subvolume snapshot info",
@@ -376,7 +389,8 @@ func (fsa *FSAdmin) SubVolumeSnapshotInfo(volume, group, subvolume, name string)
// ProtectSubVolumeSnapshot protects the specified snapshot.
//
// Similar To:
// ceph fs subvolume snapshot protect <volume> --group-name=<group> <subvolume> <name>
//
// ceph fs subvolume snapshot protect <volume> --group-name=<group> <subvolume> <name>
func (fsa *FSAdmin) ProtectSubVolumeSnapshot(volume, group, subvolume, name string) error {
m := map[string]string{
"prefix": "fs subvolume snapshot protect",
@@ -394,7 +408,8 @@ func (fsa *FSAdmin) ProtectSubVolumeSnapshot(volume, group, subvolume, name stri
// UnprotectSubVolumeSnapshot removes protection from the specified snapshot.
//
// Similar To:
// ceph fs subvolume snapshot unprotect <volume> --group-name=<group> <subvolume> <name>
//
// ceph fs subvolume snapshot unprotect <volume> --group-name=<group> <subvolume> <name>
func (fsa *FSAdmin) UnprotectSubVolumeSnapshot(volume, group, subvolume, name string) error {
m := map[string]string{
"prefix": "fs subvolume snapshot unprotect",
+10 -5
View File
@@ -40,7 +40,8 @@ func (s *SubVolumeGroupOptions) toFields(v, g string) *subVolumeGroupFields {
// CreateSubVolumeGroup sends a request to create a subvolume group in a volume.
//
// Similar To:
// ceph fs subvolumegroup create <volume> <group_name> ...
//
// ceph fs subvolumegroup create <volume> <group_name> ...
func (fsa *FSAdmin) CreateSubVolumeGroup(volume, name string, o *SubVolumeGroupOptions) error {
if o == nil {
o = &SubVolumeGroupOptions{}
@@ -53,7 +54,8 @@ func (fsa *FSAdmin) CreateSubVolumeGroup(volume, name string, o *SubVolumeGroupO
// specified volume.
//
// Similar To:
// ceph fs subvolumegroup ls cephfs <volume>
//
// ceph fs subvolumegroup ls cephfs <volume>
func (fsa *FSAdmin) ListSubVolumeGroups(volume string) ([]string, error) {
res := fsa.marshalMgrCommand(map[string]string{
"prefix": "fs subvolumegroup ls",
@@ -65,14 +67,16 @@ func (fsa *FSAdmin) ListSubVolumeGroups(volume string) ([]string, error) {
// RemoveSubVolumeGroup will delete a subvolume group in a volume.
// Similar To:
// ceph fs subvolumegroup rm <volume> <group_name>
//
// ceph fs subvolumegroup rm <volume> <group_name>
func (fsa *FSAdmin) RemoveSubVolumeGroup(volume, name string) error {
return fsa.rmSubVolumeGroup(volume, name, commonRmFlags{})
}
// ForceRemoveSubVolumeGroup will delete a subvolume group in a volume.
// Similar To:
// ceph fs subvolumegroup rm <volume> <group_name> --force
//
// ceph fs subvolumegroup rm <volume> <group_name> --force
func (fsa *FSAdmin) ForceRemoveSubVolumeGroup(volume, name string) error {
return fsa.rmSubVolumeGroup(volume, name, commonRmFlags{force: true})
}
@@ -91,7 +95,8 @@ func (fsa *FSAdmin) rmSubVolumeGroup(volume, name string, o commonRmFlags) error
// file system.
//
// Similar To:
// ceph fs subvolumegroup getpath <volume> <group_name>
//
// ceph fs subvolumegroup getpath <volume> <group_name>
func (fsa *FSAdmin) SubVolumeGroupPath(volume, name string) (string, error) {
m := map[string]string{
"prefix": "fs subvolumegroup getpath",
+6 -3
View File
@@ -14,7 +14,8 @@ var (
// ListVolumes return a list of volumes in this Ceph cluster.
//
// Similar To:
// ceph fs volume ls
//
// ceph fs volume ls
func (fsa *FSAdmin) ListVolumes() ([]string, error) {
res := fsa.rawMgrCommand(listVolumesCmd)
return parseListNames(res)
@@ -34,7 +35,8 @@ type FSPoolInfo struct {
// file systems.
//
// Similar To:
// ceph fs ls
//
// ceph fs ls
func (fsa *FSAdmin) ListFileSystems() ([]FSPoolInfo, error) {
res := fsa.rawMonCommand(listFsCmd)
return parseFsList(res)
@@ -168,7 +170,8 @@ func parseVolumeStatus(res response) (*volumeStatusResponse, error) {
// VolumeStatus returns a VolumeStatus object for the given volume name.
//
// Similar To:
// ceph fs status cephfs <name>
//
// ceph fs status cephfs <name>
func (fsa *FSAdmin) VolumeStatus(name string) (*VolumeStatus, error) {
res := fsa.marshalMgrCommand(map[string]string{
"fs": name,
+48
View File
@@ -0,0 +1,48 @@
//go:build !(nautilus || octopus)
// +build !nautilus,!octopus
package admin
// PoolInfo reports various properties of a pool.
type PoolInfo struct {
Available int `json:"avail"`
Name string `json:"name"`
Used int `json:"used"`
}
// PoolType indicates the type of pool related to a volume.
type PoolType struct {
DataPool []PoolInfo `json:"data"`
MetadataPool []PoolInfo `json:"metadata"`
}
// VolInfo holds various informational values about a volume.
type VolInfo struct {
MonAddrs []string `json:"mon_addrs"`
PendingSubvolDels int `json:"pending_subvolume_deletions"`
Pools PoolType `json:"pools"`
UsedSize int `json:"used_size"`
}
func parseVolumeInfo(res response) (*VolInfo, error) {
var info VolInfo
if err := res.NoStatus().Unmarshal(&info).End(); err != nil {
return nil, err
}
return &info, nil
}
// FetchVolumeInfo fetches the information of a CephFS volume.
//
// Similar To:
//
// ceph fs volume info <vol_name>
func (fsa *FSAdmin) FetchVolumeInfo(volume string) (*VolInfo, error) {
m := map[string]string{
"prefix": "fs volume info",
"vol_name": volume,
"format": "json",
}
return parseVolumeInfo(fsa.marshalMgrCommand(m))
}
+24 -12
View File
@@ -61,7 +61,8 @@ func CreateMountWithId(id string) (*MountInfo, error) {
// connection.
//
// Implements:
// int ceph_create_from_rados(struct ceph_mount_info **cmount, rados_t cluster);
//
// int ceph_create_from_rados(struct ceph_mount_info **cmount, rados_t cluster);
func CreateFromRados(conn *rados.Conn) (*MountInfo, error) {
mount := &MountInfo{}
ret := C.ceph_create_from_rados(&mount.mount, C.rados_t(conn.Cluster()))
@@ -74,7 +75,8 @@ func CreateFromRados(conn *rados.Conn) (*MountInfo, error) {
// ReadDefaultConfigFile loads the ceph configuration from the default config file.
//
// Implements:
// int ceph_conf_read_file(struct ceph_mount_info *cmount, const char *path_list);
//
// int ceph_conf_read_file(struct ceph_mount_info *cmount, const char *path_list);
func (mount *MountInfo) ReadDefaultConfigFile() error {
ret := C.ceph_conf_read_file(mount.mount, nil)
return getError(ret)
@@ -83,7 +85,8 @@ func (mount *MountInfo) ReadDefaultConfigFile() error {
// ReadConfigFile loads the ceph configuration from the specified config file.
//
// Implements:
// int ceph_conf_read_file(struct ceph_mount_info *cmount, const char *path_list);
//
// int ceph_conf_read_file(struct ceph_mount_info *cmount, const char *path_list);
func (mount *MountInfo) ReadConfigFile(path string) error {
cPath := C.CString(path)
defer C.free(unsafe.Pointer(cPath))
@@ -95,7 +98,8 @@ func (mount *MountInfo) ReadConfigFile(path string) error {
// argument vector.
//
// Implements:
// int ceph_conf_parse_argv(struct ceph_mount_info *cmount, int argc, const char **argv);
//
// int ceph_conf_parse_argv(struct ceph_mount_info *cmount, int argc, const char **argv);
func (mount *MountInfo) ParseConfigArgv(argv []string) error {
if err := mount.validate(); err != nil {
return err
@@ -117,7 +121,8 @@ func (mount *MountInfo) ParseConfigArgv(argv []string) error {
// environment variable CEPH_ARGS.
//
// Implements:
// int ceph_conf_parse_env(struct ceph_mount_info *cmount, const char *var);
//
// int ceph_conf_parse_env(struct ceph_mount_info *cmount, const char *var);
func (mount *MountInfo) ParseDefaultConfigEnv() error {
if err := mount.validate(); err != nil {
return err
@@ -130,7 +135,8 @@ func (mount *MountInfo) ParseDefaultConfigEnv() error {
// the given name.
//
// Implements:
// int ceph_conf_set(struct ceph_mount_info *cmount, const char *option, const char *value);
//
// int ceph_conf_set(struct ceph_mount_info *cmount, const char *option, const char *value);
func (mount *MountInfo) SetConfigOption(option, value string) error {
cOption := C.CString(option)
defer C.free(unsafe.Pointer(cOption))
@@ -143,7 +149,8 @@ func (mount *MountInfo) SetConfigOption(option, value string) error {
// identified by the given name.
//
// Implements:
// int ceph_conf_get(struct ceph_mount_info *cmount, const char *option, char *buf, size_t len);
//
// int ceph_conf_get(struct ceph_mount_info *cmount, const char *option, char *buf, size_t len);
func (mount *MountInfo) GetConfigOption(option string) (string, error) {
cOption := C.CString(option)
defer C.free(unsafe.Pointer(cOption))
@@ -173,7 +180,8 @@ func (mount *MountInfo) GetConfigOption(option string) (string, error) {
// Init the file system client without actually mounting the file system.
//
// Implements:
// int ceph_init(struct ceph_mount_info *cmount);
//
// int ceph_init(struct ceph_mount_info *cmount);
func (mount *MountInfo) Init() error {
return getError(C.ceph_init(mount.mount))
}
@@ -181,7 +189,8 @@ func (mount *MountInfo) Init() error {
// Mount the file system, establishing a connection capable of I/O.
//
// Implements:
// int ceph_mount(struct ceph_mount_info *cmount, const char *root);
//
// int ceph_mount(struct ceph_mount_info *cmount, const char *root);
func (mount *MountInfo) Mount() error {
ret := C.ceph_mount(mount.mount, nil)
return getError(ret)
@@ -191,7 +200,8 @@ func (mount *MountInfo) Mount() error {
// the mount. This establishes a connection capable of I/O.
//
// Implements:
// int ceph_mount(struct ceph_mount_info *cmount, const char *root);
//
// int ceph_mount(struct ceph_mount_info *cmount, const char *root);
func (mount *MountInfo) MountWithRoot(root string) error {
croot := C.CString(root)
defer C.free(unsafe.Pointer(croot))
@@ -201,7 +211,8 @@ func (mount *MountInfo) MountWithRoot(root string) error {
// Unmount the file system.
//
// Implements:
// int ceph_unmount(struct ceph_mount_info *cmount);
//
// int ceph_unmount(struct ceph_mount_info *cmount);
func (mount *MountInfo) Unmount() error {
ret := C.ceph_unmount(mount.mount)
return getError(ret)
@@ -210,7 +221,8 @@ func (mount *MountInfo) Unmount() error {
// Release destroys the mount handle.
//
// Implements:
// int ceph_release(struct ceph_mount_info *cmount);
//
// int ceph_release(struct ceph_mount_info *cmount);
func (mount *MountInfo) Release() error {
if mount.mount == nil {
return nil
+8 -7
View File
@@ -32,13 +32,14 @@ func (mount *MountInfo) MdsCommandWithInputBuffer(mdsSpec string, args [][]byte,
// mdsCommand supports sending formatted commands to MDS.
//
// Implements:
// int ceph_mds_command(struct ceph_mount_info *cmount,
// const char *mds_spec,
// const char **cmd,
// size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
//
// int ceph_mds_command(struct ceph_mount_info *cmount,
// const char *mds_spec,
// const char **cmd,
// size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
func (mount *MountInfo) mdsCommand(mdsSpec string, args [][]byte, inputBuffer []byte) (buffer []byte, info string, err error) {
spec := C.CString(mdsSpec)
defer C.free(unsafe.Pointer(spec))
+4 -2
View File
@@ -15,10 +15,12 @@ import "C"
// will be returned.
//
// Note:
// Only supported in Ceph Nautilus and newer.
//
// Only supported in Ceph Nautilus and newer.
//
// Implements:
// int64_t ceph_get_fs_cid(struct ceph_mount_info *cmount);
//
// int64_t ceph_get_fs_cid(struct ceph_mount_info *cmount);
func (mount *MountInfo) GetFsCid() (int64, error) {
ret := C.ceph_get_fs_cid(mount.mount)
if ret < 0 {
+11 -6
View File
@@ -22,7 +22,8 @@ type Directory struct {
// OpenDir returns a new Directory handle open for I/O.
//
// Implements:
// int ceph_opendir(struct ceph_mount_info *cmount, const char *name, struct ceph_dir_result **dirpp);
//
// int ceph_opendir(struct ceph_mount_info *cmount, const char *name, struct ceph_dir_result **dirpp);
func (mount *MountInfo) OpenDir(path string) (*Directory, error) {
var dir *C.struct_ceph_dir_result
@@ -43,7 +44,8 @@ func (mount *MountInfo) OpenDir(path string) (*Directory, error) {
// Close the open directory handle.
//
// Implements:
// int ceph_closedir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
//
// int ceph_closedir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
func (dir *Directory) Close() error {
return getError(C.ceph_closedir(dir.mount.mount, dir.dir))
}
@@ -136,7 +138,8 @@ func toDirEntryPlus(de *C.struct_dirent, s C.struct_ceph_statx) *DirEntryPlus {
// exhausted.
//
// Implements:
// int ceph_readdir_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de);
//
// int ceph_readdir_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de);
func (dir *Directory) ReadDir() (*DirEntry, error) {
var de C.struct_dirent
ret := C.ceph_readdir_r(dir.mount.mount, dir.dir, &de)
@@ -156,8 +159,9 @@ func (dir *Directory) ReadDir() (*DirEntry, error) {
// See Statx for a description of the wants and flags parameters.
//
// Implements:
// int ceph_readdirplus_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de,
// struct ceph_statx *stx, unsigned want, unsigned flags, struct Inode **out);
//
// int ceph_readdirplus_r(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp, struct dirent *de,
// struct ceph_statx *stx, unsigned want, unsigned flags, struct Inode **out);
func (dir *Directory) ReadDirPlus(
want StatxMask, flags AtFlags) (*DirEntryPlus, error) {
@@ -186,7 +190,8 @@ func (dir *Directory) ReadDirPlus(
// RewindDir sets the directory stream to the beginning of the directory.
//
// Implements:
// void ceph_rewinddir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
//
// void ceph_rewinddir(struct ceph_mount_info *cmount, struct ceph_dir_result *dirp);
func (dir *Directory) RewindDir() {
C.ceph_rewinddir(dir.mount.mount, dir.dir)
}
+8 -28
View File
@@ -11,23 +11,8 @@ import (
"github.com/ceph/go-ceph/internal/errutil"
)
// cephFSError represents an error condition returned from the CephFS APIs.
type cephFSError int
// Error returns the error string for the cephFSError type.
func (e cephFSError) Error() string {
return errutil.FormatErrorCode("cephfs", int(e))
}
func (e cephFSError) ErrorCode() int {
return int(e)
}
func getError(e C.int) error {
if e == 0 {
return nil
}
return cephFSError(e)
return errutil.GetError("cephfs", int(e))
}
// getErrorIfNegative converts a ceph return code to error if negative.
@@ -46,21 +31,16 @@ var (
// ErrEmptyArgument may be returned if a function argument is passed
// a zero-length slice or map.
ErrEmptyArgument = errors.New("Argument must contain at least one item")
)
// Public CephFSErrors:
const (
// ErrNotConnected may be returned when client is not connected
// to a cluster.
ErrNotConnected = cephFSError(-C.ENOTCONN)
)
ErrNotConnected = getError(-C.ENOTCONN)
// ErrNotExist indicates a non-specific missing resource.
ErrNotExist = getError(-C.ENOENT)
// Private errors:
// Private errors:
const (
errInvalid = cephFSError(-C.EINVAL)
errNameTooLong = cephFSError(-C.ENAMETOOLONG)
errNoEntry = cephFSError(-C.ENOENT)
errRange = cephFSError(-C.ERANGE)
errInvalid = getError(-C.EINVAL)
errNameTooLong = getError(-C.ENAMETOOLONG)
errRange = getError(-C.ERANGE)
)
+31 -17
View File
@@ -48,7 +48,8 @@ type File struct {
// a local open call. Mode is the same mode bits as a local open call.
//
// Implements:
// int ceph_open(struct ceph_mount_info *cmount, const char *path, int flags, mode_t mode);
//
// int ceph_open(struct ceph_mount_info *cmount, const char *path, int flags, mode_t mode);
func (mount *MountInfo) Open(path string, flags int, mode uint32) (*File, error) {
if mount.mount == nil {
return nil, ErrNotConnected
@@ -72,7 +73,8 @@ func (f *File) validate() error {
// Close the file.
//
// Implements:
// int ceph_close(struct ceph_mount_info *cmount, int fd);
//
// int ceph_close(struct ceph_mount_info *cmount, int fd);
func (f *File) Close() error {
if f.fd == -1 {
// already closed
@@ -93,7 +95,8 @@ func (f *File) Close() error {
// wrappers for external callers of the library.
//
// Implements:
// int ceph_read(struct ceph_mount_info *cmount, int fd, char *buf, int64_t size, int64_t offset);
//
// int ceph_read(struct ceph_mount_info *cmount, int fd, char *buf, int64_t size, int64_t offset);
func (f *File) read(buf []byte, offset int64) (int, error) {
if err := f.validate(); err != nil {
return 0, err
@@ -137,8 +140,9 @@ func (f *File) ReadAt(buf []byte, offset int64) (int, error) {
// 0, io.EOF.
//
// Implements:
// int ceph_preadv(struct ceph_mount_info *cmount, int fd, const struct iovec *iov, int iovcnt,
// int64_t offset);
//
// int ceph_preadv(struct ceph_mount_info *cmount, int fd, const struct iovec *iov, int iovcnt,
// int64_t offset);
func (f *File) Preadv(data [][]byte, offset int64) (int, error) {
if err := f.validate(); err != nil {
return 0, err
@@ -167,8 +171,9 @@ func (f *File) Preadv(data [][]byte, offset int64) (int, error) {
// wrappers for external callers of the library.
//
// Implements:
// int ceph_write(struct ceph_mount_info *cmount, int fd, const char *buf,
// int64_t size, int64_t offset);
//
// int ceph_write(struct ceph_mount_info *cmount, int fd, const char *buf,
// int64_t size, int64_t offset);
func (f *File) write(buf []byte, offset int64) (int, error) {
if err := f.validate(); err != nil {
return 0, err
@@ -202,8 +207,9 @@ func (f *File) WriteAt(buf []byte, offset int64) (int, error) {
// The number of bytes written is returned.
//
// Implements:
// int ceph_pwritev(struct ceph_mount_info *cmount, int fd, const struct iovec *iov, int iovcnt,
// int64_t offset);
//
// int ceph_pwritev(struct ceph_mount_info *cmount, int fd, const struct iovec *iov, int iovcnt,
// int64_t offset);
func (f *File) Pwritev(data [][]byte, offset int64) (int, error) {
if err := f.validate(); err != nil {
return 0, err
@@ -226,7 +232,8 @@ func (f *File) Pwritev(data [][]byte, offset int64) (int, error) {
// Seek will reposition the file stream based on the given offset.
//
// Implements:
// int64_t ceph_lseek(struct ceph_mount_info *cmount, int fd, int64_t offset, int whence);
//
// int64_t ceph_lseek(struct ceph_mount_info *cmount, int fd, int64_t offset, int whence);
func (f *File) Seek(offset int64, whence int) (int64, error) {
if err := f.validate(); err != nil {
return 0, err
@@ -250,7 +257,8 @@ func (f *File) Seek(offset int64, whence int) (int64, error) {
// Fchmod changes the mode bits (permissions) of a file.
//
// Implements:
// int ceph_fchmod(struct ceph_mount_info *cmount, int fd, mode_t mode);
//
// int ceph_fchmod(struct ceph_mount_info *cmount, int fd, mode_t mode);
func (f *File) Fchmod(mode uint32) error {
if err := f.validate(); err != nil {
return err
@@ -263,7 +271,8 @@ func (f *File) Fchmod(mode uint32) error {
// Fchown changes the ownership of a file.
//
// Implements:
// int ceph_fchown(struct ceph_mount_info *cmount, int fd, int uid, int gid);
//
// int ceph_fchown(struct ceph_mount_info *cmount, int fd, int uid, int gid);
func (f *File) Fchown(user uint32, group uint32) error {
if err := f.validate(); err != nil {
return err
@@ -276,8 +285,9 @@ func (f *File) Fchown(user uint32, group uint32) error {
// Fstatx returns information about an open file.
//
// Implements:
// int ceph_fstatx(struct ceph_mount_info *cmount, int fd, struct ceph_statx *stx,
// unsigned int want, unsigned int flags);
//
// int ceph_fstatx(struct ceph_mount_info *cmount, int fd, struct ceph_statx *stx,
// unsigned int want, unsigned int flags);
func (f *File) Fstatx(want StatxMask, flags AtFlags) (*CephStatx, error) {
if err := f.validate(); err != nil {
return nil, err
@@ -317,6 +327,7 @@ const (
// on the given range.
//
// Implements:
//
// int ceph_fallocate(struct ceph_mount_info *cmount, int fd, int mode,
// int64_t offset, int64_t length);
func (f *File) Fallocate(mode FallocFlags, offset, length int64) error {
@@ -348,7 +359,8 @@ const (
// lock, must be an arbitrary integer.
//
// Implements:
// int ceph_flock(struct ceph_mount_info *cmount, int fd, int operation, uint64_t owner);
//
// int ceph_flock(struct ceph_mount_info *cmount, int fd, int operation, uint64_t owner);
func (f *File) Flock(operation LockOp, owner uint64) error {
if err := f.validate(); err != nil {
return err
@@ -372,7 +384,8 @@ func (f *File) Flock(operation LockOp, owner uint64) error {
// Pass SyncDataOnly to have this call behave more like fdatasync (on linux).
//
// Implements:
// int ceph_fsync(struct ceph_mount_info *cmount, int fd, int syncdataonly);
//
// int ceph_fsync(struct ceph_mount_info *cmount, int fd, int syncdataonly);
func (f *File) Fsync(sync SyncChoice) error {
if err := f.validate(); err != nil {
return err
@@ -401,7 +414,8 @@ func (f *File) Sync() error {
// https://tracker.ceph.com/issues/48202
//
// Implements:
// int ceph_ftruncate(struct ceph_mount_info *cmount, int fd, int64_t size);
//
// int ceph_ftruncate(struct ceph_mount_info *cmount, int fd, int64_t size);
func (f *File) Truncate(size int64) error {
if err := f.validate(); err != nil {
return err
+132
View File
@@ -0,0 +1,132 @@
//go:build !nautilus
// +build !nautilus
package cephfs
/*
#cgo LDFLAGS: -lcephfs
#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
#include <errno.h>
#include <stdlib.h>
#include <cephfs/libcephfs.h>
*/
import "C"
import (
ts "github.com/ceph/go-ceph/internal/timespec"
"unsafe"
)
// Mknod creates a regular, block or character special file.
//
// Implements:
//
// int ceph_mknod(struct ceph_mount_info *cmount, const char *path, mode_t mode,
// dev_t rdev);
func (mount *MountInfo) Mknod(path string, mode uint16, dev uint16) error {
if err := mount.validate(); err != nil {
return err
}
cPath := C.CString(path)
defer C.free(unsafe.Pointer(cPath))
ret := C.ceph_mknod(mount.mount, cPath, C.mode_t(mode), C.dev_t(dev))
return getError(ret)
}
// Utime struct is the equivalent of C.struct_utimbuf
type Utime struct {
// AcTime represents the file's access time in seconds since the Unix epoch.
AcTime int64
// ModTime represents the file's modification time in seconds since the Unix epoch.
ModTime int64
}
// Futime changes file/directory last access and modification times.
//
// Implements:
//
// int ceph_futime(struct ceph_mount_info *cmount, int fd, struct utimbuf *buf);
func (mount *MountInfo) Futime(fd int, times *Utime) error {
if err := mount.validate(); err != nil {
return err
}
cFd := C.int(fd)
uTimeBuf := &C.struct_utimbuf{
actime: C.time_t(times.AcTime),
modtime: C.time_t(times.ModTime),
}
ret := C.ceph_futime(mount.mount, cFd, uTimeBuf)
return getError(ret)
}
// Timeval struct is the go equivalent of C.struct_timeval type
type Timeval struct {
// Sec represents seconds
Sec int64
// USec represents microseconds
USec int64
}
// Futimens changes file/directory last access and modification times, here times param
// is an array of Timespec struct having length 2, where times[0] represents the access time
// and times[1] represents the modification time.
//
// Implements:
//
// int ceph_futimens(struct ceph_mount_info *cmount, int fd, struct timespec times[2]);
func (mount *MountInfo) Futimens(fd int, times []Timespec) error {
if err := mount.validate(); err != nil {
return err
}
if len(times) != 2 {
return getError(-C.EINVAL)
}
cFd := C.int(fd)
cTimes := []C.struct_timespec{}
for _, val := range times {
cTs := &C.struct_timespec{}
ts.CopyToCStruct(
ts.Timespec(val),
ts.CTimespecPtr(cTs),
)
cTimes = append(cTimes, *cTs)
}
ret := C.ceph_futimens(mount.mount, cFd, &cTimes[0])
return getError(ret)
}
// Futimes changes file/directory last access and modification times, here times param
// is an array of Timeval struct type having length 2, where times[0] represents the access time
// and times[1] represents the modification time.
//
// Implements:
//
// int ceph_futimes(struct ceph_mount_info *cmount, int fd, struct timeval times[2]);
func (mount *MountInfo) Futimes(fd int, times []Timeval) error {
if err := mount.validate(); err != nil {
return err
}
if len(times) != 2 {
return getError(-C.EINVAL)
}
cFd := C.int(fd)
cTimes := []C.struct_timeval{}
for _, val := range times {
cTimes = append(cTimes, C.struct_timeval{
tv_sec: C.time_t(val.Sec),
tv_usec: C.suseconds_t(val.USec),
})
}
ret := C.ceph_futimes(mount.mount, cFd, &cTimes[0])
return getError(ret)
}
+10 -6
View File
@@ -39,8 +39,9 @@ const (
// Please refer to https://tracker.ceph.com/issues/46084
//
// Implements:
// int ceph_fsetxattr(struct ceph_mount_info *cmount, int fd, const char *name,
// const void *value, size_t size, int flags);
//
// int ceph_fsetxattr(struct ceph_mount_info *cmount, int fd, const char *name,
// const void *value, size_t size, int flags);
func (f *File) SetXattr(name string, value []byte, flags XattrFlags) error {
if err := f.validate(); err != nil {
return err
@@ -68,8 +69,9 @@ func (f *File) SetXattr(name string, value []byte, flags XattrFlags) error {
// GetXattr gets an extended attribute from the open file.
//
// Implements:
// int ceph_fgetxattr(struct ceph_mount_info *cmount, int fd, const char *name,
// void *value, size_t size);
//
// int ceph_fgetxattr(struct ceph_mount_info *cmount, int fd, const char *name,
// void *value, size_t size);
func (f *File) GetXattr(name string) ([]byte, error) {
if err := f.validate(); err != nil {
return nil, err
@@ -107,7 +109,8 @@ func (f *File) GetXattr(name string) ([]byte, error) {
// on the file.
//
// Implements:
// int ceph_flistxattr(struct ceph_mount_info *cmount, int fd, char *list, size_t size);
//
// int ceph_flistxattr(struct ceph_mount_info *cmount, int fd, char *list, size_t size);
func (f *File) ListXattr() ([]string, error) {
if err := f.validate(); err != nil {
return nil, err
@@ -140,7 +143,8 @@ func (f *File) ListXattr() ([]string, error) {
// RemoveXattr removes the named xattr from the open file.
//
// Implements:
// int ceph_fremovexattr(struct ceph_mount_info *cmount, int fd, const char *name);
//
// int ceph_fremovexattr(struct ceph_mount_info *cmount, int fd, const char *name);
func (f *File) RemoveXattr(name string) error {
if err := f.validate(); err != nil {
return err
+29
View File
@@ -0,0 +1,29 @@
package cephfs
/*
#cgo LDFLAGS: -lcephfs
#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
#include <stdlib.h>
#include <cephfs/libcephfs.h>
*/
import "C"
import (
"unsafe"
)
// MakeDirs creates multiple directories at once.
//
// Implements:
//
// int ceph_mkdirs(struct ceph_mount_info *cmount, const char *path, mode_t mode);
func (mount *MountInfo) MakeDirs(path string, mode uint32) error {
if err := mount.validate(); err != nil {
return err
}
cPath := C.CString(path)
defer C.free(unsafe.Pointer(cPath))
ret := C.ceph_mkdirs(mount.mount, cPath, C.mode_t(mode))
return getError(ret)
}
+2 -1
View File
@@ -15,7 +15,8 @@ import "C"
// This function must be called after Init but before Mount.
//
// Implements:
// int ceph_mount_perms_set(struct ceph_mount_info *cmount, UserPerm *perm);
//
// int ceph_mount_perms_set(struct ceph_mount_info *cmount, UserPerm *perm);
func (mount *MountInfo) SetMountPerms(perm *UserPerm) error {
return getError(C.ceph_mount_perms_set(mount.mount, perm.userPerm))
}
+15 -8
View File
@@ -60,7 +60,8 @@ func (mount *MountInfo) RemoveDir(path string) error {
// Unlink removes a file.
//
// Implements:
// int ceph_unlink(struct ceph_mount_info *cmount, const char *path);
//
// int ceph_unlink(struct ceph_mount_info *cmount, const char *path);
func (mount *MountInfo) Unlink(path string) error {
if err := mount.validate(); err != nil {
return err
@@ -75,7 +76,8 @@ func (mount *MountInfo) Unlink(path string) error {
// Link creates a new link to an existing file.
//
// Implements:
// int ceph_link (struct ceph_mount_info *cmount, const char *existing, const char *newname);
//
// int ceph_link (struct ceph_mount_info *cmount, const char *existing, const char *newname);
func (mount *MountInfo) Link(oldname, newname string) error {
if err := mount.validate(); err != nil {
return err
@@ -93,7 +95,8 @@ func (mount *MountInfo) Link(oldname, newname string) error {
// Symlink creates a symbolic link to an existing path.
//
// Implements:
// int ceph_symlink(struct ceph_mount_info *cmount, const char *existing, const char *newname);
//
// int ceph_symlink(struct ceph_mount_info *cmount, const char *existing, const char *newname);
func (mount *MountInfo) Symlink(existing, newname string) error {
if err := mount.validate(); err != nil {
return err
@@ -111,7 +114,8 @@ func (mount *MountInfo) Symlink(existing, newname string) error {
// Readlink returns the value of a symbolic link.
//
// Implements:
// int ceph_readlink(struct ceph_mount_info *cmount, const char *path, char *buf, int64_t size);
//
// int ceph_readlink(struct ceph_mount_info *cmount, const char *path, char *buf, int64_t size);
func (mount *MountInfo) Readlink(path string) (string, error) {
if err := mount.validate(); err != nil {
return "", err
@@ -134,8 +138,9 @@ func (mount *MountInfo) Readlink(path string) (string, error) {
// Statx returns information about a file/directory.
//
// Implements:
// int ceph_statx(struct ceph_mount_info *cmount, const char *path, struct ceph_statx *stx,
// unsigned int want, unsigned int flags);
//
// int ceph_statx(struct ceph_mount_info *cmount, const char *path, struct ceph_statx *stx,
// unsigned int want, unsigned int flags);
func (mount *MountInfo) Statx(path string, want StatxMask, flags AtFlags) (*CephStatx, error) {
if err := mount.validate(); err != nil {
return nil, err
@@ -160,7 +165,8 @@ func (mount *MountInfo) Statx(path string, want StatxMask, flags AtFlags) (*Ceph
// Rename a file or directory.
//
// Implements:
// int ceph_rename(struct ceph_mount_info *cmount, const char *from, const char *to);
//
// int ceph_rename(struct ceph_mount_info *cmount, const char *from, const char *to);
func (mount *MountInfo) Rename(from, to string) error {
if err := mount.validate(); err != nil {
return err
@@ -177,7 +183,8 @@ func (mount *MountInfo) Rename(from, to string) error {
// Truncate sets the size of the specified file.
//
// Implements:
// int ceph_truncate(struct ceph_mount_info *cmount, const char *path, int64_t size);
//
// int ceph_truncate(struct ceph_mount_info *cmount, const char *path, int64_t size);
func (mount *MountInfo) Truncate(path string, size int64) error {
if err := mount.validate(); err != nil {
return err
+20 -12
View File
@@ -22,8 +22,9 @@ import (
// the xattr to be unset. Please refer to https://tracker.ceph.com/issues/46084
//
// Implements:
// int ceph_setxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// const void *value, size_t size, int flags);
//
// int ceph_setxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// const void *value, size_t size, int flags);
func (mount *MountInfo) SetXattr(path, name string, value []byte, flags XattrFlags) error {
if err := mount.validate(); err != nil {
return err
@@ -53,8 +54,9 @@ func (mount *MountInfo) SetXattr(path, name string, value []byte, flags XattrFla
// GetXattr gets an extended attribute from the file at the supplied path.
//
// Implements:
// int ceph_getxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// void *value, size_t size);
//
// int ceph_getxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// void *value, size_t size);
func (mount *MountInfo) GetXattr(path, name string) ([]byte, error) {
if err := mount.validate(); err != nil {
return nil, err
@@ -94,7 +96,8 @@ func (mount *MountInfo) GetXattr(path, name string) ([]byte, error) {
// on the file at the supplied path.
//
// Implements:
// int ceph_listxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size);
//
// int ceph_listxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size);
func (mount *MountInfo) ListXattr(path string) ([]string, error) {
if err := mount.validate(); err != nil {
return nil, err
@@ -129,7 +132,8 @@ func (mount *MountInfo) ListXattr(path string) ([]string, error) {
// RemoveXattr removes the named xattr from the open file.
//
// Implements:
// int ceph_removexattr(struct ceph_mount_info *cmount, const char *path, const char *name);
//
// int ceph_removexattr(struct ceph_mount_info *cmount, const char *path, const char *name);
func (mount *MountInfo) RemoveXattr(path, name string) error {
if err := mount.validate(); err != nil {
return err
@@ -155,8 +159,9 @@ func (mount *MountInfo) RemoveXattr(path, name string) error {
// the xattr to be unset. Please refer to https://tracker.ceph.com/issues/46084
//
// Implements:
// int ceph_lsetxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// const void *value, size_t size, int flags);
//
// int ceph_lsetxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// const void *value, size_t size, int flags);
func (mount *MountInfo) LsetXattr(path, name string, value []byte, flags XattrFlags) error {
if err := mount.validate(); err != nil {
return err
@@ -186,8 +191,9 @@ func (mount *MountInfo) LsetXattr(path, name string, value []byte, flags XattrFl
// LgetXattr gets an extended attribute from the file at the supplied path.
//
// Implements:
// int ceph_lgetxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// void *value, size_t size);
//
// int ceph_lgetxattr(struct ceph_mount_info *cmount, const char *path, const char *name,
// void *value, size_t size);
func (mount *MountInfo) LgetXattr(path, name string) ([]byte, error) {
if err := mount.validate(); err != nil {
return nil, err
@@ -227,7 +233,8 @@ func (mount *MountInfo) LgetXattr(path, name string) ([]byte, error) {
// on the file at the supplied path.
//
// Implements:
// int ceph_llistxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size);
//
// int ceph_llistxattr(struct ceph_mount_info *cmount, const char *path, char *list, size_t size);
func (mount *MountInfo) LlistXattr(path string) ([]string, error) {
if err := mount.validate(); err != nil {
return nil, err
@@ -262,7 +269,8 @@ func (mount *MountInfo) LlistXattr(path string) ([]string, error) {
// LremoveXattr removes the named xattr from the open file.
//
// Implements:
// int ceph_lremovexattr(struct ceph_mount_info *cmount, const char *path, const char *name);
//
// int ceph_lremovexattr(struct ceph_mount_info *cmount, const char *path, const char *name);
func (mount *MountInfo) LremoveXattr(path, name string) error {
if err := mount.validate(); err != nil {
return err
+31
View File
@@ -0,0 +1,31 @@
package cephfs
/*
#cgo LDFLAGS: -lcephfs
#cgo CPPFLAGS: -D_FILE_OFFSET_BITS=64
#define _GNU_SOURCE
#include <stdlib.h>
#include <cephfs/libcephfs.h>
*/
import "C"
import (
"unsafe"
)
// SelectFilesystem selects a file system to be mounted. If the ceph cluster
// supports more than one cephfs this optional function selects which one to
// use. Can only be called prior to calling Mount. The name of the file system
// is not validated by this call - if the supplied file system name is not
// valid then only the subsequent mount call will fail.
//
// Implements:
//
// int ceph_select_filesystem(struct ceph_mount_info *cmount, const char *fs_name);
func (mount *MountInfo) SelectFilesystem(name string) error {
cName := C.CString(name)
defer C.free(unsafe.Pointer(cName))
ret := C.ceph_select_filesystem(mount.mount, cName)
return getError(ret)
}
+2 -1
View File
@@ -44,7 +44,8 @@ type CephStatVFS struct {
// useful values.
//
// Implements:
// int ceph_statfs(struct ceph_mount_info *cmount, const char *path, struct statvfs *stbuf);
//
// int ceph_statfs(struct ceph_mount_info *cmount, const char *path, struct statvfs *stbuf);
func (mount *MountInfo) StatFS(path string) (*CephStatVFS, error) {
if err := mount.validate(); err != nil {
return nil, err
+4 -2
View File
@@ -29,7 +29,8 @@ type UserPerm struct {
// NewUserPerm creates a UserPerm pointer and the underlying ceph resources.
//
// Implements:
// UserPerm *ceph_userperm_new(uid_t uid, gid_t gid, int ngids, gid_t *gidlist);
//
// UserPerm *ceph_userperm_new(uid_t uid, gid_t gid, int ngids, gid_t *gidlist);
func NewUserPerm(uid, gid int, gidlist []int) *UserPerm {
// the C code does not copy the content of the gid list so we keep the
// inputs stashed in the go type. For completeness we stash everything.
@@ -59,7 +60,8 @@ func NewUserPerm(uid, gid int, gidlist []int) *UserPerm {
// Destroy will explicitly free ceph resources associated with the UserPerm.
//
// Implements:
// void ceph_userperm_destroy(UserPerm *perm);
//
// void ceph_userperm_destroy(UserPerm *perm);
func (p *UserPerm) Destroy() {
if p.userPerm == nil || !p.managed {
return
+4 -2
View File
@@ -7,7 +7,8 @@ import (
// EnableModule will enable the specified manager module.
//
// Similar To:
// ceph mgr module enable <module> [--force]
//
// ceph mgr module enable <module> [--force]
func (fsa *MgrAdmin) EnableModule(module string, force bool) error {
m := map[string]string{
"prefix": "mgr module enable",
@@ -25,7 +26,8 @@ func (fsa *MgrAdmin) EnableModule(module string, force bool) error {
// DisableModule will disable the specified manager module.
//
// Similar To:
// ceph mgr module disable <module>
//
// ceph mgr module disable <module>
func (fsa *MgrAdmin) DisableModule(module string) error {
m := map[string]string{
"prefix": "mgr module disable",
+15 -68
View File
@@ -1,90 +1,37 @@
//go:build !go1.21
// +build !go1.21
// This code assumes a non-moving garbage collector, which is the case until at
// least go 1.20
package cutil
import (
"sync"
"unsafe"
)
// PtrGuard respresents a guarded Go pointer (pointing to memory allocated by Go
// runtime) stored in C memory (allocated by C)
type PtrGuard struct {
// These mutexes will be used as binary semaphores for signalling events from
// one thread to another, which - in contrast to other languages like C++ - is
// possible in Go, that is a Mutex can be locked in one thread and unlocked in
// another.
stored, release sync.Mutex
released bool
cPtr CPtr
goPtr unsafe.Pointer
}
// WARNING: using binary semaphores (mutexes) for signalling like this is quite
// a delicate task in order to avoid deadlocks or panics. Whenever changing the
// code logic, please review at least three times that there is no unexpected
// state possible. Usually the natural choice would be to use channels instead,
// but these can not easily passed to C code because of the pointer-to-pointer
// cgo rule, and would require the use of a Go object registry.
// NewPtrGuard writes the goPtr (pointing to Go memory) into C memory at the
// position cPtr, and returns a PtrGuard object.
func NewPtrGuard(cPtr CPtr, goPtr unsafe.Pointer) *PtrGuard {
var v PtrGuard
// Since the mutexes are used for signalling, they have to be initialized to
// locked state, so that following lock attempts will block.
v.release.Lock()
v.stored.Lock()
// Start a background go routine that lives until Release is called. This
// calls a special function that makes sure the garbage collector doesn't touch
// goPtr, stores it into C memory at position cPtr and then waits until it
// reveices the "release" signal, after which it nulls out the C memory at
// cPtr and then exits.
go func() {
storeUntilRelease(&v, (*CPtr)(cPtr), uintptr(goPtr))
}()
// Wait for the "stored" signal from the go routine when the Go pointer has
// been stored to the C memory. <--(1)
v.stored.Lock()
v.cPtr = cPtr
v.goPtr = goPtr
p := (*unsafe.Pointer)(unsafe.Pointer(cPtr))
*p = goPtr
return &v
}
// Release removes the guarded Go pointer from the C memory by overwriting it
// with NULL.
func (v *PtrGuard) Release() {
if !v.released {
v.released = true
v.release.Unlock() // Send the "release" signal to the go routine. -->(2)
v.stored.Lock() // Wait for the second "stored" signal when the C memory
// has been nulled out. <--(3)
}
}
// The uintptrPtr() helper function below assumes that uintptr has the same size
// as a pointer, although in theory it could be larger. Therefore we use this
// constant expression to assert size equality as a safeguard at compile time.
// How it works: if sizes are different, either the inner or outer expression is
// negative, which always fails with "constant ... overflows uintptr", because
// unsafe.Sizeof() is a uintptr typed constant.
const _ = -(unsafe.Sizeof(uintptr(0)) - PtrSize) // size assert
func uintptrPtr(p *CPtr) *uintptr {
return (*uintptr)(unsafe.Pointer(p))
}
//go:uintptrescapes
// From https://golang.org/src/cmd/compile/internal/gc/lex.go:
// For the next function declared in the file any uintptr arguments may be
// pointer values converted to uintptr. This directive ensures that the
// referenced allocated object, if any, is retained and not moved until the call
// completes, even though from the types alone it would appear that the object
// is no longer needed during the call. The conversion to uintptr must appear in
// the argument list.
// Also see https://golang.org/cmd/compile/#hdr-Compiler_Directives
func storeUntilRelease(v *PtrGuard, cPtr *CPtr, goPtr uintptr) {
uip := uintptrPtr(cPtr)
*uip = goPtr // store Go pointer in C memory at c_ptr
v.stored.Unlock() // send "stored" signal to main thread -->(1)
v.release.Lock() // wait for "release" signal from main thread when
// Release() has been called. <--(2)
*uip = 0 // reset C memory to NULL
v.stored.Unlock() // send second "stored" signal to main thread -->(3)
p := (*unsafe.Pointer)(unsafe.Pointer(v.cPtr))
*p = nil
v.goPtr = nil
}
+35
View File
@@ -0,0 +1,35 @@
//go:build go1.21
// +build go1.21
package cutil
import (
"runtime"
"unsafe"
)
// PtrGuard respresents a guarded Go pointer (pointing to memory allocated by Go
// runtime) stored in C memory (allocated by C)
type PtrGuard struct {
cPtr CPtr
pinner runtime.Pinner
}
// NewPtrGuard writes the goPtr (pointing to Go memory) into C memory at the
// position cPtr, and returns a PtrGuard object.
func NewPtrGuard(cPtr CPtr, goPtr unsafe.Pointer) *PtrGuard {
var v PtrGuard
v.pinner.Pin(goPtr)
v.cPtr = cPtr
p := (*unsafe.Pointer)(unsafe.Pointer(cPtr))
*p = goPtr
return &v
}
// Release removes the guarded Go pointer from the C memory by overwriting it
// with NULL.
func (v *PtrGuard) Release() {
p := (*unsafe.Pointer)(unsafe.Pointer(v.cPtr))
*p = nil
v.pinner.Unpin()
}
+57
View File
@@ -0,0 +1,57 @@
package errutil
type cephErrno int
// Error returns the error string for the errno.
func (e cephErrno) Error() string {
_, strerror := FormatErrno(int(e))
return strerror
}
// cephError combines the source/component that generated the error and its
// related errno.
type cephError struct {
source string
errno cephErrno
}
// Error returns the error string with the source and errno.
func (e cephError) Error() string {
return FormatErrorCode(e.source, int(e.errno))
}
// Unwrap returns an error without the source.
func (e cephError) Unwrap() error {
if e.errno == 0 {
return nil
}
return e.errno
}
// Is checks if both errors have the same errno.
func (e cephError) Is(err error) bool {
ce, ok := err.(cephError)
if !ok {
return false
}
return e.errno == ce.errno
}
// ErrorCode returns the errno of the error.
func (e cephError) ErrorCode() int {
return int(e.errno)
}
// GetError returns a new error that can be compared with errors.Is(),
// independently of the source/component of the error.
func GetError(source string, e int) error {
if e == 0 {
return nil
}
return cephError{
source: source,
errno: cephErrno(int(e)),
}
}
-3
View File
@@ -1,6 +1,3 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
+30 -25
View File
@@ -44,11 +44,12 @@ func (c *Conn) MonCommandWithInputBuffer(args, inputBuffer []byte) ([]byte, stri
// PGCommand sends a command to one of the PGs
//
// Implements:
// int rados_pg_command(rados_t cluster, const char *pgstr,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
//
// int rados_pg_command(rados_t cluster, const char *pgstr,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
func (c *Conn) PGCommand(pgid []byte, args [][]byte) ([]byte, string, error) {
return c.PGCommandWithInputBuffer(pgid, args, nil)
}
@@ -56,11 +57,12 @@ func (c *Conn) PGCommand(pgid []byte, args [][]byte) ([]byte, string, error) {
// PGCommandWithInputBuffer sends a command to one of the PGs, with an input buffer
//
// Implements:
// int rados_pg_command(rados_t cluster, const char *pgstr,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
//
// int rados_pg_command(rados_t cluster, const char *pgstr,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
func (c *Conn) PGCommandWithInputBuffer(pgid []byte, args [][]byte, inputBuffer []byte) ([]byte, string, error) {
name := C.CString(string(pgid))
defer C.free(unsafe.Pointer(name))
@@ -92,11 +94,12 @@ func (c *Conn) MgrCommand(args [][]byte) ([]byte, string, error) {
// MgrCommandWithInputBuffer sends a command, with an input buffer, to a ceph-mgr.
//
// Implements:
// int rados_mgr_command(rados_t cluster, const char **cmd,
// size_t cmdlen, const char *inbuf,
// size_t inbuflen, char **outbuf,
// size_t *outbuflen, char **outs,
// size_t *outslen);
//
// int rados_mgr_command(rados_t cluster, const char **cmd,
// size_t cmdlen, const char *inbuf,
// size_t inbuflen, char **outbuf,
// size_t *outbuflen, char **outs,
// size_t *outslen);
func (c *Conn) MgrCommandWithInputBuffer(args [][]byte, inputBuffer []byte) ([]byte, string, error) {
ci := cutil.NewCommandInput(args, inputBuffer)
defer ci.Free()
@@ -126,11 +129,12 @@ func (c *Conn) OsdCommand(osd int, args [][]byte) ([]byte, string, error) {
// specified ceph OSD.
//
// Implements:
// int rados_osd_command(rados_t cluster, int osdid,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
//
// int rados_osd_command(rados_t cluster, int osdid,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
func (c *Conn) OsdCommandWithInputBuffer(
osd int, args [][]byte, inputBuffer []byte) ([]byte, string, error) {
@@ -162,11 +166,12 @@ func (c *Conn) MonCommandTarget(name string, args [][]byte) ([]byte, string, err
// MonCommandTargetWithInputBuffer sends a command, with an input buffer, to a specified monitor.
//
// Implements:
// int rados_mon_command_target(rados_t cluster, const char *name,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
//
// int rados_mon_command_target(rados_t cluster, const char *name,
// const char **cmd, size_t cmdlen,
// const char *inbuf, size_t inbuflen,
// char **outbuf, size_t *outbuflen,
// char **outs, size_t *outslen);
func (c *Conn) MonCommandTargetWithInputBuffer(
name string, args [][]byte, inputBuffer []byte) ([]byte, string, error) {
+11 -9
View File
@@ -95,8 +95,9 @@ func (c *Conn) ReadDefaultConfigFile() error {
// OpenIOContext creates and returns a new IOContext for the given pool.
//
// Implements:
// int rados_ioctx_create(rados_t cluster, const char *pool_name,
// rados_ioctx_t *ioctx);
//
// int rados_ioctx_create(rados_t cluster, const char *pool_name,
// rados_ioctx_t *ioctx);
func (c *Conn) OpenIOContext(pool string) (*IOContext, error) {
cPool := C.CString(pool)
defer C.free(unsafe.Pointer(cPool))
@@ -200,8 +201,9 @@ func (c *Conn) GetClusterStats() (stat ClusterStat, err error) {
// argument vector.
//
// Implements:
// int rados_conf_parse_argv(rados_t cluster, int argc,
// const char **argv);
//
// int rados_conf_parse_argv(rados_t cluster, int argc,
// const char **argv);
func (c *Conn) ParseConfigArgv(argv []string) error {
if c.cluster == nil {
return ErrNotConnected
@@ -289,11 +291,11 @@ func (c *Conn) GetPoolByName(name string) (int64, error) {
}
cName := C.CString(name)
defer C.free(unsafe.Pointer(cName))
ret := int64(C.rados_pool_lookup(c.cluster, cName))
ret := C.rados_pool_lookup(c.cluster, cName)
if ret < 0 {
return 0, radosError(ret)
return 0, getError(C.int(ret))
}
return ret, nil
return int64(ret), nil
}
// GetPoolByID returns the name of a pool by a given ID.
@@ -303,9 +305,9 @@ func (c *Conn) GetPoolByID(id int64) (string, error) {
return "", err
}
cid := C.int64_t(id)
ret := int(C.rados_pool_reverse_lookup(c.cluster, cid, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf))))
ret := C.rados_pool_reverse_lookup(c.cluster, cid, (*C.char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)))
if ret < 0 {
return "", radosError(ret)
return "", getError(ret)
}
return C.GoString((*C.char)(unsafe.Pointer(&buf[0]))), nil
}
+37 -59
View File
@@ -11,23 +11,46 @@ import (
"github.com/ceph/go-ceph/internal/errutil"
)
// radosError represents an error condition returned from the Ceph RADOS APIs.
type radosError int
// Public go errors:
// Error returns the error string for the radosError type.
func (e radosError) Error() string {
return errutil.FormatErrorCode("rados", int(e))
}
var (
// ErrNotConnected is returned when functions are called
// without a RADOS connection.
ErrNotConnected = getError(-C.ENOTCONN)
// ErrEmptyArgument may be returned if a function argument is passed
// a zero-length slice or map.
ErrEmptyArgument = errors.New("Argument must contain at least one item")
// ErrInvalidIOContext may be returned if an api call requires an IOContext
// but IOContext is not ready for use.
ErrInvalidIOContext = errors.New("IOContext is not ready for use")
// ErrOperationIncomplete is returned from write op or read op steps for
// which the operation has not been performed yet.
ErrOperationIncomplete = errors.New("Operation has not been performed yet")
func (e radosError) ErrorCode() int {
return int(e)
}
// ErrNotFound indicates a missing resource.
ErrNotFound = getError(-C.ENOENT)
// ErrPermissionDenied indicates a permissions issue.
ErrPermissionDenied = getError(-C.EPERM)
// ErrObjectExists indicates that an exclusive object creation failed.
ErrObjectExists = getError(-C.EEXIST)
func getError(e C.int) error {
if e == 0 {
return nil
}
return radosError(e)
// RadosErrorNotFound indicates a missing resource.
//
// Deprecated: use ErrNotFound instead
RadosErrorNotFound = ErrNotFound
// RadosErrorPermissionDenied indicates a permissions issue.
//
// Deprecated: use ErrPermissionDenied instead
RadosErrorPermissionDenied = ErrPermissionDenied
// Private errors:
errNameTooLong = getError(-C.ENAMETOOLONG)
errRange = getError(-C.ERANGE)
)
func getError(errno C.int) error {
return errutil.GetError("rados", int(errno))
}
// getErrorIfNegative converts a ceph return code to error if negative.
@@ -39,48 +62,3 @@ func getErrorIfNegative(ret C.int) error {
}
return getError(ret)
}
// Public go errors:
var (
// ErrNotConnected is returned when functions are called
// without a RADOS connection.
ErrNotConnected = errors.New("RADOS not connected")
// ErrEmptyArgument may be returned if a function argument is passed
// a zero-length slice or map.
ErrEmptyArgument = errors.New("Argument must contain at least one item")
// ErrInvalidIOContext may be returned if an api call requires an IOContext
// but IOContext is not ready for use.
ErrInvalidIOContext = errors.New("IOContext is not ready for use")
// ErrOperationIncomplete is returned from write op or read op steps for
// which the operation has not been performed yet.
ErrOperationIncomplete = errors.New("Operation has not been performed yet")
)
// Public radosErrors:
const (
// ErrNotFound indicates a missing resource.
ErrNotFound = radosError(-C.ENOENT)
// ErrPermissionDenied indicates a permissions issue.
ErrPermissionDenied = radosError(-C.EPERM)
// ErrObjectExists indicates that an exclusive object creation failed.
ErrObjectExists = radosError(-C.EEXIST)
// RadosErrorNotFound indicates a missing resource.
//
// Deprecated: use ErrNotFound instead
RadosErrorNotFound = ErrNotFound
// RadosErrorPermissionDenied indicates a permissions issue.
//
// Deprecated: use ErrPermissionDenied instead
RadosErrorPermissionDenied = ErrPermissionDenied
)
// Private errors:
const (
errNameTooLong = radosError(-C.ENAMETOOLONG)
errRange = radosError(-C.ERANGE)
)
+20 -12
View File
@@ -125,8 +125,9 @@ func (ioctx *IOContext) Pointer() unsafe.Pointer {
// Setting namespace to a empty or zero length string sets the pool to the default namespace.
//
// Implements:
// void rados_ioctx_set_namespace(rados_ioctx_t io,
// const char *nspace);
//
// void rados_ioctx_set_namespace(rados_ioctx_t io,
// const char *nspace);
func (ioctx *IOContext) SetNamespace(namespace string) {
var cns *C.char
if len(namespace) > 0 {
@@ -139,8 +140,9 @@ func (ioctx *IOContext) SetNamespace(namespace string) {
// Create a new object with key oid.
//
// Implements:
// void rados_write_op_create(rados_write_op_t write_op, int exclusive,
// const char* category)
//
// void rados_write_op_create(rados_write_op_t write_op, int exclusive,
// const char* category)
func (ioctx *IOContext) Create(oid string, exclusive CreateOption) error {
op := CreateWriteOp()
defer op.Release()
@@ -247,8 +249,9 @@ func (ioctx *IOContext) Destroy() {
// context.
//
// Implements:
// int rados_ioctx_pool_stat(rados_ioctx_t io,
// struct rados_pool_stat_t *stats);
//
// int rados_ioctx_pool_stat(rados_ioctx_t io,
// struct rados_pool_stat_t *stats);
func (ioctx *IOContext) GetPoolStats() (stat PoolStat, err error) {
cStat := C.struct_rados_pool_stat_t{}
ret := C.rados_ioctx_pool_stat(ioctx.ioctx, &cStat)
@@ -274,7 +277,8 @@ func (ioctx *IOContext) GetPoolStats() (stat PoolStat, err error) {
// GetPoolID returns the pool ID associated with the I/O context.
//
// Implements:
// int64_t rados_ioctx_get_id(rados_ioctx_t io)
//
// int64_t rados_ioctx_get_id(rados_ioctx_t io)
func (ioctx *IOContext) GetPoolID() int64 {
ret := C.rados_ioctx_get_id(ioctx.ioctx)
return int64(ret)
@@ -328,7 +332,8 @@ func (ioctx *IOContext) ListObjects(listFn ObjectListFunc) error {
defer C.rados_object_list_cursor_free(ioctx.ioctx, finish)
for {
ret := C.rados_object_list(ioctx.ioctx, next, finish, pageResults, nil, filterLen, (*C.rados_object_list_item)(unsafe.Pointer(&results[0])), &next)
res := (*C.rados_object_list_item)(unsafe.Pointer(&results[0]))
ret := C.rados_object_list(ioctx.ioctx, next, finish, pageResults, nil, filterLen, res, &next)
if ret < 0 {
return getError(ret)
}
@@ -338,6 +343,7 @@ func (ioctx *IOContext) ListObjects(listFn ObjectListFunc) error {
item := results[i]
listFn(C.GoStringN(item.oid, (C.int)(item.oid_length)))
}
C.rados_object_list_free(C.size_t(ret), res)
if C.rados_object_list_is_end(ioctx.ioctx, next) == listEndSentinel {
return nil
@@ -634,7 +640,7 @@ func (ioctx *IOContext) ListLockers(oid, name string) (*LockInfo, error) {
}
if ret < 0 {
return nil, radosError(ret)
return nil, getError(C.int(ret))
}
return &LockInfo{int(ret), cExclusive == 1, C.GoString(cTag), splitCString(cClients, cClientsLen), splitCString(cCookies, cCookiesLen), splitCString(cAddrs, cAddrsLen)}, nil
}
@@ -678,7 +684,8 @@ func (ioctx *IOContext) BreakLock(oid, name, client, cookie string) (int, error)
// written to.
//
// Implements:
// uint64_t rados_get_last_version(rados_ioctx_t io);
//
// uint64_t rados_get_last_version(rados_ioctx_t io);
func (ioctx *IOContext) GetLastVersion() (uint64, error) {
if err := ioctx.validate(); err != nil {
return 0, err
@@ -690,8 +697,9 @@ func (ioctx *IOContext) GetLastVersion() (uint64, error) {
// GetNamespace gets the namespace used for objects within this IO context.
//
// Implements:
// int rados_ioctx_get_namespace(rados_ioctx_t io, char *buf,
// unsigned maxlen);
//
// int rados_ioctx_get_namespace(rados_ioctx_t io, char *buf,
// unsigned maxlen);
func (ioctx *IOContext) GetNamespace() (string, error) {
if err := ioctx.validate(); err != nil {
return "", err
+4 -2
View File
@@ -13,7 +13,8 @@ import "C"
// or return EDQUOT or ENOSPC.
//
// Implements:
// void rados_set_osdmap_full_try(rados_ioctx_t io);
//
// void rados_set_osdmap_full_try(rados_ioctx_t io);
func (ioctx *IOContext) SetPoolFullTry() error {
if err := ioctx.validate(); err != nil {
return err
@@ -25,7 +26,8 @@ func (ioctx *IOContext) SetPoolFullTry() error {
// UnsetPoolFullTry unsets the flag set by SetPoolFullTry()
//
// Implements:
// void rados_unset_osdmap_full_try(rados_ioctx_t io);
//
// void rados_unset_osdmap_full_try(rados_ioctx_t io);
func (ioctx *IOContext) UnsetPoolFullTry() error {
if err := ioctx.validate(); err != nil {
return err
+4 -2
View File
@@ -16,7 +16,8 @@ import "C"
// or return EDQUOT or ENOSPC.
//
// Implements:
// void rados_set_pool_full_try(rados_ioctx_t io);
//
// void rados_set_pool_full_try(rados_ioctx_t io);
func (ioctx *IOContext) SetPoolFullTry() error {
if err := ioctx.validate(); err != nil {
return err
@@ -28,7 +29,8 @@ func (ioctx *IOContext) SetPoolFullTry() error {
// UnsetPoolFullTry unsets the flag set by SetPoolFullTry()
//
// Implements:
// void rados_unset_pool_full_try(rados_ioctx_t io);
//
// void rados_unset_pool_full_try(rados_ioctx_t io);
func (ioctx *IOContext) UnsetPoolFullTry() error {
if err := ioctx.validate(); err != nil {
return err
+2 -4
View File
@@ -1,6 +1,3 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
@@ -14,7 +11,8 @@ import "C"
// alignment or not, use RequiresAlignment.
//
// Implements:
// int rados_ioctx_pool_required_alignment2(rados_ioctx_t io, uint64_t *alignment)
//
// int rados_ioctx_pool_required_alignment2(rados_ioctx_t io, uint64_t *alignment)
func (ioctx *IOContext) Alignment() (uint64, error) {
var alignSizeBytes C.uint64_t
ret := C.rados_ioctx_pool_required_alignment2(
+2 -4
View File
@@ -1,6 +1,3 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
@@ -14,7 +11,8 @@ import "C"
// Alignment to know how to get the stripe size for pools requiring it.
//
// Implements:
// int rados_ioctx_pool_requires_alignment2(rados_ioctx_t io, int *req)
//
// int rados_ioctx_pool_requires_alignment2(rados_ioctx_t io, int *req)
func (ioctx *IOContext) RequiresAlignment() (bool, error) {
var alignRequired C.int
ret := C.rados_ioctx_pool_requires_alignment2(
+6 -8
View File
@@ -1,6 +1,3 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
@@ -19,11 +16,12 @@ import (
// the backend.
//
// Implements:
// int rados_set_alloc_hint2(rados_ioctx_t io,
// const char *o,
// uint64_t expected_object_size,
// uint64_t expected_write_size,
// uint32_t flags);
//
// int rados_set_alloc_hint2(rados_ioctx_t io,
// const char *o,
// uint64_t expected_object_size,
// uint64_t expected_write_size,
// uint32_t flags);
func (ioctx *IOContext) SetAllocationHint(oid string, expectedObjectSize uint64, expectedWriteSize uint64, flags AllocHintFlags) error {
coid := C.CString(oid)
defer C.free(unsafe.Pointer(coid))
+1 -1
View File
@@ -42,13 +42,13 @@ func (iter *Iter) Seek(token IterToken) {
// end of the iterator was reached, or the iterator received an error.
//
// Example:
//
// iter := pool.Iter()
// defer iter.Close()
// for iter.Next() {
// fmt.Printf("%v\n", iter.Value())
// }
// return iter.Err()
//
func (iter *Iter) Next() bool {
var cEntry *C.char
var cNamespace *C.char
+3 -2
View File
@@ -11,8 +11,9 @@ import "C"
// previously obtained with IOContext.GetLastVersion().
//
// Implements:
// void rados_read_op_assert_version(rados_read_op_t read_op,
// uint64_t ver)
//
// void rados_read_op_assert_version(rados_read_op_t read_op,
// uint64_t ver)
func (r *ReadOp) AssertVersion(ver uint64) {
C.rados_read_op_assert_version(r.op, C.uint64_t(ver))
}
+2 -1
View File
@@ -15,7 +15,8 @@ import (
// To reset the locator, an empty string must be set.
//
// Implements:
// void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key);
//
// void rados_ioctx_locator_set_key(rados_ioctx_t io, const char *key);
func (ioctx *IOContext) SetLocator(locator string) {
if locator == "" {
C.rados_ioctx_locator_set_key(ioctx.ioctx, nil)
+3 -2
View File
@@ -11,8 +11,9 @@ import "C"
// previously obtained with IOContext.GetLastVersion().
//
// Implements:
// void rados_read_op_assert_version(rados_read_op_t read_op,
// uint64_t ver)
//
// void rados_read_op_assert_version(rados_read_op_t read_op,
// uint64_t ver)
func (w *WriteOp) AssertVersion(ver uint64) {
C.rados_write_op_assert_version(w.op, C.uint64_t(ver))
}
+2 -1
View File
@@ -9,7 +9,8 @@ import "C"
// Remove object.
//
// Implements:
// void rados_write_op_remove(rados_write_op_t write_op)
//
// void rados_write_op_remove(rados_write_op_t write_op)
func (w *WriteOp) Remove() {
C.rados_write_op_remove(w.op)
}
+5 -4
View File
@@ -13,10 +13,11 @@ import (
// SetXattr sets an xattr.
//
// Implements:
// void rados_write_op_setxattr(rados_write_op_t write_op,
// const char * name,
// const char * value,
// size_t value_len)
//
// void rados_write_op_setxattr(rados_write_op_t write_op,
// const char * name,
// const char * value,
// size_t value_len)
func (w *WriteOp) SetXattr(name string, value []byte) {
cName := C.CString(name)
defer C.free(unsafe.Pointer(cName))
+2 -1
View File
@@ -59,7 +59,8 @@ func (r *ReadOp) operateCompat(ioctx *IOContext, oid string) error {
// AssertExists assures the object targeted by the read op exists.
//
// Implements:
// void rados_read_op_assert_exists(rados_read_op_t read_op);
//
// void rados_read_op_assert_exists(rados_read_op_t read_op);
func (r *ReadOp) AssertExists() {
C.rados_read_op_assert_exists(r.op)
}
+111
View File
@@ -0,0 +1,111 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
// #include <stdlib.h>
// #include <rados/librados.h>
//
import "C"
import (
"runtime"
"unsafe"
)
// ReadOpExecStep - step for exec operation code.
type ReadOpExecStep struct {
withoutFree
inBuffPtr *C.char
inBuffLen C.size_t
outBuffPtr *C.char
outBuffLen C.size_t
prval C.int
canReadOutput bool
}
// newExecStepOp - init new *execStepOp.
func newReadOpExecStep(in []byte) *ReadOpExecStep {
es := &ReadOpExecStep{
outBuffPtr: nil,
outBuffLen: 0,
prval: 0,
}
if len(in) > 0 {
es.inBuffPtr = (*C.char)(unsafe.Pointer(&in[0]))
es.inBuffLen = C.size_t(len(in))
}
runtime.SetFinalizer(es, func(es *ReadOpExecStep) {
if es != nil {
es.freeBuffer()
es = nil
}
})
return es
}
// freeBuffer - releases C allocated buffer. It is separated from es.free() because lifespan of C allocated buffer is
// longer than lifespan of read operation.
func (es *ReadOpExecStep) freeBuffer() {
if es.outBuffPtr != nil {
C.free(unsafe.Pointer(es.outBuffPtr))
es.outBuffPtr = nil
es.canReadOutput = false
}
}
// update - update state operation.
func (es *ReadOpExecStep) update() error {
err := getError(es.prval)
es.canReadOutput = err == nil
return err
}
// Bytes returns the result of the executed command as a byte slice.
func (es *ReadOpExecStep) Bytes() ([]byte, error) {
if !es.canReadOutput {
return nil, ErrOperationIncomplete
}
return C.GoBytes(unsafe.Pointer(es.outBuffPtr), C.int(es.outBuffLen)), nil
}
// Exec executes an OSD class method on an object.
// See rados_exec() in the RADOS C api documentation for a general description.
//
// Implements:
//
// void rados_read_op_exec(rados_read_op_t read_op,
// const char *cls,
// const char *method,
// const char *in_buf,
// size_t in_len,
// char **out_buf,
// size_t *out_len,
// int *prval);
func (r *ReadOp) Exec(clsName, method string, in []byte) *ReadOpExecStep {
cClsName := C.CString(clsName)
defer C.free(unsafe.Pointer(cClsName))
cMethod := C.CString(method)
defer C.free(unsafe.Pointer(cMethod))
es := newReadOpExecStep(in)
r.steps = append(r.steps, es)
C.rados_read_op_exec(
r.op,
cClsName,
cMethod,
es.inBuffPtr,
es.inBuffLen,
&es.outBuffPtr,
&es.outBuffLen,
&es.prval,
)
return es
}
+7 -6
View File
@@ -86,12 +86,13 @@ func (s *ReadOpOmapGetValsByKeysStep) Next() (*OmapKeyValue, error) {
// GetOmapValuesByKeys starts iterating over specific key/value pairs.
//
// Implements:
// void rados_read_op_omap_get_vals_by_keys2(rados_read_op_t read_op,
// char const * const * keys,
// size_t num_keys,
// const size_t * key_lens,
// rados_omap_iter_t * iter,
// int * prval)
//
// void rados_read_op_omap_get_vals_by_keys2(rados_read_op_t read_op,
// char const * const * keys,
// size_t num_keys,
// const size_t * key_lens,
// rados_omap_iter_t * iter,
// int * prval)
func (r *ReadOp) GetOmapValuesByKeys(keys []string) *ReadOpOmapGetValsByKeysStep {
s := newReadOpOmapGetValsByKeysStep()
r.steps = append(r.steps, s)
+7 -6
View File
@@ -48,12 +48,13 @@ func newReadOpReadStep() *ReadOpReadStep {
// buffer[:ReadOpReadStep.BytesRead] then contains object data.
//
// Implements:
// void rados_read_op_read(rados_read_op_t read_op,
// uint64_t offset,
// size_t len,
// char * buffer,
// size_t * bytes_read,
// int * prval)
//
// void rados_read_op_read(rados_read_op_t read_op,
// uint64_t offset,
// size_t len,
// char * buffer,
// size_t * bytes_read,
// int * prval)
func (r *ReadOp) Read(offset uint64, buffer []byte) *ReadOpReadStep {
oe := newReadStep(buffer, offset)
readStep := newReadOpReadStep()
+16 -9
View File
@@ -31,7 +31,8 @@ func (ioctx *IOContext) CreateSnap(snapName string) error {
// RemoveSnap deletes the pool snapshot.
//
// Implements:
// int rados_ioctx_snap_remove(rados_ioctx_t io, const char *snapname)
//
// int rados_ioctx_snap_remove(rados_ioctx_t io, const char *snapname)
func (ioctx *IOContext) RemoveSnap(snapName string) error {
if err := ioctx.validate(); err != nil {
return err
@@ -50,7 +51,8 @@ type SnapID C.rados_snap_t
// LookupSnap returns the ID of a pool snapshot.
//
// Implements:
// int rados_ioctx_snap_lookup(rados_ioctx_t io, const char *name, rados_snap_t *id)
//
// int rados_ioctx_snap_lookup(rados_ioctx_t io, const char *name, rados_snap_t *id)
func (ioctx *IOContext) LookupSnap(snapName string) (SnapID, error) {
var snapID SnapID
@@ -71,7 +73,8 @@ func (ioctx *IOContext) LookupSnap(snapName string) (SnapID, error) {
// GetSnapName returns the name of a pool snapshot with the given snapshot ID.
//
// Implements:
// int rados_ioctx_snap_get_name(rados_ioctx_t io, rados_snap_t id, char *name, int maxlen)
//
// int rados_ioctx_snap_get_name(rados_ioctx_t io, rados_snap_t id, char *name, int maxlen)
func (ioctx *IOContext) GetSnapName(snapID SnapID) (string, error) {
if err := ioctx.validate(); err != nil {
return "", err
@@ -82,8 +85,8 @@ func (ioctx *IOContext) GetSnapName(snapID SnapID) (string, error) {
err error
)
// range from 1k to 64KiB
retry.WithSizes(1024, 1<<16, func(len int) retry.Hint {
cLen := C.int(len)
retry.WithSizes(1024, 1<<16, func(length int) retry.Hint {
cLen := C.int(length)
buf = make([]byte, cLen)
ret := C.rados_ioctx_snap_get_name(
ioctx.ioctx,
@@ -103,7 +106,8 @@ func (ioctx *IOContext) GetSnapName(snapID SnapID) (string, error) {
// GetSnapStamp returns the time of the pool snapshot creation.
//
// Implements:
// int rados_ioctx_snap_get_stamp(rados_ioctx_t io, rados_snap_t id, time_t *t)
//
// int rados_ioctx_snap_get_stamp(rados_ioctx_t io, rados_snap_t id, time_t *t)
func (ioctx *IOContext) GetSnapStamp(snapID SnapID) (time.Time, error) {
var cTime C.time_t
@@ -121,7 +125,8 @@ func (ioctx *IOContext) GetSnapStamp(snapID SnapID) (time.Time, error) {
// ListSnaps returns a slice containing the SnapIDs of existing pool snapshots.
//
// Implements:
// int rados_ioctx_snap_list(rados_ioctx_t io, rados_snap_t *snaps, int maxlen)
//
// int rados_ioctx_snap_list(rados_ioctx_t io, rados_snap_t *snaps, int maxlen)
func (ioctx *IOContext) ListSnaps() ([]SnapID, error) {
if err := ioctx.validate(); err != nil {
return nil, err
@@ -154,7 +159,8 @@ func (ioctx *IOContext) ListSnaps() ([]SnapID, error) {
// The contents of the object will be the same as when the snapshot was taken.
//
// Implements:
// int rados_ioctx_snap_rollback(rados_ioctx_t io, const char *oid, const char *snapname);
//
// int rados_ioctx_snap_rollback(rados_ioctx_t io, const char *oid, const char *snapname);
func (ioctx *IOContext) RollbackSnap(oid, snapName string) error {
if err := ioctx.validate(); err != nil {
return err
@@ -178,7 +184,8 @@ const SnapHead = SnapID(C.LIBRADOS_SNAP_HEAD)
// Pass SnapHead for no snapshot (i.e. normal operation).
//
// Implements:
// void rados_ioctx_snap_set_read(rados_ioctx_t io, rados_snap_t snap);
//
// void rados_ioctx_snap_set_read(rados_ioctx_t io, rados_snap_t snap);
func (ioctx *IOContext) SetReadSnap(snapID SnapID) error {
if err := ioctx.validate(); err != nil {
return err
+48 -40
View File
@@ -72,27 +72,28 @@ var (
// the NotifyEvents and Errors() receives all occuring errors. A typical code
// creating a Watcher could look like this:
//
// watcher, err := ioctx.Watch(oid)
// go func() { // event handler
// for ne := range watcher.Events() {
// ...
// ne.Ack([]byte("response data..."))
// ...
// }
// }()
// go func() { // error handler
// for err := range watcher.Errors() {
// ... handle err ...
// }
// }()
// watcher, err := ioctx.Watch(oid)
// go func() { // event handler
// for ne := range watcher.Events() {
// ...
// ne.Ack([]byte("response data..."))
// ...
// }
// }()
// go func() { // error handler
// for err := range watcher.Errors() {
// ... handle err ...
// }
// }()
//
// CAUTION: the Watcher references the IOContext in which it has been created.
// Therefore all watchers must be deleted with the Delete() method before the
// IOContext is being destroyed.
//
// Implements:
// int rados_watch2(rados_ioctx_t io, const char* o, uint64_t* cookie,
// rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, void* arg)
//
// int rados_watch2(rados_ioctx_t io, const char* o, uint64_t* cookie,
// rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, void* arg)
func (ioctx *IOContext) Watch(obj string) (*Watcher, error) {
return ioctx.WatchWithTimeout(obj, 0)
}
@@ -101,9 +102,10 @@ func (ioctx *IOContext) Watch(obj string) (*Watcher, error) {
// different timeout than the default can be specified.
//
// Implements:
// int rados_watch3(rados_ioctx_t io, const char *o, uint64_t *cookie,
// rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, uint32_t timeout,
// void *arg);
//
// int rados_watch3(rados_ioctx_t io, const char *o, uint64_t *cookie,
// rados_watchcb2_t watchcb, rados_watcherrcb_t watcherrcb, uint32_t timeout,
// void *arg);
func (ioctx *IOContext) WatchWithTimeout(oid string, timeout time.Duration) (*Watcher, error) {
cObj := C.CString(oid)
defer C.free(unsafe.Pointer(cObj))
@@ -158,7 +160,8 @@ func (w *Watcher) Errors() <-chan error {
// Watcher is no longer valid, and should be destroyed with the Delete() method.
//
// Implements:
// int rados_watch_check(rados_ioctx_t io, uint64_t cookie)
//
// int rados_watch_check(rados_ioctx_t io, uint64_t cookie)
func (w *Watcher) Check() (time.Duration, error) {
ret := C.rados_watch_check(w.ioctx.ioctx, C.uint64_t(w.id))
if ret < 0 {
@@ -170,7 +173,8 @@ func (w *Watcher) Check() (time.Duration, error) {
// Delete the watcher. This closes both the event and error channel.
//
// Implements:
// int rados_unwatch2(rados_ioctx_t io, uint64_t cookie)
//
// int rados_unwatch2(rados_ioctx_t io, uint64_t cookie)
func (w *Watcher) Delete() error {
watchersMtx.Lock()
_, ok := watchers[w.id]
@@ -204,8 +208,9 @@ func (ioctx *IOContext) Notify(obj string, data []byte) ([]NotifyAck, []NotifyTi
// default.
//
// Implements:
// int rados_notify2(rados_ioctx_t io, const char* o, const char* buf, int buf_len,
// uint64_t timeout_ms, char** reply_buffer, size_t* reply_buffer_len)
//
// int rados_notify2(rados_ioctx_t io, const char* o, const char* buf, int buf_len,
// uint64_t timeout_ms, char** reply_buffer, size_t* reply_buffer_len)
func (ioctx *IOContext) NotifyWithTimeout(obj string, data []byte, timeout time.Duration) ([]NotifyAck,
[]NotifyTimeout, error) {
cObj := C.CString(obj)
@@ -236,8 +241,9 @@ func (ioctx *IOContext) NotifyWithTimeout(obj string, data []byte, timeout time.
// blocks and eventiually times out.
//
// Implements:
// int rados_notify_ack(rados_ioctx_t io, const char *o, uint64_t notify_id,
// uint64_t cookie, const char *buf, int buf_len)
//
// int rados_notify_ack(rados_ioctx_t io, const char *o, uint64_t notify_id,
// uint64_t cookie, const char *buf, int buf_len)
func (ne *NotifyEvent) Ack(response []byte) error {
watchersMtx.RLock()
w, ok := watchers[ne.WatcherID]
@@ -265,7 +271,8 @@ func (ne *NotifyEvent) Ack(response []byte) error {
// WatcherFlush flushes all pending notifications of the cluster.
//
// Implements:
// int rados_watch_flush(rados_t cluster)
//
// int rados_watch_flush(rados_t cluster)
func (c *Conn) WatcherFlush() error {
if !c.connected {
return ErrNotConnected
@@ -275,26 +282,27 @@ func (c *Conn) WatcherFlush() error {
}
// decoder for this notify response format:
// le32 num_acks
// {
// le64 gid global id for the client (for client.1234 that's 1234)
// le64 cookie cookie for the client
// le32 buflen length of reply message buffer
// u8 buflen payload
// } num_acks
// le32 num_timeouts
// {
// le64 gid global id for the client
// le64 cookie cookie for the client
// } num_timeouts
//
// le32 num_acks
// {
// le64 gid global id for the client (for client.1234 that's 1234)
// le64 cookie cookie for the client
// le32 buflen length of reply message buffer
// u8 buflen payload
// } num_acks
// le32 num_timeouts
// {
// le64 gid global id for the client
// le64 cookie cookie for the client
// } num_timeouts
//
// NOTE: starting with pacific this is implemented as a C function and this can
// be replaced later
func decodeNotifyResponse(response *C.char, len C.size_t) ([]NotifyAck, []NotifyTimeout) {
if len == 0 || response == nil {
func decodeNotifyResponse(response *C.char, length C.size_t) ([]NotifyAck, []NotifyTimeout) {
if length == 0 || response == nil {
return nil, nil
}
b := (*[math.MaxInt32]byte)(unsafe.Pointer(response))[:len:len]
b := (*[math.MaxInt32]byte)(unsafe.Pointer(response))[:length:length]
pos := 0
num := binary.LittleEndian.Uint32(b[pos:])
+17 -13
View File
@@ -136,7 +136,8 @@ func (w *WriteOp) CleanOmap() {
// AssertExists assures the object targeted by the write op exists.
//
// Implements:
// void rados_write_op_assert_exists(rados_write_op_t write_op);
//
// void rados_write_op_assert_exists(rados_write_op_t write_op);
func (w *WriteOp) AssertExists() {
C.rados_write_op_assert_exists(w.op)
}
@@ -144,10 +145,11 @@ func (w *WriteOp) AssertExists() {
// Write a given byte slice at the supplied offset.
//
// Implements:
// void rados_write_op_write(rados_write_op_t write_op,
// const char *buffer,
// size_t len,
// uint64_t offset);
//
// void rados_write_op_write(rados_write_op_t write_op,
// const char *buffer,
// size_t len,
// uint64_t offset);
func (w *WriteOp) Write(b []byte, offset uint64) {
oe := newWriteStep(b, 0, offset)
w.steps = append(w.steps, oe)
@@ -162,9 +164,10 @@ func (w *WriteOp) Write(b []byte, offset uint64) {
// atomically replacing it.
//
// Implements:
// void rados_write_op_write_full(rados_write_op_t write_op,
// const char *buffer,
// size_t len);
//
// void rados_write_op_write_full(rados_write_op_t write_op,
// const char *buffer,
// size_t len);
func (w *WriteOp) WriteFull(b []byte) {
oe := newWriteStep(b, 0, 0)
w.steps = append(w.steps, oe)
@@ -178,11 +181,12 @@ func (w *WriteOp) WriteFull(b []byte) {
// writeLen is satisfied.
//
// Implements:
// void rados_write_op_writesame(rados_write_op_t write_op,
// const char *buffer,
// size_t data_len,
// size_t write_len,
// uint64_t offset);
//
// void rados_write_op_writesame(rados_write_op_t write_op,
// const char *buffer,
// size_t data_len,
// size_t write_len,
// uint64_t offset);
func (w *WriteOp) WriteSame(b []byte, writeLen, offset uint64) {
oe := newWriteStep(b, writeLen, offset)
w.steps = append(w.steps, oe)
+6 -5
View File
@@ -39,11 +39,12 @@ func newWriteOpCmpExtStep() *WriteOpCmpExtStep {
// CmpExt ensures that given object range (extent) satisfies comparison.
//
// Implements:
// void rados_write_op_cmpext(rados_write_op_t write_op,
// const char * cmp_buf,
// size_t cmp_len,
// uint64_t off,
// int * prval);
//
// void rados_write_op_cmpext(rados_write_op_t write_op,
// const char * cmp_buf,
// size_t cmp_len,
// uint64_t off,
// int * prval);
func (w *WriteOp) CmpExt(b []byte, offset uint64) *WriteOpCmpExtStep {
oe := newWriteStep(b, 0, offset)
cmpExtStep := newWriteOpCmpExtStep()
+71
View File
@@ -0,0 +1,71 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
// #include <stdlib.h>
// #include <rados/librados.h>
//
import "C"
import (
"unsafe"
)
// writeOpExecStep - exec step in write operation.
type writeOpExecStep struct {
withoutFree
inBuffPtr *C.char
inBuffLen C.size_t
prval C.int
}
// newWriteOpExecStep - init new *writeOpExecStep.
func newWriteOpExecStep(in []byte) *writeOpExecStep {
es := &writeOpExecStep{
prval: 0,
}
if len(in) > 0 {
es.inBuffPtr = (*C.char)(unsafe.Pointer(&in[0]))
es.inBuffLen = C.size_t(len(in))
}
return es
}
// update - update state operation.
func (es *writeOpExecStep) update() error {
return getError(es.prval)
}
// Exec executes an OSD class method on an object.
// See rados_exec() in the RADOS C api documentation for a general description.
//
// Implements:
//
// void rados_write_op_exec(rados_write_op_t write_op,
// const char *cls,
// const char *method,
// const char *in_buf,
// size_t in_len,
// int *prval)
func (w *WriteOp) Exec(clsName, method string, in []byte) {
cClsName := C.CString(clsName)
defer C.free(unsafe.Pointer(cClsName))
cMethod := C.CString(method)
defer C.free(unsafe.Pointer(cMethod))
es := newWriteOpExecStep(in)
w.steps = append(w.steps, es)
C.rados_write_op_exec(
w.op,
cClsName,
cMethod,
es.inBuffPtr,
es.inBuffLen,
&es.prval,
)
}
+5 -7
View File
@@ -1,6 +1,3 @@
//go:build ceph_preview
// +build ceph_preview
package rados
// #cgo LDFLAGS: -lrados
@@ -15,10 +12,11 @@ import "C"
// the backend.
//
// Implements:
// void rados_write_op_set_alloc_hint2(rados_write_op_t write_op,
// uint64_t expected_object_size,
// uint64_t expected_write_size,
// uint32_t flags);
//
// void rados_write_op_set_alloc_hint2(rados_write_op_t write_op,
// uint64_t expected_object_size,
// uint64_t expected_write_size,
// uint32_t flags);
func (w *WriteOp) SetAllocationHint(expectedObjectSize uint64, expectedWriteSize uint64, flags AllocHintFlags) {
C.rados_write_op_set_alloc_hint2(
w.op,
+62
View File
@@ -0,0 +1,62 @@
package md2man
import (
"fmt"
"io"
"os"
"strings"
"github.com/russross/blackfriday/v2"
)
func fmtListFlags(flags blackfriday.ListType) string {
knownFlags := []struct {
name string
flag blackfriday.ListType
}{
{"ListTypeOrdered", blackfriday.ListTypeOrdered},
{"ListTypeDefinition", blackfriday.ListTypeDefinition},
{"ListTypeTerm", blackfriday.ListTypeTerm},
{"ListItemContainsBlock", blackfriday.ListItemContainsBlock},
{"ListItemBeginningOfList", blackfriday.ListItemBeginningOfList},
{"ListItemEndOfList", blackfriday.ListItemEndOfList},
}
var f []string
for _, kf := range knownFlags {
if flags&kf.flag != 0 {
f = append(f, kf.name)
flags &^= kf.flag
}
}
if flags != 0 {
f = append(f, fmt.Sprintf("Unknown(%#x)", flags))
}
return strings.Join(f, "|")
}
type debugDecorator struct {
blackfriday.Renderer
}
func depth(node *blackfriday.Node) int {
d := 0
for n := node.Parent; n != nil; n = n.Parent {
d++
}
return d
}
func (d *debugDecorator) RenderNode(w io.Writer, node *blackfriday.Node, entering bool) blackfriday.WalkStatus {
fmt.Fprintf(os.Stderr, "%s%s %v %v\n",
strings.Repeat(" ", depth(node)),
map[bool]string{true: "+", false: "-"}[entering],
node,
fmtListFlags(node.ListFlags))
var b strings.Builder
status := d.Renderer.RenderNode(io.MultiWriter(&b, w), node, entering)
if b.Len() > 0 {
fmt.Fprintf(os.Stderr, ">> %q\n", b.String())
}
return status
}
+8 -1
View File
@@ -1,16 +1,23 @@
package md2man
import (
"os"
"strconv"
"github.com/russross/blackfriday/v2"
)
// Render converts a markdown document into a roff formatted document.
func Render(doc []byte) []byte {
renderer := NewRoffRenderer()
var r blackfriday.Renderer = renderer
if v, _ := strconv.ParseBool(os.Getenv("MD2MAN_DEBUG")); v {
r = &debugDecorator{Renderer: r}
}
return blackfriday.Run(doc,
[]blackfriday.Option{
blackfriday.WithRenderer(renderer),
blackfriday.WithRenderer(r),
blackfriday.WithExtensions(renderer.GetExtensions()),
}...)
}
+57 -31
View File
@@ -14,10 +14,8 @@ import (
// roffRenderer implements the blackfriday.Renderer interface for creating
// roff format (manpages) from markdown text
type roffRenderer struct {
extensions blackfriday.Extensions
listCounters []int
firstHeader bool
firstDD bool
listDepth int
}
@@ -43,7 +41,7 @@ const (
quoteTag = "\n.PP\n.RS\n"
quoteCloseTag = "\n.RE\n"
listTag = "\n.RS\n"
listCloseTag = "\n.RE\n"
listCloseTag = ".RE\n"
dtTag = "\n.TP\n"
dd2Tag = "\n"
tableStart = "\n.TS\nallbox;\n"
@@ -56,23 +54,18 @@ const (
// NewRoffRenderer creates a new blackfriday Renderer for generating roff documents
// from markdown
func NewRoffRenderer() *roffRenderer { // nolint: golint
var extensions blackfriday.Extensions
extensions |= blackfriday.NoIntraEmphasis
extensions |= blackfriday.Tables
extensions |= blackfriday.FencedCode
extensions |= blackfriday.SpaceHeadings
extensions |= blackfriday.Footnotes
extensions |= blackfriday.Titleblock
extensions |= blackfriday.DefinitionLists
return &roffRenderer{
extensions: extensions,
}
return &roffRenderer{}
}
// GetExtensions returns the list of extensions used by this renderer implementation
func (r *roffRenderer) GetExtensions() blackfriday.Extensions {
return r.extensions
func (*roffRenderer) GetExtensions() blackfriday.Extensions {
return blackfriday.NoIntraEmphasis |
blackfriday.Tables |
blackfriday.FencedCode |
blackfriday.SpaceHeadings |
blackfriday.Footnotes |
blackfriday.Titleblock |
blackfriday.DefinitionLists
}
// RenderHeader handles outputting the header at document start
@@ -103,7 +96,23 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering
switch node.Type {
case blackfriday.Text:
escapeSpecialChars(w, node.Literal)
// Special case: format the NAME section as required for proper whatis parsing.
// Refer to the lexgrog(1) and groff_man(7) manual pages for details.
if node.Parent != nil &&
node.Parent.Type == blackfriday.Paragraph &&
node.Parent.Prev != nil &&
node.Parent.Prev.Type == blackfriday.Heading &&
node.Parent.Prev.FirstChild != nil &&
bytes.EqualFold(node.Parent.Prev.FirstChild.Literal, []byte("NAME")) {
before, after, found := bytes.Cut(node.Literal, []byte(" - "))
escapeSpecialChars(w, before)
if found {
out(w, ` \- `)
escapeSpecialChars(w, after)
}
} else {
escapeSpecialChars(w, node.Literal)
}
case blackfriday.Softbreak:
out(w, crTag)
case blackfriday.Hardbreak:
@@ -141,14 +150,25 @@ func (r *roffRenderer) RenderNode(w io.Writer, node *blackfriday.Node, entering
case blackfriday.Document:
break
case blackfriday.Paragraph:
// roff .PP markers break lists
if r.listDepth > 0 {
return blackfriday.GoToNext
}
if entering {
out(w, paraTag)
if r.listDepth > 0 {
// roff .PP markers break lists
if node.Prev != nil { // continued paragraph
if node.Prev.Type == blackfriday.List && node.Prev.ListFlags&blackfriday.ListTypeDefinition == 0 {
out(w, ".IP\n")
} else {
out(w, crTag)
}
}
} else if node.Prev != nil && node.Prev.Type == blackfriday.Heading {
out(w, crTag)
} else {
out(w, paraTag)
}
} else {
out(w, crTag)
if node.Next == nil || node.Next.Type != blackfriday.List {
out(w, crTag)
}
}
case blackfriday.BlockQuote:
if entering {
@@ -211,6 +231,10 @@ func (r *roffRenderer) handleHeading(w io.Writer, node *blackfriday.Node, enteri
func (r *roffRenderer) handleList(w io.Writer, node *blackfriday.Node, entering bool) {
openTag := listTag
closeTag := listCloseTag
if (entering && r.listDepth == 0) || (!entering && r.listDepth == 1) {
openTag = crTag
closeTag = ""
}
if node.ListFlags&blackfriday.ListTypeDefinition != 0 {
// tags for definition lists handled within Item node
openTag = ""
@@ -239,23 +263,25 @@ func (r *roffRenderer) handleItem(w io.Writer, node *blackfriday.Node, entering
} else if node.ListFlags&blackfriday.ListTypeTerm != 0 {
// DT (definition term): line just before DD (see below).
out(w, dtTag)
r.firstDD = true
} else if node.ListFlags&blackfriday.ListTypeDefinition != 0 {
// DD (definition description): line that starts with ": ".
//
// We have to distinguish between the first DD and the
// subsequent ones, as there should be no vertical
// whitespace between the DT and the first DD.
if r.firstDD {
r.firstDD = false
} else {
out(w, dd2Tag)
if node.Prev != nil && node.Prev.ListFlags&(blackfriday.ListTypeTerm|blackfriday.ListTypeDefinition) == blackfriday.ListTypeDefinition {
if node.Prev.Type == blackfriday.Item &&
node.Prev.LastChild != nil &&
node.Prev.LastChild.Type == blackfriday.List &&
node.Prev.LastChild.ListFlags&blackfriday.ListTypeDefinition == 0 {
out(w, ".IP\n")
} else {
out(w, dd2Tag)
}
}
} else {
out(w, ".IP \\(bu 2\n")
}
} else {
out(w, "\n")
}
}

Some files were not shown because too many files have changed in this diff Show More