sevenhill/QSfera
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

enhancement(search): implement index manager and remove the use of index templates

Browse Source
This commit is contained in:
fschade
2025-08-28 09:32:04 +02:00
parent 9faa09e4c6
commit 1586f7fcbb
17 changed files with 1346 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+2 -1
View File
@@ -23,6 +23,7 @@ require (
github.com/ggwhite/go-masker v1.1.0 github.com/ggwhite/go-masker v1.1.0
github.com/go-chi/chi/v5 v5.2.2 github.com/go-chi/chi/v5 v5.2.2
github.com/go-chi/render v1.0.3 github.com/go-chi/render v1.0.3
github.com/go-jose/go-jose/v3 v3.0.4
github.com/go-ldap/ldap/v3 v3.4.11 github.com/go-ldap/ldap/v3 v3.4.11
github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3 github.com/go-ldap/ldif v0.0.0-20200320164324-fd88d9b715b3
github.com/go-micro/plugins/v4/client/grpc v1.2.1 github.com/go-micro/plugins/v4/client/grpc v1.2.1
@@ -84,6 +85,7 @@ require (
github.com/theckman/yacspin v0.13.12 github.com/theckman/yacspin v0.13.12
github.com/thejerf/suture/v4 v4.0.6 github.com/thejerf/suture/v4 v4.0.6
github.com/tidwall/gjson v1.18.0 github.com/tidwall/gjson v1.18.0
github.com/tidwall/sjson v1.2.5
github.com/tus/tusd/v2 v2.8.0 github.com/tus/tusd/v2 v2.8.0
github.com/unrolled/secure v1.16.0 github.com/unrolled/secure v1.16.0
github.com/urfave/cli/v2 v2.27.7 github.com/urfave/cli/v2 v2.27.7
@@ -188,7 +190,6 @@ require (
github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect
github.com/go-git/go-git/v5 v5.13.2 // indirect github.com/go-git/go-git/v5 v5.13.2 // indirect
github.com/go-ini/ini v1.67.0 // indirect github.com/go-ini/ini v1.67.0 // indirect
github.com/go-jose/go-jose/v3 v3.0.4 // indirect
github.com/go-jose/go-jose/v4 v4.1.1 // indirect github.com/go-jose/go-jose/v4 v4.1.1 // indirect
github.com/go-kit/log v0.2.1 // indirect github.com/go-kit/log v0.2.1 // indirect
github.com/go-logfmt/logfmt v0.5.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect
go.sum
+1
View File
@@ -1087,6 +1087,7 @@ github.com/theckman/yacspin v0.13.12 h1:CdZ57+n0U6JMuh2xqjnjRq5Haj6v1ner2djtLQRz
github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg= github.com/theckman/yacspin v0.13.12/go.mod h1:Rd2+oG2LmQi5f3zC3yeZAOl245z8QOvrH4OPOJNZxLg=
github.com/thejerf/suture/v4 v4.0.6 h1:QsuCEsCqb03xF9tPAsWAj8QOAJBgQI1c0VqJNaingg8= github.com/thejerf/suture/v4 v4.0.6 h1:QsuCEsCqb03xF9tPAsWAj8QOAJBgQI1c0VqJNaingg8=
github.com/thejerf/suture/v4 v4.0.6/go.mod h1:gu9Y4dXNUWFrByqRt30Rm9/UZ0wzRSt9AJS6xu/ZGxU= github.com/thejerf/suture/v4 v4.0.6/go.mod h1:gu9Y4dXNUWFrByqRt30Rm9/UZ0wzRSt9AJS6xu/ZGxU=
github.com/tidwall/gjson v1.14.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=
github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA=
services/search/pkg/opensearch/engine.go
+2 -28
View File
@@ -39,36 +39,10 @@ func NewEngine(index string, client *opensearchgoAPI.Client) (*Engine, error) {
} }
// apply the index template // apply the index template
if err := IndexTemplateResourceV1.Apply(context.TODO(), client); err != nil { if err := IndexManagerLatest.Apply(context.TODO(), index, client); err != nil {
return nil, fmt.Errorf("failed to apply index template: %w", err) return nil, fmt.Errorf("failed to apply index template: %w", err)
} }
indicesExistsResp, err := client.Indices.Exists(context.TODO(), opensearchgoAPI.IndicesExistsReq{
Indices: []string{index},
})
switch {
case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404:
break
case err != nil:
return nil, fmt.Errorf("failed to check if index exists: %w", err)
case indicesExistsResp == nil:
return nil, fmt.Errorf("unexpected nil response when checking if index exists")
}
// if the index does not exist, we need to create it
if indicesExistsResp.StatusCode == 404 {
resp, err := client.Indices.Create(context.TODO(), opensearchgoAPI.IndicesCreateReq{
Index: index,
// the body is not necessary; we will use an index template to define the index settings and mappings
})
switch {
case err != nil:
return nil, fmt.Errorf("failed to create index: %w", err)
case !resp.Acknowledged:
return nil, fmt.Errorf("failed to create index: %s", index)
}
}
// first check if the cluster is healthy // first check if the cluster is healthy
_, healthy, err := clusterHealth(context.TODO(), client, []string{index}) _, healthy, err := clusterHealth(context.TODO(), client, []string{index})
switch { switch {
@@ -116,7 +90,7 @@ func (e *Engine) Search(ctx context.Context, sir *searchService.SearchIndexReque
} }
body, err := NewRootQuery(boolQuery, RootQueryOptions{ body, err := NewRootQuery(boolQuery, RootQueryOptions{
Highlight: RootQueryHighlight{ Highlight: &RootQueryHighlight{
PreTags: []string{"<mark>"}, PreTags: []string{"<mark>"},
PostTags: []string{"</mark>"}, PostTags: []string{"</mark>"},
Fields: map[string]RootQueryHighlight{ Fields: map[string]RootQueryHighlight{
services/search/pkg/opensearch/engine_test.go
+63 -63
View File
@@ -23,26 +23,26 @@ func TestNewEngine(t *testing.T) {
}) })
require.NoError(t, err, "failed to create OpenSearch client") require.NoError(t, err, "failed to create OpenSearch client")
engine, err := opensearch.NewEngine("test-engine-new-engine", client) backend, err := opensearch.NewEngine("test-engine-new-engine", client)
require.Nil(t, engine) require.Nil(t, backend)
require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster) require.ErrorIs(t, err, opensearch.ErrUnhealthyCluster)
}) })
} }
func TestEngine_Search(t *testing.T) { func TestEngine_Search(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err)
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
backend, err := opensearch.NewEngine(index, tc.Client())
require.NoError(t, err)
t.Run("most simple search", func(t *testing.T) { t.Run("most simple search", func(t *testing.T) {
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{ resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
@@ -59,8 +59,8 @@ func TestEngine_Search(t *testing.T) {
deletedDocument.ID = "1$2!4" deletedDocument.ID = "1$2!4"
deletedDocument.Deleted = true deletedDocument.Deleted = true
tc.Require.DocumentCreate(index, deletedDocument.ID, opensearchtest.JSONMustMarshal(t, deletedDocument)) tc.Require.DocumentCreate(indexName, deletedDocument.ID, opensearchtest.JSONMustMarshal(t, deletedDocument))
tc.Require.IndicesCount([]string{index}, "", 2) tc.Require.IndicesCount([]string{indexName}, "", 2)
resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{ resp, err := backend.Search(t.Context(), &searchService.SearchIndexRequest{
Query: fmt.Sprintf(`"%s"`, document.Name), Query: fmt.Sprintf(`"%s"`, document.Name),
@@ -73,43 +73,43 @@ func TestEngine_Search(t *testing.T) {
} }
func TestEngine_Upsert(t *testing.T) { func TestEngine_Upsert(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(index, tc.Client()) backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err) require.NoError(t, err)
t.Run("upsert with full document", func(t *testing.T) { t.Run("upsert with full document", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
require.NoError(t, backend.Upsert(document.ID, document)) require.NoError(t, backend.Upsert(document.ID, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
}) })
} }
func TestEngine_Move(t *testing.T) { func TestEngine_Move(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(index, tc.Client()) backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err) require.NoError(t, err)
t.Run("moves the document to a new path", func(t *testing.T) { t.Run("moves the document to a new path", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
resources := opensearchtest.SearchHitsMustBeConverted[engine.Resource](t, resources := opensearchtest.SearchHitsMustBeConverted[engine.Resource](t,
tc.Require.Search( tc.Require.Search(
index, indexName,
opensearch.NewRootQuery( opensearch.NewRootQuery(
opensearch.NewIDsQuery([]string{document.ID}), opensearch.NewIDsQuery([]string{document.ID}),
).String(), ).String(),
@@ -123,7 +123,7 @@ func TestEngine_Move(t *testing.T) {
resources = opensearchtest.SearchHitsMustBeConverted[engine.Resource](t, resources = opensearchtest.SearchHitsMustBeConverted[engine.Resource](t,
tc.Require.Search( tc.Require.Search(
index, indexName,
opensearch.NewRootQuery( opensearch.NewRootQuery(
opensearch.NewIDsQuery([]string{document.ID}), opensearch.NewIDsQuery([]string{document.ID}),
).String(), ).String(),
@@ -135,109 +135,109 @@ func TestEngine_Move(t *testing.T) {
} }
func TestEngine_Delete(t *testing.T) { func TestEngine_Delete(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(index, tc.Client()) backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err) require.NoError(t, err)
t.Run("mark document as deleted", func(t *testing.T) { t.Run("mark document as deleted", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery(
opensearch.NewTermQuery[bool]("Deleted").Value(true), opensearch.NewTermQuery[bool]("Deleted").Value(true),
).String(), 0) ).String(), 0)
require.NoError(t, backend.Delete(document.ID)) require.NoError(t, backend.Delete(document.ID))
tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery(
opensearch.NewTermQuery[bool]("Deleted").Value(true), opensearch.NewTermQuery[bool]("Deleted").Value(true),
).String(), 1) ).String(), 1)
}) })
} }
func TestEngine_Restore(t *testing.T) { func TestEngine_Restore(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(index, tc.Client()) backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err) require.NoError(t, err)
t.Run("mark document as not deleted", func(t *testing.T) { t.Run("mark document as not deleted", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
document.Deleted = true document.Deleted = true
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery(
opensearch.NewTermQuery[bool]("Deleted").Value(true), opensearch.NewTermQuery[bool]("Deleted").Value(true),
).String(), 1) ).String(), 1)
require.NoError(t, backend.Restore(document.ID)) require.NoError(t, backend.Restore(document.ID))
tc.Require.IndicesCount([]string{index}, opensearch.NewRootQuery( tc.Require.IndicesCount([]string{indexName}, opensearch.NewRootQuery(
opensearch.NewTermQuery[bool]("Deleted").Value(true), opensearch.NewTermQuery[bool]("Deleted").Value(true),
).String(), 0) ).String(), 0)
}) })
} }
func TestEngine_Purge(t *testing.T) { func TestEngine_Purge(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(index, tc.Client()) backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err) require.NoError(t, err)
t.Run("purge with full document", func(t *testing.T) { t.Run("purge with full document", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
require.NoError(t, backend.Purge(document.ID)) require.NoError(t, backend.Purge(document.ID))
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
}) })
} }
func TestEngine_DocCount(t *testing.T) { func TestEngine_DocCount(t *testing.T) {
index := "opencloud-default-resource" indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t) tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{index}) tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCount([]string{index}, "", 0) tc.Require.IndicesCount([]string{indexName}, "", 0)
defer tc.Require.IndicesDelete([]string{index}) defer tc.Require.IndicesDelete([]string{indexName})
backend, err := opensearch.NewEngine(index, tc.Client()) backend, err := opensearch.NewEngine(indexName, tc.Client())
require.NoError(t, err) require.NoError(t, err)
t.Run("ignore deleted documents", func(t *testing.T) { t.Run("ignore deleted documents", func(t *testing.T) {
document := opensearchtest.Testdata.Resources.File document := opensearchtest.Testdata.Resources.File
tc.Require.DocumentCreate(index, document.ID, opensearchtest.JSONMustMarshal(t, document)) tc.Require.DocumentCreate(indexName, document.ID, opensearchtest.JSONMustMarshal(t, document))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
count, err := backend.DocCount() count, err := backend.DocCount()
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, uint64(1), count) require.Equal(t, uint64(1), count)
tc.Require.Update(index, document.ID, opensearchtest.JSONMustMarshal(t, map[string]any{ tc.Require.Update(indexName, document.ID, opensearchtest.JSONMustMarshal(t, map[string]any{
"doc": map[string]any{ "doc": map[string]any{
"Deleted": true, "Deleted": true,
}, },
})) }))
tc.Require.IndicesCount([]string{index}, "", 1) tc.Require.IndicesCount([]string{indexName}, "", 1)
count, err = backend.DocCount() count, err = backend.DocCount()
require.NoError(t, err) require.NoError(t, err)
services/search/pkg/opensearch/internal/indexes/resource_v1.json
+33
View File
@@ -0,0 +1,33 @@
{
"settings": {
"number_of_shards": "1",
"number_of_replicas": "1"
},
"mappings": {
"properties": {
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard",
"doc_values": false
},
"Path": {
"type": "wildcard",
"doc_values": false
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
}
}
}
}
services/search/pkg/opensearch/internal/indices/resource_v1.json
-40
View File
@@ -1,40 +0,0 @@
{
"index_patterns": [
"opencloud-default-*"
],
"template": {
"settings": {
"number_of_shards": 1,
"number_of_replicas": 1
},
"mappings": {
"properties": {
"ID": {
"type": "keyword"
},
"ParentID": {
"type": "keyword"
},
"RootID": {
"type": "keyword"
},
"MimeType": {
"type": "wildcard"
},
"Path": {
"type": "wildcard"
},
"Deleted": {
"type": "boolean"
},
"Hidden": {
"type": "boolean"
}
}
}
},
"version": 1,
"_meta": {
"description": "using component templates"
}
}
services/search/pkg/opensearch/os_dsl_query_root.go
+1 -1
View File
@@ -50,7 +50,7 @@ func (q *RootQuery) String() string {
} }
type RootQueryOptions struct { type RootQueryOptions struct {
Highlight RootQueryHighlight `json:"highlight,omitempty"` Highlight *RootQueryHighlight `json:"highlight,omitempty"`
} }
type RootQueryHighlight struct { type RootQueryHighlight struct {
services/search/pkg/opensearch/os_dsl_query_root_test.go
+1 -1
View File
@@ -29,7 +29,7 @@ func TestRootQuery(t *testing.T) {
Got: opensearch.NewRootQuery( Got: opensearch.NewRootQuery(
opensearch.NewTermQuery[string]("content").Value("content"), opensearch.NewTermQuery[string]("content").Value("content"),
opensearch.RootQueryOptions{ opensearch.RootQueryOptions{
Highlight: opensearch.RootQueryHighlight{ Highlight: &opensearch.RootQueryHighlight{
PreTags: []string{"<b>"}, PreTags: []string{"<b>"},
PostTags: []string{"</b>"}, PostTags: []string{"</b>"},
Fields: map[string]opensearch.RootQueryHighlight{ Fields: map[string]opensearch.RootQueryHighlight{
services/search/pkg/opensearch/os_index.go
+142
View File
@@ -0,0 +1,142 @@
package opensearch
import (
"bytes"
"context"
"embed"
"errors"
"fmt"
"path"
"reflect"
"github.com/go-jose/go-jose/v3/json"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
"github.com/tidwall/gjson"
)
var (
ErrManualActionRequired = errors.New("manual action required")
IndexManagerLatest = IndexIndexManagerResourceV1
IndexIndexManagerResourceV1 IndexManager = "resource_v1.json"
)
//go:embed internal/indexes/*.json
var indexes embed.FS
type IndexManager string
func (m IndexManager) String() string {
b, err := m.MarshalJSON()
if err != nil {
return ""
}
return string(b)
}
func (m IndexManager) MarshalJSON() ([]byte, error) {
filePath := string(m)
body, err := indexes.ReadFile(path.Join("./internal/indexes", filePath))
switch {
case err != nil:
return nil, fmt.Errorf("failed to read index file %s: %w", filePath, err)
case len(body) <= 0:
return nil, fmt.Errorf("index file %s is empty", filePath)
}
return body, nil
}
func (m IndexManager) Apply(ctx context.Context, name string, client *opensearchgoAPI.Client) error {
localIndexB, err := m.MarshalJSON()
if err != nil {
return fmt.Errorf("failed to marshal index %s: %w", name, err)
}
indicesExistsResp, err := client.Indices.Exists(ctx, opensearchgoAPI.IndicesExistsReq{
Indices: []string{name},
})
switch {
case indicesExistsResp != nil && indicesExistsResp.StatusCode == 404:
break
case err != nil:
return fmt.Errorf("failed to check if index %s exists: %w", name, err)
case indicesExistsResp == nil:
return fmt.Errorf("indicesExistsResp is nil for index %s", name)
}
if indicesExistsResp.StatusCode == 200 {
resp, err := client.Indices.Get(ctx, opensearchgoAPI.IndicesGetReq{
Indices: []string{name},
})
if err != nil {
return fmt.Errorf("failed to get index %s: %w", name, err)
}
remoteIndex, ok := resp.Indices[name]
if !ok {
return fmt.Errorf("index %s not found in response", name)
}
remoteIndexB, err := json.Marshal(remoteIndex)
if err != nil {
return fmt.Errorf("failed to marshal index %s: %w", name, err)
}
localIndexJson := gjson.ParseBytes(localIndexB)
remoteIndexJson := gjson.ParseBytes(remoteIndexB)
compare := func(lvPath, rvPath string) (any, any, bool) {
lv := localIndexJson.Get(lvPath).Raw
rv := remoteIndexJson.Get(rvPath).Raw
var lvv, rvv interface{}
if err := json.Unmarshal([]byte(lv), &lvv); err != nil {
return nil, nil, false
}
if err := json.Unmarshal([]byte(rv), &rvv); err != nil {
return nil, nil, false
}
return lv, rv, reflect.DeepEqual(lvv, rvv)
}
var errs []error
for k := range localIndexJson.Get("settings").Map() {
if lv, rv, ok := compare("settings."+k, "settings.index."+k); !ok {
errs = append(errs, fmt.Errorf("settings.%s local %s, remote %s", k, lv, rv))
}
}
for k := range localIndexJson.Get("mappings.properties").Map() {
if _, _, ok := compare("mappings.properties."+k, "mappings.properties."+k); !ok {
errs = append(errs, fmt.Errorf("mappings.properties.%s", k))
}
}
if errs != nil {
return fmt.Errorf(
"index %s allready exists and is different from the requested version, %w: %w",
name,
ErrManualActionRequired,
errors.Join(errs...),
)
}
return nil // Index is already up to date, no action needed
}
createResp, err := client.Indices.Create(ctx, opensearchgoAPI.IndicesCreateReq{
Index: name,
Body: bytes.NewReader(localIndexB),
})
switch {
case err != nil:
return fmt.Errorf("failed to create index %s: %w", name, err)
case !createResp.Acknowledged:
return fmt.Errorf("failed to create index %s: not acknowledged", name)
}
return nil
}
services/search/pkg/opensearch/os_index_template.go
-65
View File
@@ -1,65 +0,0 @@
package opensearch
import (
"bytes"
"context"
"embed"
"fmt"
"path"
opensearchgoAPI "github.com/opensearch-project/opensearch-go/v4/opensearchapi"
)
var (
IndexTemplateResourceV1 IndexTemplate = [2]string{"opencloud-default-resource", "resource_v1.json"}
)
//go:embed internal/indices/*.json
var indexTemplates embed.FS
type IndexTemplate [2]string
func (t IndexTemplate) Name() string {
return t[0]
}
func (t IndexTemplate) String() string {
b, err := t.MarshalJSON()
if err != nil {
return ""
}
return string(b)
}
func (t IndexTemplate) MarshalJSON() ([]byte, error) {
file := t[1]
body, err := indexTemplates.ReadFile(path.Join("./internal/indices", file))
switch {
case err != nil:
return nil, fmt.Errorf("failed to read index template file %s: %w", file, err)
case len(body) <= 0:
return nil, fmt.Errorf("index template file %s is empty", file)
}
return body, nil
}
func (t IndexTemplate) Apply(ctx context.Context, client *opensearchgoAPI.Client) error {
body, err := t.MarshalJSON()
if err != nil {
return fmt.Errorf("failed to inspect index template %s: %w", t[1], err)
}
resp, err := client.IndexTemplate.Create(ctx, opensearchgoAPI.IndexTemplateCreateReq{
IndexTemplate: t.Name(),
Body: bytes.NewBuffer(body),
})
switch {
case err != nil:
return fmt.Errorf("failed to create index template %s: %w", t.Name(), err)
case !resp.Acknowledged:
return fmt.Errorf("failed to create index template %s: not acknowledged", t.Name())
default:
return nil
}
}
services/search/pkg/opensearch/os_index_template_test.go
-34
View File
@@ -1,34 +0,0 @@
package opensearch_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestIndexTemplates(t *testing.T) {
tc := opensearchtest.NewDefaultTestClient(t)
t.Run("index templates plausibility", func(t *testing.T) {
tests := []opensearchtest.TableTest[opensearch.IndexTemplate, struct{}]{
{
Name: "empty",
Got: opensearch.IndexTemplateResourceV1,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
body, err := test.Got.MarshalJSON()
require.NoError(t, err)
require.NotEmpty(t, body)
require.NotEmpty(t, test.Got.String())
require.JSONEq(t, test.Got.String(), string(body))
require.NotEmpty(t, test.Got.Name())
require.NoError(t, test.Got.Apply(t.Context(), tc.Client()))
})
}
})
}
services/search/pkg/opensearch/os_index_test.go
+62
View File
@@ -0,0 +1,62 @@
package opensearch_test
import (
"testing"
"github.com/stretchr/testify/require"
"github.com/tidwall/sjson"
"github.com/opencloud-eu/opencloud/services/search/pkg/opensearch"
opensearchtest "github.com/opencloud-eu/opencloud/services/search/pkg/opensearch/internal/test"
)
func TestIndex(t *testing.T) {
t.Run("index plausibility", func(t *testing.T) {
tests := []opensearchtest.TableTest[opensearch.IndexManager, struct{}]{
{
Name: "empty",
Got: opensearch.IndexManagerLatest,
},
}
tc := opensearchtest.NewDefaultTestClient(t)
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
indexName := "opencloud-test-resource"
tc.Require.IndicesReset([]string{indexName})
body, err := test.Got.MarshalJSON()
require.NoError(t, err)
require.NotEmpty(t, body)
require.NotEmpty(t, test.Got.String())
require.JSONEq(t, test.Got.String(), string(body))
require.NoError(t, test.Got.Apply(t.Context(), indexName, tc.Client()))
})
}
})
t.Run("does not create index if it already exists and is up to date", func(t *testing.T) {
indexManager := opensearch.IndexManagerLatest
indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
tc.Require.IndicesCreate(indexName, indexManager.String())
require.NoError(t, indexManager.Apply(t.Context(), indexName, tc.Client()))
})
t.Run("fails to create index if it already exists but is not up to date", func(t *testing.T) {
indexManager := opensearch.IndexManagerLatest
indexName := "opencloud-test-resource"
tc := opensearchtest.NewDefaultTestClient(t)
tc.Require.IndicesReset([]string{indexName})
body, err := sjson.Set(indexManager.String(), "settings.number_of_shards", "2")
require.NoError(t, err)
tc.Require.IndicesCreate(indexName, body)
require.ErrorIs(t, indexManager.Apply(t.Context(), indexName, tc.Client()), opensearch.ErrManualActionRequired)
})
}
vendor/github.com/tidwall/sjson/LICENSE
Generated Vendored
+21
View File
@@ -0,0 +1,21 @@
The MIT License (MIT)
Copyright (c) 2016 Josh Baker
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
vendor/github.com/tidwall/sjson/README.md
Generated Vendored
+278
View File
@@ -0,0 +1,278 @@
<p align="center">
<img
src="logo.png"
width="240" height="78" border="0" alt="SJSON">
<br>
<a href="https://godoc.org/github.com/tidwall/sjson"><img src="https://img.shields.io/badge/api-reference-blue.svg?style=flat-square" alt="GoDoc"></a>
</p>
<p align="center">set a json value quickly</p>
SJSON is a Go package that provides a [very fast](#performance) and simple way to set a value in a json document.
For quickly retrieving json values check out [GJSON](https://github.com/tidwall/gjson).
For a command line interface check out [JJ](https://github.com/tidwall/jj).
Getting Started
===============
Installing
----------
To start using SJSON, install Go and run `go get`:
```sh
$ go get -u github.com/tidwall/sjson
```
This will retrieve the library.
Set a value
-----------
Set sets the value for the specified path.
A path is in dot syntax, such as "name.last" or "age".
This function expects that the json is well-formed and validated.
Invalid json will not panic, but it may return back unexpected results.
Invalid paths may return an error.
```go
package main
import "github.com/tidwall/sjson"
const json = `{"name":{"first":"Janet","last":"Prichard"},"age":47}`
func main() {
value, _ := sjson.Set(json, "name.last", "Anderson")
println(value)
}
```
This will print:
```json
{"name":{"first":"Janet","last":"Anderson"},"age":47}
```
Path syntax
-----------
A path is a series of keys separated by a dot.
The dot and colon characters can be escaped with ``\``.
```json
{
"name": {"first": "Tom", "last": "Anderson"},
"age":37,
"children": ["Sara","Alex","Jack"],
"fav.movie": "Deer Hunter",
"friends": [
{"first": "James", "last": "Murphy"},
{"first": "Roger", "last": "Craig"}
]
}
```
```
"name.last" >> "Anderson"
"age" >> 37
"children.1" >> "Alex"
"friends.1.last" >> "Craig"
```
The `-1` key can be used to append a value to an existing array:
```
"children.-1" >> appends a new value to the end of the children array
```
Normally number keys are used to modify arrays, but it's possible to force a numeric object key by using the colon character:
```json
{
"users":{
"2313":{"name":"Sara"},
"7839":{"name":"Andy"}
}
}
```
A colon path would look like:
```
"users.:2313.name" >> "Sara"
```
Supported types
---------------
Pretty much any type is supported:
```go
sjson.Set(`{"key":true}`, "key", nil)
sjson.Set(`{"key":true}`, "key", false)
sjson.Set(`{"key":true}`, "key", 1)
sjson.Set(`{"key":true}`, "key", 10.5)
sjson.Set(`{"key":true}`, "key", "hello")
sjson.Set(`{"key":true}`, "key", []string{"hello", "world"})
sjson.Set(`{"key":true}`, "key", map[string]interface{}{"hello":"world"})
```
When a type is not recognized, SJSON will fallback to the `encoding/json` Marshaller.
Examples
--------
Set a value from empty document:
```go
value, _ := sjson.Set("", "name", "Tom")
println(value)
// Output:
// {"name":"Tom"}
```
Set a nested value from empty document:
```go
value, _ := sjson.Set("", "name.last", "Anderson")
println(value)
// Output:
// {"name":{"last":"Anderson"}}
```
Set a new value:
```go
value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.first", "Sara")
println(value)
// Output:
// {"name":{"first":"Sara","last":"Anderson"}}
```
Update an existing value:
```go
value, _ := sjson.Set(`{"name":{"last":"Anderson"}}`, "name.last", "Smith")
println(value)
// Output:
// {"name":{"last":"Smith"}}
```
Set a new array value:
```go
value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.2", "Sara")
println(value)
// Output:
// {"friends":["Andy","Carol","Sara"]
```
Append an array value by using the `-1` key in a path:
```go
value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.-1", "Sara")
println(value)
// Output:
// {"friends":["Andy","Carol","Sara"]
```
Append an array value that is past the end:
```go
value, _ := sjson.Set(`{"friends":["Andy","Carol"]}`, "friends.4", "Sara")
println(value)
// Output:
// {"friends":["Andy","Carol",null,null,"Sara"]
```
Delete a value:
```go
value, _ := sjson.Delete(`{"name":{"first":"Sara","last":"Anderson"}}`, "name.first")
println(value)
// Output:
// {"name":{"last":"Anderson"}}
```
Delete an array value:
```go
value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.1")
println(value)
// Output:
// {"friends":["Andy"]}
```
Delete the last array value:
```go
value, _ := sjson.Delete(`{"friends":["Andy","Carol"]}`, "friends.-1")
println(value)
// Output:
// {"friends":["Andy"]}
```
## Performance
Benchmarks of SJSON alongside [encoding/json](https://golang.org/pkg/encoding/json/),
[ffjson](https://github.com/pquerna/ffjson),
[EasyJSON](https://github.com/mailru/easyjson),
and [Gabs](https://github.com/Jeffail/gabs)
```
Benchmark_SJSON-8 3000000 805 ns/op 1077 B/op 3 allocs/op
Benchmark_SJSON_ReplaceInPlace-8 3000000 449 ns/op 0 B/op 0 allocs/op
Benchmark_JSON_Map-8 300000 21236 ns/op 6392 B/op 150 allocs/op
Benchmark_JSON_Struct-8 300000 14691 ns/op 1789 B/op 24 allocs/op
Benchmark_Gabs-8 300000 21311 ns/op 6752 B/op 150 allocs/op
Benchmark_FFJSON-8 300000 17673 ns/op 3589 B/op 47 allocs/op
Benchmark_EasyJSON-8 1500000 3119 ns/op 1061 B/op 13 allocs/op
```
JSON document used:
```json
{
"widget": {
"debug": "on",
"window": {
"title": "Sample Konfabulator Widget",
"name": "main_window",
"width": 500,
"height": 500
},
"image": {
"src": "Images/Sun.png",
"hOffset": 250,
"vOffset": 250,
"alignment": "center"
},
"text": {
"data": "Click Here",
"size": 36,
"style": "bold",
"vOffset": 100,
"alignment": "center",
"onMouseUp": "sun1.opacity = (sun1.opacity / 100) * 90;"
}
}
}
```
Each operation was rotated though one of the following search paths:
```
widget.window.name
widget.image.hOffset
widget.text.onMouseUp
```
*These benchmarks were run on a MacBook Pro 15" 2.8 GHz Intel Core i7 using Go 1.7 and can be be found [here](https://github.com/tidwall/sjson-benchmarks)*.
## Contact
Josh Baker [@tidwall](http://twitter.com/tidwall)
## License
SJSON source code is available under the MIT [License](/LICENSE).
vendor/github.com/tidwall/sjson/logo.png
Generated Vendored
BIN
View File
Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

vendor/github.com/tidwall/sjson/sjson.go
Generated Vendored
+737
View File
@@ -0,0 +1,737 @@
// Package sjson provides setting json values.
package sjson
import (
jsongo "encoding/json"
"sort"
"strconv"
"unsafe"
"github.com/tidwall/gjson"
)
type errorType struct {
msg string
}
func (err *errorType) Error() string {
return err.msg
}
// Options represents additional options for the Set and Delete functions.
type Options struct {
// Optimistic is a hint that the value likely exists which
// allows for the sjson to perform a fast-track search and replace.
Optimistic bool
// ReplaceInPlace is a hint to replace the input json rather than
// allocate a new json byte slice. When this field is specified
// the input json will not longer be valid and it should not be used
// In the case when the destination slice doesn't have enough free
// bytes to replace the data in place, a new bytes slice will be
// created under the hood.
// The Optimistic flag must be set to true and the input must be a
// byte slice in order to use this field.
ReplaceInPlace bool
}
type pathResult struct {
part string // current key part
gpart string // gjson get part
path string // remaining path
force bool // force a string key
more bool // there is more path to parse
}
func isSimpleChar(ch byte) bool {
switch ch {
case '|', '#', '@', '*', '?':
return false
default:
return true
}
}
func parsePath(path string) (res pathResult, simple bool) {
var r pathResult
if len(path) > 0 && path[0] == ':' {
r.force = true
path = path[1:]
}
for i := 0; i < len(path); i++ {
if path[i] == '.' {
r.part = path[:i]
r.gpart = path[:i]
r.path = path[i+1:]
r.more = true
return r, true
}
if !isSimpleChar(path[i]) {
return r, false
}
if path[i] == '\\' {
// go into escape mode. this is a slower path that
// strips off the escape character from the part.
epart := []byte(path[:i])
gpart := []byte(path[:i+1])
i++
if i < len(path) {
epart = append(epart, path[i])
gpart = append(gpart, path[i])
i++
for ; i < len(path); i++ {
if path[i] == '\\' {
gpart = append(gpart, '\\')
i++
if i < len(path) {
epart = append(epart, path[i])
gpart = append(gpart, path[i])
}
continue
} else if path[i] == '.' {
r.part = string(epart)
r.gpart = string(gpart)
r.path = path[i+1:]
r.more = true
return r, true
} else if !isSimpleChar(path[i]) {
return r, false
}
epart = append(epart, path[i])
gpart = append(gpart, path[i])
}
}
// append the last part
r.part = string(epart)
r.gpart = string(gpart)
return r, true
}
}
r.part = path
r.gpart = path
return r, true
}
func mustMarshalString(s string) bool {
for i := 0; i < len(s); i++ {
if s[i] < ' ' || s[i] > 0x7f || s[i] == '"' || s[i] == '\\' {
return true
}
}
return false
}
// appendStringify makes a json string and appends to buf.
func appendStringify(buf []byte, s string) []byte {
if mustMarshalString(s) {
b, _ := jsongo.Marshal(s)
return append(buf, b...)
}
buf = append(buf, '"')
buf = append(buf, s...)
buf = append(buf, '"')
return buf
}
// appendBuild builds a json block from a json path.
func appendBuild(buf []byte, array bool, paths []pathResult, raw string,
stringify bool) []byte {
if !array {
buf = appendStringify(buf, paths[0].part)
buf = append(buf, ':')
}
if len(paths) > 1 {
n, numeric := atoui(paths[1])
if numeric || (!paths[1].force && paths[1].part == "-1") {
buf = append(buf, '[')
buf = appendRepeat(buf, "null,", n)
buf = appendBuild(buf, true, paths[1:], raw, stringify)
buf = append(buf, ']')
} else {
buf = append(buf, '{')
buf = appendBuild(buf, false, paths[1:], raw, stringify)
buf = append(buf, '}')
}
} else {
if stringify {
buf = appendStringify(buf, raw)
} else {
buf = append(buf, raw...)
}
}
return buf
}
// atoui does a rip conversion of string -> unigned int.
func atoui(r pathResult) (n int, ok bool) {
if r.force {
return 0, false
}
for i := 0; i < len(r.part); i++ {
if r.part[i] < '0' || r.part[i] > '9' {
return 0, false
}
n = n*10 + int(r.part[i]-'0')
}
return n, true
}
// appendRepeat repeats string "n" times and appends to buf.
func appendRepeat(buf []byte, s string, n int) []byte {
for i := 0; i < n; i++ {
buf = append(buf, s...)
}
return buf
}
// trim does a rip trim
func trim(s string) string {
for len(s) > 0 {
if s[0] <= ' ' {
s = s[1:]
continue
}
break
}
for len(s) > 0 {
if s[len(s)-1] <= ' ' {
s = s[:len(s)-1]
continue
}
break
}
return s
}
// deleteTailItem deletes the previous key or comma.
func deleteTailItem(buf []byte) ([]byte, bool) {
loop:
for i := len(buf) - 1; i >= 0; i-- {
// look for either a ',',':','['
switch buf[i] {
case '[':
return buf, true
case ',':
return buf[:i], false
case ':':
// delete tail string
i--
for ; i >= 0; i-- {
if buf[i] == '"' {
i--
for ; i >= 0; i-- {
if buf[i] == '"' {
i--
if i >= 0 && buf[i] == '\\' {
i--
continue
}
for ; i >= 0; i-- {
// look for either a ',','{'
switch buf[i] {
case '{':
return buf[:i+1], true
case ',':
return buf[:i], false
}
}
}
}
break
}
}
break loop
}
}
return buf, false
}
var errNoChange = &errorType{"no change"}
func appendRawPaths(buf []byte, jstr string, paths []pathResult, raw string,
stringify, del bool) ([]byte, error) {
var err error
var res gjson.Result
var found bool
if del {
if paths[0].part == "-1" && !paths[0].force {
res = gjson.Get(jstr, "#")
if res.Int() > 0 {
res = gjson.Get(jstr, strconv.FormatInt(int64(res.Int()-1), 10))
found = true
}
}
}
if !found {
res = gjson.Get(jstr, paths[0].gpart)
}
if res.Index > 0 {
if len(paths) > 1 {
buf = append(buf, jstr[:res.Index]...)
buf, err = appendRawPaths(buf, res.Raw, paths[1:], raw,
stringify, del)
if err != nil {
return nil, err
}
buf = append(buf, jstr[res.Index+len(res.Raw):]...)
return buf, nil
}
buf = append(buf, jstr[:res.Index]...)
var exidx int // additional forward stripping
if del {
var delNextComma bool
buf, delNextComma = deleteTailItem(buf)
if delNextComma {
i, j := res.Index+len(res.Raw), 0
for ; i < len(jstr); i, j = i+1, j+1 {
if jstr[i] <= ' ' {
continue
}
if jstr[i] == ',' {
exidx = j + 1
}
break
}
}
} else {
if stringify {
buf = appendStringify(buf, raw)
} else {
buf = append(buf, raw...)
}
}
buf = append(buf, jstr[res.Index+len(res.Raw)+exidx:]...)
return buf, nil
}
if del {
return nil, errNoChange
}
n, numeric := atoui(paths[0])
isempty := true
for i := 0; i < len(jstr); i++ {
if jstr[i] > ' ' {
isempty = false
break
}
}
if isempty {
if numeric {
jstr = "[]"
} else {
jstr = "{}"
}
}
jsres := gjson.Parse(jstr)
if jsres.Type != gjson.JSON {
if numeric {
jstr = "[]"
} else {
jstr = "{}"
}
jsres = gjson.Parse(jstr)
}
var comma bool
for i := 1; i < len(jsres.Raw); i++ {
if jsres.Raw[i] <= ' ' {
continue
}
if jsres.Raw[i] == '}' || jsres.Raw[i] == ']' {
break
}
comma = true
break
}
switch jsres.Raw[0] {
default:
return nil, &errorType{"json must be an object or array"}
case '{':
end := len(jsres.Raw) - 1
for ; end > 0; end-- {
if jsres.Raw[end] == '}' {
break
}
}
buf = append(buf, jsres.Raw[:end]...)
if comma {
buf = append(buf, ',')
}
buf = appendBuild(buf, false, paths, raw, stringify)
buf = append(buf, '}')
return buf, nil
case '[':
var appendit bool
if !numeric {
if paths[0].part == "-1" && !paths[0].force {
appendit = true
} else {
return nil, &errorType{
"cannot set array element for non-numeric key '" +
paths[0].part + "'"}
}
}
if appendit {
njson := trim(jsres.Raw)
if njson[len(njson)-1] == ']' {
njson = njson[:len(njson)-1]
}
buf = append(buf, njson...)
if comma {
buf = append(buf, ',')
}
buf = appendBuild(buf, true, paths, raw, stringify)
buf = append(buf, ']')
return buf, nil
}
buf = append(buf, '[')
ress := jsres.Array()
for i := 0; i < len(ress); i++ {
if i > 0 {
buf = append(buf, ',')
}
buf = append(buf, ress[i].Raw...)
}
if len(ress) == 0 {
buf = appendRepeat(buf, "null,", n-len(ress))
} else {
buf = appendRepeat(buf, ",null", n-len(ress))
if comma {
buf = append(buf, ',')
}
}
buf = appendBuild(buf, true, paths, raw, stringify)
buf = append(buf, ']')
return buf, nil
}
}
func isOptimisticPath(path string) bool {
for i := 0; i < len(path); i++ {
if path[i] < '.' || path[i] > 'z' {
return false
}
if path[i] > '9' && path[i] < 'A' {
return false
}
if path[i] > 'z' {
return false
}
}
return true
}
// Set sets a json value for the specified path.
// A path is in dot syntax, such as "name.last" or "age".
// This function expects that the json is well-formed, and does not validate.
// Invalid json will not panic, but it may return back unexpected results.
// An error is returned if the path is not valid.
//
// A path is a series of keys separated by a dot.
//
// {
// "name": {"first": "Tom", "last": "Anderson"},
// "age":37,
// "children": ["Sara","Alex","Jack"],
// "friends": [
// {"first": "James", "last": "Murphy"},
// {"first": "Roger", "last": "Craig"}
// ]
// }
// "name.last" >> "Anderson"
// "age" >> 37
// "children.1" >> "Alex"
//
func Set(json, path string, value interface{}) (string, error) {
return SetOptions(json, path, value, nil)
}
// SetBytes sets a json value for the specified path.
// If working with bytes, this method preferred over
// Set(string(data), path, value)
func SetBytes(json []byte, path string, value interface{}) ([]byte, error) {
return SetBytesOptions(json, path, value, nil)
}
// SetRaw sets a raw json value for the specified path.
// This function works the same as Set except that the value is set as a
// raw block of json. This allows for setting premarshalled json objects.
func SetRaw(json, path, value string) (string, error) {
return SetRawOptions(json, path, value, nil)
}
// SetRawOptions sets a raw json value for the specified path with options.
// This furnction works the same as SetOptions except that the value is set
// as a raw block of json. This allows for setting premarshalled json objects.
func SetRawOptions(json, path, value string, opts *Options) (string, error) {
var optimistic bool
if opts != nil {
optimistic = opts.Optimistic
}
res, err := set(json, path, value, false, false, optimistic, false)
if err == errNoChange {
return json, nil
}
return string(res), err
}
// SetRawBytes sets a raw json value for the specified path.
// If working with bytes, this method preferred over
// SetRaw(string(data), path, value)
func SetRawBytes(json []byte, path string, value []byte) ([]byte, error) {
return SetRawBytesOptions(json, path, value, nil)
}
type dtype struct{}
// Delete deletes a value from json for the specified path.
func Delete(json, path string) (string, error) {
return Set(json, path, dtype{})
}
// DeleteBytes deletes a value from json for the specified path.
func DeleteBytes(json []byte, path string) ([]byte, error) {
return SetBytes(json, path, dtype{})
}
type stringHeader struct {
data unsafe.Pointer
len int
}
type sliceHeader struct {
data unsafe.Pointer
len int
cap int
}
func set(jstr, path, raw string,
stringify, del, optimistic, inplace bool) ([]byte, error) {
if path == "" {
return []byte(jstr), &errorType{"path cannot be empty"}
}
if !del && optimistic && isOptimisticPath(path) {
res := gjson.Get(jstr, path)
if res.Exists() && res.Index > 0 {
sz := len(jstr) - len(res.Raw) + len(raw)
if stringify {
sz += 2
}
if inplace && sz <= len(jstr) {
if !stringify || !mustMarshalString(raw) {
jsonh := *(*stringHeader)(unsafe.Pointer(&jstr))
jsonbh := sliceHeader{
data: jsonh.data, len: jsonh.len, cap: jsonh.len}
jbytes := *(*[]byte)(unsafe.Pointer(&jsonbh))
if stringify {
jbytes[res.Index] = '"'
copy(jbytes[res.Index+1:], []byte(raw))
jbytes[res.Index+1+len(raw)] = '"'
copy(jbytes[res.Index+1+len(raw)+1:],
jbytes[res.Index+len(res.Raw):])
} else {
copy(jbytes[res.Index:], []byte(raw))
copy(jbytes[res.Index+len(raw):],
jbytes[res.Index+len(res.Raw):])
}
return jbytes[:sz], nil
}
return []byte(jstr), nil
}
buf := make([]byte, 0, sz)
buf = append(buf, jstr[:res.Index]...)
if stringify {
buf = appendStringify(buf, raw)
} else {
buf = append(buf, raw...)
}
buf = append(buf, jstr[res.Index+len(res.Raw):]...)
return buf, nil
}
}
var paths []pathResult
r, simple := parsePath(path)
if simple {
paths = append(paths, r)
for r.more {
r, simple = parsePath(r.path)
if !simple {
break
}
paths = append(paths, r)
}
}
if !simple {
if del {
return []byte(jstr),
&errorType{"cannot delete value from a complex path"}
}
return setComplexPath(jstr, path, raw, stringify)
}
njson, err := appendRawPaths(nil, jstr, paths, raw, stringify, del)
if err != nil {
return []byte(jstr), err
}
return njson, nil
}
func setComplexPath(jstr, path, raw string, stringify bool) ([]byte, error) {
res := gjson.Get(jstr, path)
if !res.Exists() || !(res.Index != 0 || len(res.Indexes) != 0) {
return []byte(jstr), errNoChange
}
if res.Index != 0 {
njson := []byte(jstr[:res.Index])
if stringify {
njson = appendStringify(njson, raw)
} else {
njson = append(njson, raw...)
}
njson = append(njson, jstr[res.Index+len(res.Raw):]...)
jstr = string(njson)
}
if len(res.Indexes) > 0 {
type val struct {
index int
res gjson.Result
}
vals := make([]val, 0, len(res.Indexes))
res.ForEach(func(_, vres gjson.Result) bool {
vals = append(vals, val{res: vres})
return true
})
if len(res.Indexes) != len(vals) {
return []byte(jstr), errNoChange
}
for i := 0; i < len(res.Indexes); i++ {
vals[i].index = res.Indexes[i]
}
sort.SliceStable(vals, func(i, j int) bool {
return vals[i].index > vals[j].index
})
for _, val := range vals {
vres := val.res
index := val.index
njson := []byte(jstr[:index])
if stringify {
njson = appendStringify(njson, raw)
} else {
njson = append(njson, raw...)
}
njson = append(njson, jstr[index+len(vres.Raw):]...)
jstr = string(njson)
}
}
return []byte(jstr), nil
}
// SetOptions sets a json value for the specified path with options.
// A path is in dot syntax, such as "name.last" or "age".
// This function expects that the json is well-formed, and does not validate.
// Invalid json will not panic, but it may return back unexpected results.
// An error is returned if the path is not valid.
func SetOptions(json, path string, value interface{},
opts *Options) (string, error) {
if opts != nil {
if opts.ReplaceInPlace {
// it's not safe to replace bytes in-place for strings
// copy the Options and set options.ReplaceInPlace to false.
nopts := *opts
opts = &nopts
opts.ReplaceInPlace = false
}
}
jsonh := *(*stringHeader)(unsafe.Pointer(&json))
jsonbh := sliceHeader{data: jsonh.data, len: jsonh.len, cap: jsonh.len}
jsonb := *(*[]byte)(unsafe.Pointer(&jsonbh))
res, err := SetBytesOptions(jsonb, path, value, opts)
return string(res), err
}
// SetBytesOptions sets a json value for the specified path with options.
// If working with bytes, this method preferred over
// SetOptions(string(data), path, value)
func SetBytesOptions(json []byte, path string, value interface{},
opts *Options) ([]byte, error) {
var optimistic, inplace bool
if opts != nil {
optimistic = opts.Optimistic
inplace = opts.ReplaceInPlace
}
jstr := *(*string)(unsafe.Pointer(&json))
var res []byte
var err error
switch v := value.(type) {
default:
b, merr := jsongo.Marshal(value)
if merr != nil {
return nil, merr
}
raw := *(*string)(unsafe.Pointer(&b))
res, err = set(jstr, path, raw, false, false, optimistic, inplace)
case dtype:
res, err = set(jstr, path, "", false, true, optimistic, inplace)
case string:
res, err = set(jstr, path, v, true, false, optimistic, inplace)
case []byte:
raw := *(*string)(unsafe.Pointer(&v))
res, err = set(jstr, path, raw, true, false, optimistic, inplace)
case bool:
if v {
res, err = set(jstr, path, "true", false, false, optimistic, inplace)
} else {
res, err = set(jstr, path, "false", false, false, optimistic, inplace)
}
case int8:
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
false, false, optimistic, inplace)
case int16:
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
false, false, optimistic, inplace)
case int32:
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
false, false, optimistic, inplace)
case int64:
res, err = set(jstr, path, strconv.FormatInt(int64(v), 10),
false, false, optimistic, inplace)
case uint8:
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
false, false, optimistic, inplace)
case uint16:
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
false, false, optimistic, inplace)
case uint32:
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
false, false, optimistic, inplace)
case uint64:
res, err = set(jstr, path, strconv.FormatUint(uint64(v), 10),
false, false, optimistic, inplace)
case float32:
res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64),
false, false, optimistic, inplace)
case float64:
res, err = set(jstr, path, strconv.FormatFloat(float64(v), 'f', -1, 64),
false, false, optimistic, inplace)
}
if err == errNoChange {
return json, nil
}
return res, err
}
// SetRawBytesOptions sets a raw json value for the specified path with options.
// If working with bytes, this method preferred over
// SetRawOptions(string(data), path, value, opts)
func SetRawBytesOptions(json []byte, path string, value []byte,
opts *Options) ([]byte, error) {
jstr := *(*string)(unsafe.Pointer(&json))
vstr := *(*string)(unsafe.Pointer(&value))
var optimistic, inplace bool
if opts != nil {
optimistic = opts.Optimistic
inplace = opts.ReplaceInPlace
}
res, err := set(jstr, path, vstr, false, false, optimistic, inplace)
if err == errNoChange {
return json, nil
}
return res, err
}
vendor/modules.txt
Vendored
+3
View File
@@ -1830,6 +1830,9 @@ github.com/tidwall/match
# github.com/tidwall/pretty v1.2.1 # github.com/tidwall/pretty v1.2.1
## explicit; go 1.16 ## explicit; go 1.16
github.com/tidwall/pretty github.com/tidwall/pretty
# github.com/tidwall/sjson v1.2.5
## explicit; go 1.14
github.com/tidwall/sjson
# github.com/tinylib/msgp v1.3.0 # github.com/tinylib/msgp v1.3.0
## explicit; go 1.20 ## explicit; go 1.20
github.com/tinylib/msgp/msgp github.com/tinylib/msgp/msgp