diff --git a/accounts/pkg/indexer/indexer/errors/errors.go b/accounts/pkg/indexer/indexer/errors/errors.go new file mode 100644 index 000000000..85e785c89 --- /dev/null +++ b/accounts/pkg/indexer/indexer/errors/errors.go @@ -0,0 +1,31 @@ +package errors + +import ( + "fmt" +) + +type AlreadyExistsErr struct { + TypeName, Key, Value string +} + +func (e *AlreadyExistsErr) Error() string { + return fmt.Sprintf("%s with %s=%s does already exist", e.TypeName, e.Key, e.Value) +} + +func IsAlreadyExistsErr(e error) bool { + _, ok := e.(*AlreadyExistsErr) + return ok +} + +type NotFoundErr struct { + TypeName, Key, Value string +} + +func (e *NotFoundErr) Error() string { + return fmt.Sprintf("%s with %s=%s not found", e.TypeName, e.Key, e.Value) +} + +func IsNotFoundErr(e error) bool { + _, ok := e.(*NotFoundErr) + return ok +} diff --git a/accounts/pkg/indexer/indexer/index/cs3/unique.go b/accounts/pkg/indexer/indexer/index/cs3/unique.go new file mode 100644 index 000000000..4d118484e --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/cs3/unique.go @@ -0,0 +1,347 @@ +package cs3 + +import ( + "context" + "fmt" + user "github.com/cs3org/go-cs3apis/cs3/identity/user/v1beta1" + v1beta11 "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1" + provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/cs3org/reva/pkg/rgrpc/todo/pool" + "github.com/cs3org/reva/pkg/token" + "github.com/cs3org/reva/pkg/token/manager/jwt" + idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "google.golang.org/grpc/metadata" + "io" + "io/ioutil" + "net/http" + "os" + "path" + "path/filepath" + "strings" +) + +type Unique struct { + indexBy string + typeName string + filesDir string + indexBaseDir string + indexRootDir string + + tokenManager token.Manager + storageProvider provider.ProviderAPIClient + dataProvider dataProviderClient // Used to create and download data via http, bypassing reva upload protocol + + cs3conf *Config +} + +type Config struct { + ProviderAddr string + DataURL string + DataPrefix string + JWTSecret string + ServiceUserName string + ServiceUserUUID string +} + +// NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be +// called afterward to ensure correct on-disk structure. +func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string, cfg *Config) Unique { + return Unique{ + indexBy: indexBy, + typeName: typeName, + filesDir: filesDir, + indexBaseDir: indexBaseDir, + indexRootDir: path.Join(indexBaseDir, strings.Join([]string{"unique", typeName, indexBy}, ".")), + cs3conf: cfg, + dataProvider: dataProviderClient{ + client: http.Client{ + Transport: http.DefaultTransport, + }, + }, + } +} + +func (idx *Unique) Init() error { + tokenManager, err := jwt.New(map[string]interface{}{ + "secret": idx.cs3conf.JWTSecret, + }) + + if err != nil { + return err + } + + idx.tokenManager = tokenManager + + client, err := pool.GetStorageProviderServiceClient(idx.cs3conf.ProviderAddr) + if err != nil { + return err + } + + idx.storageProvider = client + + ctx := context.Background() + tk, err := idx.authenticate(ctx) + if err != nil { + return err + } + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, tk) + + if err := idx.makeDirIfNotExists(ctx, idx.indexBaseDir); err != nil { + return err + } + + if err := idx.makeDirIfNotExists(ctx, idx.indexRootDir); err != nil { + return err + } + + return nil +} + +// Add adds a value to the index, returns the path to the root-document +func (idx *Unique) Add(id, v string) (string, error) { + newName := idx.indexURL(v) + if err := idx.createSymlink(id, newName); err != nil { + if os.IsExist(err) { + return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} + } + + return "", err + } + + return newName, nil +} + +func (idx *Unique) Lookup(v string) ([]string, error) { + searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, v)) + oldname, err := idx.resolveSymlink(searchPath) + if err != nil { + if os.IsNotExist(err) { + err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + return nil, err + } + + return []string{oldname}, nil +} + +// 97d28b57 +func (idx *Unique) Remove(id string, v string) error { + searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, v)) + _, err := idx.resolveSymlink(searchPath) + if err != nil { + if os.IsNotExist(err) { + err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + return err + } + + ctx := context.Background() + t, err := idx.authenticate(ctx) + if err != nil { + return err + } + + deletePath := path.Join("/meta", idx.indexRootDir, v) + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + resp, err := idx.storageProvider.Delete(ctx, &provider.DeleteRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: deletePath}, + }, + }) + + if err != nil { + return err + } + + // TODO Handle other error codes? + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + return &idxerrs.NotFoundErr{} + } + + return err +} + +func (idx *Unique) Update(id, oldV, newV string) error { + if err := idx.Remove(id, oldV); err != nil { + return err + } + + if _, err := idx.Add(id, newV); err != nil { + return err + } + + return nil +} + +func (idx *Unique) Search(pattern string) ([]string, error) { + ctx := context.Background() + t, err := idx.authenticate(ctx) + if err != nil { + return nil, err + } + + ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t) + res, err := idx.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{ + Ref: &provider.Reference{ + Spec: &provider.Reference_Path{Path: path.Join("/meta", idx.indexRootDir)}, + }, + }) + + searchPath := singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir)) + + matches := []string{} + for _, i := range res.GetInfos() { + if found, err := filepath.Match(pattern, path.Base(i.Path)); found { + if err != nil { + return nil, err + } + + oldPath, err := idx.resolveSymlink(singleJoiningSlash(searchPath, path.Base(i.Path))) + if err != nil { + return nil, err + } + matches = append(matches, oldPath) + } + } + + return matches, nil + +} + +func (idx *Unique) IndexBy() string { + return idx.indexBy +} + +func (idx *Unique) TypeName() string { + return idx.typeName +} + +func (idx *Unique) FilesDir() string { + return idx.filesDir +} + +func (idx *Unique) createSymlink(oldname, newname string) error { + t, err := idx.authenticate(context.TODO()) + if err != nil { + return err + } + + if _, err := idx.resolveSymlink(newname); err == nil { + return os.ErrExist + } + + _, err = idx.dataProvider.put(newname, strings.NewReader(oldname), t) + if err != nil { + return err + } + + return nil + +} + +func (idx *Unique) resolveSymlink(name string) (string, error) { + t, err := idx.authenticate(context.TODO()) + if err != nil { + return "", err + } + + resp, err := idx.dataProvider.get(name, t) + if err != nil { + return "", err + } + + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusNotFound { + return "", os.ErrNotExist + } + + return "", fmt.Errorf("could not resolve symlink %s, got status %v", name, resp.StatusCode) + } + + b, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + + } + return string(b), err +} + +func (idx *Unique) makeDirIfNotExists(ctx context.Context, folder string) error { + var rootPathRef = &provider.Reference{ + Spec: &provider.Reference_Path{Path: fmt.Sprintf("/meta/%v", folder)}, + } + + resp, err := idx.storageProvider.Stat(ctx, &provider.StatRequest{ + Ref: rootPathRef, + }) + + if err != nil { + return err + } + + if resp.Status.Code == v1beta11.Code_CODE_NOT_FOUND { + _, err := idx.storageProvider.CreateContainer(ctx, &provider.CreateContainerRequest{ + Ref: rootPathRef, + }) + + if err != nil { + return err + } + } + + return nil +} + +func (idx *Unique) indexURL(id string) string { + return singleJoiningSlash(idx.cs3conf.DataURL, path.Join(idx.cs3conf.DataPrefix, idx.indexRootDir, id)) +} + +// TODO: this is copied from proxy. Find a better solution or move it to ocis-pkg +func singleJoiningSlash(a, b string) string { + aslash := strings.HasSuffix(a, "/") + bslash := strings.HasPrefix(b, "/") + switch { + case aslash && bslash: + return a + b[1:] + case !aslash && !bslash: + return a + "/" + b + } + return a + b +} + +func (idx *Unique) authenticate(ctx context.Context) (token string, err error) { + u := &user.User{ + Id: &user.UserId{}, + Groups: []string{}, + } + if idx.cs3conf.ServiceUserName != "" { + u.Id.OpaqueId = idx.cs3conf.ServiceUserUUID + } + return idx.tokenManager.MintToken(ctx, u) +} + +type dataProviderClient struct { + client http.Client +} + +func (d dataProviderClient) put(url string, body io.Reader, token string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodPut, url, body) + if err != nil { + return nil, err + } + + req.Header.Add("x-access-token", token) + return d.client.Do(req) +} + +func (d dataProviderClient) get(url string, token string) (*http.Response, error) { + req, err := http.NewRequest(http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + req.Header.Add("x-access-token", token) + return d.client.Do(req) +} diff --git a/accounts/pkg/indexer/indexer/index/cs3/unique_test.go b/accounts/pkg/indexer/indexer/index/cs3/unique_test.go new file mode 100644 index 000000000..96b57ba7d --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/cs3/unique_test.go @@ -0,0 +1,71 @@ +package cs3 + +import ( + . "github.com/owncloud/ocis/accounts/pkg/indexer/test" + "github.com/stretchr/testify/assert" + "os" + "testing" +) + +func TestCS3UniqueIndex_FakeSymlink(t *testing.T) { + dataDir := WriteIndexTestDataCS3(t, TestData, "Id") + sut := NewUniqueIndex("User", "UserName", "/meta", "index.cs3", &Config{ + ProviderAddr: "0.0.0.0:9215", + DataURL: "http://localhost:9216", + DataPrefix: "data", + JWTSecret: "Pive-Fumkiu4", + ServiceUserName: "", + ServiceUserUUID: "", + }) + + err := sut.Init() + assert.NoError(t, err) + + res, err := sut.Add("abcdefg-123", "mikey") + assert.NoError(t, err) + t.Log(res) + + resLookup, err := sut.Lookup("mikey") + assert.NoError(t, err) + t.Log(resLookup) + + err = sut.Update("abcdefg-123", "mikey", "mickeyX") + assert.NoError(t, err) + + _, err = sut.Search("mi*") + assert.NoError(t, err) + + err = sut.Remove("", "mikey") + assert.NoError(t, err) + + _ = os.RemoveAll(dataDir) + +} + +func TestCS3UniqueIndexSearch(t *testing.T) { + dataDir := WriteIndexTestDataCS3(t, TestData, "Id") + sut := NewUniqueIndex("User", "UserName", "/meta", "index.cs3", &Config{ + ProviderAddr: "0.0.0.0:9215", + DataURL: "http://localhost:9216", + DataPrefix: "data", + JWTSecret: "Pive-Fumkiu4", + ServiceUserName: "", + ServiceUserUUID: "", + }) + + err := sut.Init() + assert.NoError(t, err) + + _, err = sut.Add("hijklmn-456", "mikey") + assert.NoError(t, err) + + _, err = sut.Add("ewf4ofk-555", "jacky") + assert.NoError(t, err) + + res, err := sut.Search("*y") + assert.NoError(t, err) + t.Log(res) + + _ = os.RemoveAll(dataDir) + +} diff --git a/accounts/pkg/indexer/indexer/index/disk/non_unique.go b/accounts/pkg/indexer/indexer/index/disk/non_unique.go new file mode 100644 index 000000000..d3194043c --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/disk/non_unique.go @@ -0,0 +1,171 @@ +package disk + +import ( + "errors" + "fmt" + idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "io/ioutil" + "os" + "path" + "path/filepath" +) + +// NonUniqueIndex is able to index an document by a key which might contain non-unique values +// +// /var/tmp/testfiles-395764020/index.disk/PetByColor/ +// ├── Brown +// │ └── rebef-123 -> /var/tmp/testfiles-395764020/pets/rebef-123 +// ├── Green +// │ ├── goefe-789 -> /var/tmp/testfiles-395764020/pets/goefe-789 +// │ └── xadaf-189 -> /var/tmp/testfiles-395764020/pets/xadaf-189 +// └── White +// └── wefwe-456 -> /var/tmp/testfiles-395764020/pets/wefwe-456 +type NonUniqueIndex struct { + indexBy string + typeName string + filesDir string + indexBaseDir string + indexRootDir string +} + +// NewNonUniqueIndex instantiates a new NonUniqueIndex instance. Init() should be +// called afterward to ensure correct on-disk structure. +func NewNonUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) NonUniqueIndex { + return NonUniqueIndex{ + indexBy: indexBy, + typeName: typeName, + filesDir: filesDir, + indexBaseDir: indexBaseDir, + indexRootDir: path.Join(indexBaseDir, fmt.Sprintf("%sBy%s", typeName, indexBy)), + } +} + +func (idx NonUniqueIndex) Init() error { + if _, err := os.Stat(idx.filesDir); err != nil { + return err + } + + if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { + return err + } + + return nil +} + +func (idx NonUniqueIndex) Lookup(v string) ([]string, error) { + searchPath := path.Join(idx.indexRootDir, v) + fi, err := ioutil.ReadDir(searchPath) + if os.IsNotExist(err) { + return []string{}, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + if err != nil { + return []string{}, err + } + + var ids []string = nil + for _, f := range fi { + ids = append(ids, f.Name()) + } + + if len(ids) == 0 { + return []string{}, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + return ids, nil +} + +func (idx NonUniqueIndex) Add(id, v string) (string, error) { + oldName := path.Join(idx.filesDir, id) + newName := path.Join(idx.indexRootDir, v, id) + + if err := os.MkdirAll(path.Join(idx.indexRootDir, v), 0777); err != nil { + return "", err + } + + err := os.Symlink(oldName, newName) + if errors.Is(err, os.ErrExist) { + return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} + } + + return newName, err + +} + +func (idx NonUniqueIndex) Remove(id string, v string) error { + res, err := filepath.Glob(path.Join(idx.indexRootDir, "/*/", id)) + if err != nil { + return err + } + + for _, p := range res { + if err := os.Remove(p); err != nil { + return err + } + } + + return nil +} + +func (idx NonUniqueIndex) Update(id, oldV, newV string) (err error) { + oldDir := path.Join(idx.indexRootDir, oldV) + oldPath := path.Join(oldDir, id) + newDir := path.Join(idx.indexRootDir, newV) + newPath := path.Join(newDir, id) + + if _, err = os.Stat(oldPath); os.IsNotExist(err) { + return &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, oldV} + } + + if err != nil { + return + } + + if err = os.MkdirAll(newDir, 0777); err != nil { + return + } + + if err = os.Rename(oldPath, newPath); err != nil { + return + } + + di, err := ioutil.ReadDir(oldDir) + if err != nil { + return err + } + + if len(di) == 0 { + err = os.RemoveAll(oldDir) + if err != nil { + return + } + } + + return + +} + +func (idx NonUniqueIndex) Search(pattern string) ([]string, error) { + paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern, "*")) + if err != nil { + return nil, err + } + + if len(paths) == 0 { + return nil, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, pattern} + } + + return paths, nil +} + +func (idx NonUniqueIndex) IndexBy() string { + return idx.indexBy +} + +func (idx NonUniqueIndex) TypeName() string { + return idx.typeName +} + +func (idx NonUniqueIndex) FilesDir() string { + return idx.filesDir +} diff --git a/accounts/pkg/indexer/indexer/index/disk/non_unique_test.go b/accounts/pkg/indexer/indexer/index/disk/non_unique_test.go new file mode 100644 index 000000000..4eb5c42e0 --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/disk/non_unique_test.go @@ -0,0 +1,111 @@ +package disk + +import ( + "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + . "github.com/owncloud/ocis/accounts/pkg/indexer/test" + "github.com/stretchr/testify/assert" + "os" + "path" + "testing" +) + +func TestNonUniqueIndexAdd(t *testing.T) { + sut, dataPath := getNonUniqueIdxSut(t) + + ids, err := sut.Lookup("Green") + assert.NoError(t, err) + assert.EqualValues(t, []string{"goefe-789", "xadaf-189"}, ids) + + ids, err = sut.Lookup("White") + assert.NoError(t, err) + assert.EqualValues(t, []string{"wefwe-456"}, ids) + + ids, err = sut.Lookup("Cyan") + assert.Error(t, err) + assert.EqualValues(t, []string{}, ids) + + _ = os.RemoveAll(dataPath) + +} + +func TestNonUniqueIndexUpdate(t *testing.T) { + sut, dataPath := getNonUniqueIdxSut(t) + + err := sut.Update("goefe-789", "", "Black") + assert.NoError(t, err) + + err = sut.Update("xadaf-189", "", "Black") + assert.NoError(t, err) + + assert.DirExists(t, path.Join(dataPath, "index.disk/PetByColor/Black")) + assert.NoDirExists(t, path.Join(dataPath, "index.disk/PetByColor/Green")) + + _ = os.RemoveAll(dataPath) +} + +func TestNonUniqueIndexDelete(t *testing.T) { + sut, dataPath := getNonUniqueIdxSut(t) + assert.FileExists(t, path.Join(dataPath, "index.disk/PetByColor/Green/goefe-789")) + err := sut.Remove("goefe-789", "") + assert.NoError(t, err) + assert.NoFileExists(t, path.Join(dataPath, "index.disk/PetByColor/Green/goefe-789")) + _ = os.RemoveAll(dataPath) +} + +func TestNonUniqueIndexInit(t *testing.T) { + dataDir := CreateTmpDir(t) + indexRootDir := path.Join(dataDir, "index.disk") + filesDir := path.Join(dataDir, "users") + + uniq := NewNonUniqueIndex("User", "DisplayName", filesDir, indexRootDir) + assert.Error(t, uniq.Init(), "Init should return an error about missing files-dir") + + if err := os.Mkdir(filesDir, 0777); err != nil { + t.Fatalf("Could not create test data-dir %s", err) + } + + assert.NoError(t, uniq.Init(), "Init shouldn't return an error") + assert.DirExists(t, indexRootDir) + assert.DirExists(t, path.Join(indexRootDir, "UserByDisplayName")) + + _ = os.RemoveAll(dataDir) +} + +func TestNonUniqueIndexSearch(t *testing.T) { + sut, dataPath := getNonUniqueIdxSut(t) + + res, err := sut.Search("Gr*") + + assert.NoError(t, err) + assert.Len(t, res, 2) + + assert.Equal(t, "goefe-789", path.Base(res[0])) + assert.Equal(t, "xadaf-189", path.Base(res[1])) + + res, err = sut.Search("does-not-exist@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.NotFoundErr{}, err) + + _ = os.RemoveAll(dataPath) +} + +func getNonUniqueIdxSut(t *testing.T) (sut index.Index, dataPath string) { + dataPath = WriteIndexTestData(t, TestData, "Id") + sut = NewNonUniqueIndex("Pet", "Color", path.Join(dataPath, "pets"), path.Join(dataPath, "index.disk")) + err := sut.Init() + if err != nil { + t.Fatal(err) + } + + for _, u := range TestData["pets"] { + pkVal := ValueOf(u, "Id") + idxByVal := ValueOf(u, "Color") + _, err := sut.Add(pkVal, idxByVal) + if err != nil { + t.Fatal(err) + } + } + + return +} diff --git a/accounts/pkg/indexer/indexer/index/disk/unique.go b/accounts/pkg/indexer/indexer/index/disk/unique.go new file mode 100644 index 000000000..7cb78766f --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/disk/unique.go @@ -0,0 +1,180 @@ +package disk + +import ( + "errors" + "fmt" + idxerrs "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "os" + "path" + "path/filepath" + "strings" +) + +// Unique ensures that only one document of the same type and key-value combination can exist in the index. +// +// Modeled by creating a indexer-folder per entity and key with symlinks which point to respective documents which contain +// the link-filename as value. +// +// Directory Layout +// +// /var/data/index.disk/UniqueUserByEmail/ +// ├── jacky@example.com -> /var/data/users/ewf4ofk-555 +// ├── jones@example.com -> /var/data/users/rulan54-777 +// └── mikey@example.com -> /var/data/users/abcdefg-123 +// +// Example user +// +// { +// "Id": "ewf4ofk-555", +// "UserName": "jacky", +// "Email": "jacky@example.com" +// } +// +type Unique struct { + indexBy string + typeName string + filesDir string + indexBaseDir string + indexRootDir string +} + +// NewUniqueIndex instantiates a new UniqueIndex instance. Init() should be +// called afterward to ensure correct on-disk structure. +func NewUniqueIndex(typeName, indexBy, filesDir, indexBaseDir string) Unique { + return Unique{ + indexBy: indexBy, + typeName: typeName, + filesDir: filesDir, + indexBaseDir: indexBaseDir, + indexRootDir: path.Join(indexBaseDir, strings.Join([]string{"unique", typeName, indexBy}, ".")), + } +} + +func (idx Unique) Init() error { + if _, err := os.Stat(idx.filesDir); err != nil { + return err + } + + if err := os.MkdirAll(idx.indexRootDir, 0777); err != nil { + return err + } + + return nil +} + +func (idx Unique) Add(id, v string) (string, error) { + oldName := path.Join(idx.filesDir, id) + newName := path.Join(idx.indexRootDir, v) + err := os.Symlink(oldName, newName) + if errors.Is(err, os.ErrExist) { + return "", &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, v} + } + + return newName, err +} + +func (idx Unique) Remove(id string, v string) (err error) { + searchPath := path.Join(idx.indexRootDir, v) + if err = isValidSymlink(searchPath); err != nil { + return + } + + return os.Remove(searchPath) +} + +// unique.github.com.owncloud.ocis.accounts.pkg.indexer.User.UserName +// unique.github.com.owncloud.ocis.accounts.pkg.indexer.User.UserName/UserName +func (idx Unique) Lookup(v string) (resultPath []string, err error) { + searchPath := path.Join(idx.indexRootDir, v) + if err = isValidSymlink(searchPath); err != nil { + if os.IsNotExist(err) { + err = &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, v} + } + + return + } + + p, err := os.Readlink(searchPath) + if err != nil { + return []string{}, nil + } + + return []string{p}, err + +} + +func (idx Unique) Update(id, oldV, newV string) (err error) { + oldPath := path.Join(idx.indexRootDir, oldV) + if err = isValidSymlink(oldPath); err != nil { + if os.IsNotExist(err) { + return &idxerrs.NotFoundErr{idx.TypeName(), idx.IndexBy(), oldV} + } + + return + } + + newPath := path.Join(idx.indexRootDir, newV) + if err = isValidSymlink(newPath); err == nil { + return &idxerrs.AlreadyExistsErr{idx.typeName, idx.indexBy, newV} + } + + if os.IsNotExist(err) { + err = os.Rename(oldPath, newPath) + } + + return +} + +func (idx Unique) Search(pattern string) ([]string, error) { + paths, err := filepath.Glob(path.Join(idx.indexRootDir, pattern)) + if err != nil { + return nil, err + } + + if len(paths) == 0 { + return nil, &idxerrs.NotFoundErr{idx.typeName, idx.indexBy, pattern} + } + + res := make([]string, 0, 0) + for _, p := range paths { + if err := isValidSymlink(p); err != nil { + return nil, err + } + + src, err := os.Readlink(p) + if err != nil { + return nil, err + } + + res = append(res, src) + } + + return res, nil +} + +func (idx Unique) IndexBy() string { + return idx.indexBy +} + +func (idx Unique) TypeName() string { + return idx.typeName +} + +func (idx Unique) FilesDir() string { + return idx.filesDir +} + +func isValidSymlink(path string) (err error) { + var symInfo os.FileInfo + if symInfo, err = os.Lstat(path); err != nil { + return + } + + if symInfo.Mode()&os.ModeSymlink == 0 { + err = fmt.Errorf("%s is not a valid symlink (bug/corruption?)", path) + return + } + + return + +} diff --git a/accounts/pkg/indexer/indexer/index/disk/unique_test.go b/accounts/pkg/indexer/indexer/index/disk/unique_test.go new file mode 100644 index 000000000..ee47d4597 --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/disk/unique_test.go @@ -0,0 +1,136 @@ +package disk + +import ( + "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "github.com/owncloud/ocis/accounts/pkg/indexer/index" + . "github.com/owncloud/ocis/accounts/pkg/indexer/test" + "github.com/stretchr/testify/assert" + "os" + "path" + "testing" +) + +func TestUniqueLookupSingleEntry(t *testing.T) { + uniq, dataDir := getUniqueIdxSut(t) + filesDir := path.Join(dataDir, "users") + + t.Log("existing lookup") + resultPath, err := uniq.Lookup("mikey@example.com") + assert.NoError(t, err) + + assert.Equal(t, []string{path.Join(filesDir, "abcdefg-123")}, resultPath) + + t.Log("non-existing lookup") + resultPath, err = uniq.Lookup("doesnotExists@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.NotFoundErr{}, err) + assert.Empty(t, resultPath) + + _ = os.RemoveAll(dataDir) + +} + +func TestUniqueUniqueConstraint(t *testing.T) { + uniq, dataDir := getUniqueIdxSut(t) + + _, err := uniq.Add("abcdefg-123", "mikey@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.AlreadyExistsErr{}, err) + + _ = os.RemoveAll(dataDir) +} + +func TestUniqueRemove(t *testing.T) { + uniq, dataDir := getUniqueIdxSut(t) + + err := uniq.Remove("", "mikey@example.com") + assert.NoError(t, err) + + _, err = uniq.Lookup("mikey@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.NotFoundErr{}, err) + + _ = os.RemoveAll(dataDir) +} + +func TestUniqueUpdate(t *testing.T) { + uniq, dataDir := getUniqueIdxSut(t) + + t.Log("successful update") + err := uniq.Update("", "", "mikey2@example.com") + assert.NoError(t, err) + + t.Log("failed update because already exists") + err = uniq.Update("", "", "mikey2@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.AlreadyExistsErr{}, err) + + t.Log("failed update because not found") + err = uniq.Update("", "", "something2@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.NotFoundErr{}, err) + + _ = os.RemoveAll(dataDir) +} + +func TestUniqueInit(t *testing.T) { + dataDir := CreateTmpDir(t) + indexRootDir := path.Join(dataDir, "index.disk") + filesDir := path.Join(dataDir, "users") + + uniq := NewUniqueIndex("User", "Email", filesDir, indexRootDir) + assert.Error(t, uniq.Init(), "Init should return an error about missing files-dir") + + if err := os.Mkdir(filesDir, 0777); err != nil { + t.Fatalf("Could not create test data-dir %s", err) + } + + assert.NoError(t, uniq.Init(), "Init shouldn't return an error") + assert.DirExists(t, indexRootDir) + assert.DirExists(t, path.Join(indexRootDir, "UniqueUserByEmail")) + + _ = os.RemoveAll(dataDir) +} + +func TestUniqueIndexSearch(t *testing.T) { + sut, dataPath := getUniqueIdxSut(t) + + res, err := sut.Search("j*@example.com") + + assert.NoError(t, err) + assert.Len(t, res, 2) + + assert.Equal(t, "ewf4ofk-555", path.Base(res[0])) + assert.Equal(t, "rulan54-777", path.Base(res[1])) + + res, err = sut.Search("does-not-exist@example.com") + assert.Error(t, err) + assert.IsType(t, &errors.NotFoundErr{}, err) + + _ = os.RemoveAll(dataPath) +} + +func TestErrors(t *testing.T) { + assert.True(t, errors.IsAlreadyExistsErr(&errors.AlreadyExistsErr{})) + assert.True(t, errors.IsNotFoundErr(&errors.NotFoundErr{})) +} + +func getUniqueIdxSut(t *testing.T) (sut index.Index, dataPath string) { + dataPath = WriteIndexTestData(t, TestData, "Id") + sut = NewUniqueIndex("User", "Email", path.Join(dataPath, "users"), path.Join(dataPath, "indexer.disk")) + err := sut.Init() + if err != nil { + t.Fatal(err) + } + + for _, u := range TestData["users"] { + pkVal := ValueOf(u, "Id") + idxByVal := ValueOf(u, "Email") + _, err := sut.Add(pkVal, idxByVal) + if err != nil { + t.Fatal(err) + } + } + + return +} diff --git a/accounts/pkg/indexer/indexer/index/index.go b/accounts/pkg/indexer/indexer/index/index.go new file mode 100644 index 000000000..151a891a3 --- /dev/null +++ b/accounts/pkg/indexer/indexer/index/index.go @@ -0,0 +1,15 @@ +package index + +// Index can be implemented to create new indexer-strategies. See Unique for example. +// Each indexer implementation is bound to one data-column (IndexBy) and a data-type (TypeName) +type Index interface { + Init() error + Lookup(v string) ([]string, error) + Add(id, v string) (string, error) + Remove(id string, v string) error + Update(id, oldV, newV string) error + Search(pattern string) ([]string, error) + IndexBy() string + TypeName() string + FilesDir() string +} diff --git a/accounts/pkg/indexer/indexer/indexer.go b/accounts/pkg/indexer/indexer/indexer.go new file mode 100644 index 000000000..76189a8db --- /dev/null +++ b/accounts/pkg/indexer/indexer/indexer.go @@ -0,0 +1,158 @@ +// Package indexer provides symlink-based indexer for on-disk document-directories. +package indexer + +import ( + "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + "github.com/owncloud/ocis/accounts/pkg/indexer/index/disk" + "github.com/rs/zerolog" + "path" +) + +// Indexer is a facade to configure and query over multiple indices. +type Indexer struct { + config *Config + indices typeMap +} + +type Config struct { + DataDir string + IndexRootDirName string + Log zerolog.Logger +} + +func NewIndex(cfg *Config) *Indexer { + return &Indexer{ + config: cfg, + indices: typeMap{}, + } +} + +func (i Indexer) AddUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error { + typeName := getTypeFQN(t) + fullDataPath := path.Join(i.config.DataDir, entityDirName) + indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName) + + idx := disk.NewUniqueIndex(typeName, indexBy, fullDataPath, indexPath) + + i.indices.addIndex(typeName, pkName, idx) + return idx.Init() +} + +func (i Indexer) AddNonUniqueIndex(t interface{}, indexBy, pkName, entityDirName string) error { + typeName := getTypeFQN(t) + fullDataPath := path.Join(i.config.DataDir, entityDirName) + indexPath := path.Join(i.config.DataDir, i.config.IndexRootDirName) + + idx := disk.NewNonUniqueIndex(typeName, indexBy, fullDataPath, indexPath) + + i.indices.addIndex(typeName, pkName, idx) + return idx.Init() +} + +// Add a new entry to the indexer +func (i Indexer) Add(t interface{}) error { + typeName := getTypeFQN(t) + if fields, ok := i.indices[typeName]; ok { + for _, indices := range fields.IndicesByField { + for _, idx := range indices { + pkVal := valueOf(t, fields.PKFieldName) + idxByVal := valueOf(t, idx.IndexBy()) + if _, err := idx.Add(pkVal, idxByVal); err != nil { + return err + } + } + } + } + + return nil +} + +func (i Indexer) FindBy(t interface{}, field string, val string) ([]string, error) { + typeName := getTypeFQN(t) + resultPaths := make([]string, 0) + if fields, ok := i.indices[typeName]; ok { + for _, idx := range fields.IndicesByField[field] { + res, err := idx.Lookup(val) + if err != nil { + if errors.IsNotFoundErr(err) { + continue + } + + if err != nil { + return nil, err + } + } + + resultPaths = append(resultPaths, res...) + + } + } + + result := make([]string, 0, len(resultPaths)) + for _, v := range resultPaths { + result = append(result, path.Base(v)) + } + + return result, nil +} + +func (i Indexer) Delete(t interface{}) error { + typeName := getTypeFQN(t) + if fields, ok := i.indices[typeName]; ok { + for _, indices := range fields.IndicesByField { + for _, idx := range indices { + pkVal := valueOf(t, fields.PKFieldName) + idxByVal := valueOf(t, idx.IndexBy()) + if err := idx.Remove(pkVal, idxByVal); err != nil { + return err + } + } + } + } + + return nil +} + +func (i Indexer) FindByPartial(t interface{}, field string, pattern string) ([]string, error) { + typeName := getTypeFQN(t) + resultPaths := make([]string, 0) + if fields, ok := i.indices[typeName]; ok { + for _, idx := range fields.IndicesByField[field] { + res, err := idx.Search(pattern) + if err != nil { + if errors.IsNotFoundErr(err) { + continue + } + + if err != nil { + return nil, err + } + } + + resultPaths = append(resultPaths, res...) + + } + } + + result := make([]string, 0, len(resultPaths)) + for _, v := range resultPaths { + result = append(result, path.Base(v)) + } + + return result, nil + +} + +func (i Indexer) Update(t interface{}, field, oldVal, newVal string) error { + typeName := getTypeFQN(t) + if fields, ok := i.indices[typeName]; ok { + for _, idx := range fields.IndicesByField[field] { + pkVal := valueOf(t, fields.PKFieldName) + if err := idx.Update(pkVal, oldVal, newVal); err != nil { + return err + } + } + } + + return nil +} diff --git a/accounts/pkg/indexer/indexer/indexer_test.go b/accounts/pkg/indexer/indexer/indexer_test.go new file mode 100644 index 000000000..618ff6e0a --- /dev/null +++ b/accounts/pkg/indexer/indexer/indexer_test.go @@ -0,0 +1,255 @@ +package indexer + +import ( + "github.com/owncloud/ocis/accounts/pkg/indexer/errors" + . "github.com/owncloud/ocis/accounts/pkg/indexer/test" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "testing" +) + +func TestIndexer_AddWithUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") + + u := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} + err := indexer.Add(u) + assert.NoError(t, err) + +} + +func TestIndexer_FindByWithUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") + + u := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} + err := indexer.Add(u) + assert.NoError(t, err) + + res, err := indexer.FindBy(User{}, "UserName", "mikey") + assert.NoError(t, err) + t.Log(res) +} + +func TestIndexer_AddWithNonUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddNonUniqueIndex(&TestPet{}, "Kind", "Id", "pets") + + pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} + pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} + + err := indexer.Add(pet1) + assert.NoError(t, err) + + err = indexer.Add(pet2) + assert.NoError(t, err) + + res, err := indexer.FindBy(TestPet{}, "Kind", "Hog") + assert.NoError(t, err) + + t.Log(res) +} + +func TestIndexer_DeleteWithNonUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddNonUniqueIndex(&TestPet{}, "Kind", "Id", "pets") + + pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} + pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} + + err := indexer.Add(pet1) + assert.NoError(t, err) + + err = indexer.Add(pet2) + assert.NoError(t, err) + + err = indexer.Delete(pet2) + assert.NoError(t, err) +} + +func TestIndexer_SearchWithNonUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddNonUniqueIndex(&TestPet{}, "Name", "Id", "pets") + + pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} + pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} + + err := indexer.Add(pet1) + assert.NoError(t, err) + + err = indexer.Add(pet2) + assert.NoError(t, err) + + res, err := indexer.FindByPartial(pet2, "Name", "*ky") + assert.NoError(t, err) + + t.Log(res) +} + +func TestIndexer_UpdateWithUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddUniqueIndex(&User{}, "UserName", "Id", "users") + + user1 := &User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"} + user2 := &User{Id: "hijklmn-456", UserName: "frank", Email: "frank@example.com"} + + err := indexer.Add(user1) + assert.NoError(t, err) + + err = indexer.Add(user2) + assert.NoError(t, err) + + // Update to non existing value + err = indexer.Update(user2, "UserName", "frank", "jane") + assert.NoError(t, err) + + // Update to non existing value + err = indexer.Update(user2, "UserName", "mikey", "jane") + assert.Error(t, err) + assert.IsType(t, &errors.AlreadyExistsErr{}, err) +} + +func TestIndexer_UpdateWithNonUniqueIndex(t *testing.T) { + dataDir := WriteIndexTestData(t, TestData, "Id") + indexer := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + indexer.AddNonUniqueIndex(&TestPet{}, "Name", "Id", "pets") + + pet1 := TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"} + pet2 := TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"} + + err := indexer.Add(pet1) + assert.NoError(t, err) + + err = indexer.Add(pet2) + assert.NoError(t, err) + + err = indexer.Update(pet2, "Name", "Ricky", "Jonny") + assert.NoError(t, err) +} + +/* +func TestManagerQueryMultipleIndices(t *testing.T) { + dataDir := writeIndexTestData(t, testData, "Id") + man := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + err := man.AddUniqueIndex("User", "Email", "users") + assert.NoError(t, err) + + err = man.AddUniqueIndex("User", "UserName", "users") + assert.NoError(t, err) + + err = man.AddNormalIndex("TestPet", "Color", "pets") + assert.NoError(t, err) + + err = man.AddUniqueIndex("TestPet", "Name", "pets") + assert.NoError(t, err) + + for path := range testData { + for _, entity := range testData[path] { + err := man.Add(valueOf(entity, "Id"), entity) + assert.NoError(t, err) + } + } + + type test struct { + typeName, key, value, wantRes string + wantErr error + } + + tests := []test{ + {typeName: "User", key: "Email", value: "jacky@example.com", wantRes: "ewf4ofk-555"}, + {typeName: "User", key: "UserName", value: "jacky", wantRes: "ewf4ofk-555"}, + {typeName: "TestPet", key: "Color", value: "Brown", wantRes: "rebef-123"}, + {typeName: "TestPet", key: "Color", value: "Cyan", wantRes: "", wantErr: ¬FoundErr{}}, + } + + for _, tc := range tests { + name := fmt.Sprintf("Query%sBy%s=%s", tc.typeName, tc.key, tc.value) + t.Run(name, func(t *testing.T) { + pk, err := man.Find(tc.typeName, tc.key, tc.value) + assert.Equal(t, tc.wantRes, pk) + assert.IsType(t, tc.wantErr, err) + }) + } + + _ = os.RemoveAll(dataDir) +} + +*/ + +/* +func TestManagerDelete(t *testing.T) { + dataDir := writeIndexTestData(t, testData, "Id") + man := NewIndex(&Config{ + DataDir: dataDir, + IndexRootDirName: "index.disk", + Log: zerolog.Logger{}, + }) + + err := man.AddUniqueIndex("User", "Email", "users") + assert.NoError(t, err) + + err = man.AddUniqueIndex("User", "UserName", "users") + assert.NoError(t, err) + + err = man.AddUniqueIndex("TestPet", "Name", "pets") + assert.NoError(t, err) + + for path := range testData { + for _, entity := range testData[path] { + err := man.Add(valueOf(entity, "Id"), entity) + assert.NoError(t, err) + } + } + + err = man.Delete("User", "hijklmn-456") + _ = os.RemoveAll(dataDir) + +} + +*/ diff --git a/accounts/pkg/indexer/indexer/map.go b/accounts/pkg/indexer/indexer/map.go new file mode 100644 index 000000000..77e8e0777 --- /dev/null +++ b/accounts/pkg/indexer/indexer/map.go @@ -0,0 +1,27 @@ +package indexer + +import "github.com/owncloud/ocis/accounts/pkg/indexer/index" + +// typeMap stores the indexer layout at runtime. + +type typeMap map[tName]typeMapping +type tName = string +type fieldName = string + +type typeMapping struct { + PKFieldName string + IndicesByField map[fieldName][]index.Index +} + +func (m typeMap) addIndex(typeName string, pkName string, idx index.Index) { + if val, ok := m[typeName]; ok { + val.IndicesByField[idx.IndexBy()] = append(val.IndicesByField[idx.IndexBy()], idx) + return + } + m[typeName] = typeMapping{ + PKFieldName: pkName, + IndicesByField: map[string][]index.Index{ + idx.IndexBy(): {idx}, + }, + } +} diff --git a/accounts/pkg/indexer/indexer/reflect.go b/accounts/pkg/indexer/indexer/reflect.go new file mode 100644 index 000000000..90036b3e1 --- /dev/null +++ b/accounts/pkg/indexer/indexer/reflect.go @@ -0,0 +1,34 @@ +package indexer + +import ( + "errors" + "path" + "reflect" + "strings" +) + +func getType(v interface{}) (reflect.Value, error) { + rv := reflect.ValueOf(v) + for rv.Kind() == reflect.Ptr || rv.Kind() == reflect.Interface { + rv = rv.Elem() + } + if !rv.IsValid() { + return reflect.Value{}, errors.New("failed to read value via reflection") + } + + return rv, nil +} + +func getTypeFQN(t interface{}) string { + typ, _ := getType(t) + typeName := path.Join(typ.Type().PkgPath(), typ.Type().Name()) + typeName = strings.ReplaceAll(typeName, "/", ".") + return typeName +} + +func valueOf(v interface{}, field string) string { + r := reflect.ValueOf(v) + f := reflect.Indirect(r).FieldByName(field) + + return f.String() +} diff --git a/accounts/pkg/indexer/indexer/reflect_test.go b/accounts/pkg/indexer/indexer/reflect_test.go new file mode 100644 index 000000000..0da505d2e --- /dev/null +++ b/accounts/pkg/indexer/indexer/reflect_test.go @@ -0,0 +1,53 @@ +package indexer + +import ( + "testing" +) + +func Test_getTypeFQN(t *testing.T) { + type someT struct{} + + type args struct { + t interface{} + } + tests := []struct { + name string + args args + want string + }{ + {name: "ByValue", args: args{&someT{}}, want: "github.com.owncloud.ocis.accounts.pkg.indexer.someT"}, + {name: "ByRef", args: args{someT{}}, want: "github.com.owncloud.ocis.accounts.pkg.indexer.someT"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getTypeFQN(tt.args.t); got != tt.want { + t.Errorf("getTypeFQN() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_valueOf(t *testing.T) { + type someT struct { + val string + } + type args struct { + v interface{} + field string + } + tests := []struct { + name string + args args + want string + }{ + {name: "ByValue", args: args{v: someT{val: "hello"}, field: "val"}, want: "hello"}, + {name: "ByRef", args: args{v: &someT{val: "hello"}, field: "val"}, want: "hello"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := valueOf(tt.args.v, tt.args.field); got != tt.want { + t.Errorf("valueOf() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/accounts/pkg/indexer/indexer/test/data.go b/accounts/pkg/indexer/indexer/test/data.go new file mode 100644 index 000000000..2b14d62ff --- /dev/null +++ b/accounts/pkg/indexer/indexer/test/data.go @@ -0,0 +1,56 @@ +package test + +import ( + "encoding/json" + "io/ioutil" + "os" + "path" + "testing" +) + +type User struct { + Id, UserName, Email string +} + +type TestPet struct { + Id, Kind, Color, Name string +} + +var TestData = map[string][]interface{}{ + "users": { + User{Id: "abcdefg-123", UserName: "mikey", Email: "mikey@example.com"}, + User{Id: "hijklmn-456", UserName: "frank", Email: "frank@example.com"}, + User{Id: "ewf4ofk-555", UserName: "jacky", Email: "jacky@example.com"}, + User{Id: "rulan54-777", UserName: "jones", Email: "jones@example.com"}, + }, + "pets": { + TestPet{Id: "rebef-123", Kind: "Dog", Color: "Brown", Name: "Waldo"}, + TestPet{Id: "wefwe-456", Kind: "Cat", Color: "White", Name: "Snowy"}, + TestPet{Id: "goefe-789", Kind: "Hog", Color: "Green", Name: "Dicky"}, + TestPet{Id: "xadaf-189", Kind: "Hog", Color: "Green", Name: "Ricky"}, + }, +} + +func WriteIndexTestData(t *testing.T, m map[string][]interface{}, pk string) string { + rootDir := CreateTmpDir(t) + for dirName := range m { + fileTypePath := path.Join(rootDir, dirName) + + if err := os.MkdirAll(fileTypePath, 0777); err != nil { + t.Fatal(err) + } + for _, u := range m[dirName] { + data, err := json.Marshal(u) + if err != nil { + t.Fatal(err) + } + + pkVal := ValueOf(u, pk) + if err := ioutil.WriteFile(path.Join(fileTypePath, pkVal), data, 0777); err != nil { + t.Fatal(err) + } + } + } + + return rootDir +} diff --git a/accounts/pkg/indexer/indexer/test/helpers.go b/accounts/pkg/indexer/indexer/test/helpers.go new file mode 100644 index 000000000..f6ccb9be4 --- /dev/null +++ b/accounts/pkg/indexer/indexer/test/helpers.go @@ -0,0 +1,23 @@ +package test + +import ( + "io/ioutil" + "reflect" + "testing" +) + +func CreateTmpDir(t *testing.T) string { + name, err := ioutil.TempDir("/var/tmp", "testfiles-*") + if err != nil { + t.Fatal(err) + } + + return name +} + +func ValueOf(v interface{}, field string) string { + r := reflect.ValueOf(v) + f := reflect.Indirect(r).FieldByName(field) + + return f.String() +}