Rebuild groups index

accounts/pkg/service/v0/index.go

@@ -21,20 +21,31 @@ func (s Service) RebuildIndex(ctx context.Context, request *proto.RebuildIndexRe
 		return err
 	}
 
-	resp := &storage.Accounts{
+	accounts := &storage.Accounts{
 		Accounts: make([]*proto.Account, 0),
 	}
-	if err := s.repo.LoadAccounts(ctx, resp); err != nil {
+	if err := s.repo.LoadAccounts(ctx, accounts); err != nil {
 		return err
 	}
-	for i := range resp.Accounts {
+	for i := range accounts.Accounts {
-		_, err := s.index.Add(resp.Accounts[i])
+		_, err := s.index.Add(accounts.Accounts[i])
 		if err != nil {
 			return err
 		}
 	}
 
-	// TODO: read all the documents and add them to the indexer.
+	groups := &storage.Groups{
+		Groups: make([]*proto.Group, 0),
+	}
+	if err := s.repo.LoadGroups(ctx, groups); err != nil {
+		return err
+	}
+	for i := range groups.Groups {
+		_, err := s.index.Add(groups.Groups[i])
+		if err != nil {
+			return err
+		}
+	}
 
 	return nil
 }

accounts/pkg/storage/cs3.go

@@ -95,27 +95,7 @@ func (r CS3Repo) LoadAccount(ctx context.Context, id string, a *proto.Account) (
 	return r.loadAccount(id, t, a)
 }
 
-func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error {
+// LoadAccounts loads all the accounts from the cs3 api
-	resp, err := r.dataProvider.get(r.accountURL(id), t)
-	if err != nil {
-		return err
-	}
-
-	if resp.StatusCode == http.StatusNotFound {
-		return &notFoundErr{"account", id}
-	}
-
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if err = resp.Body.Close(); err != nil {
-		return err
-	}
-	return json.Unmarshal(b, &a)
-}
-
-// LoadAccounts loads all the accounts from the cs3 api. If ids are given, the result set will be filtered.
 func (r CS3Repo) LoadAccounts(ctx context.Context, a *Accounts) (err error) {
 	t, err := r.authenticate(ctx)
 	if err != nil {
@@ -145,6 +125,26 @@ func (r CS3Repo) LoadAccounts(ctx context.Context, a *Accounts) (err error) {
 	return nil
 }
 
+func (r CS3Repo) loadAccount(id string, t string, a *proto.Account) error {
+	resp, err := r.dataProvider.get(r.accountURL(id), t)
+	if err != nil {
+		return err
+	}
+
+	if resp.StatusCode == http.StatusNotFound {
+		return &notFoundErr{"account", id}
+	}
+
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return err
+	}
+	if err = resp.Body.Close(); err != nil {
+		return err
+	}
+	return json.Unmarshal(b, &a)
+}
+
 // DeleteAccount deletes an account via cs3 by id
 func (r CS3Repo) DeleteAccount(ctx context.Context, id string) (err error) {
 	t, err := r.authenticate(ctx)
@@ -206,6 +206,40 @@ func (r CS3Repo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err
 		return err
 	}
 
+	return r.loadGroup(id, t, g)
+}
+
+// LoadGroups loads all the groups from the cs3 api
+func (r CS3Repo) LoadGroups(ctx context.Context, g *Groups) (err error) {
+	t, err := r.authenticate(ctx)
+	if err != nil {
+		return err
+	}
+
+	ctx = metadata.AppendToOutgoingContext(ctx, token.TokenHeader, t)
+	res, err := r.storageProvider.ListContainer(ctx, &provider.ListContainerRequest{
+		Ref: &provider.Reference{
+			Spec: &provider.Reference_Path{Path: path.Join("/meta", groupsFolder)},
+		},
+	})
+	if err != nil {
+		return err
+	}
+
+	log := olog.NewLogger(olog.Pretty(r.cfg.Log.Pretty), olog.Color(r.cfg.Log.Color), olog.Level(r.cfg.Log.Level))
+	for i := range res.Infos {
+		grp := &proto.Group{}
+		err := r.loadGroup(filepath.Base(res.Infos[i].Path), t, grp)
+		if err != nil {
+			log.Err(err).Msg("could not load account")
+			continue
+		}
+		g.Groups = append(g.Groups, grp)
+	}
+	return nil
+}
+
+func (r CS3Repo) loadGroup(id string, t string, g *proto.Group) error {
 	resp, err := r.dataProvider.get(r.groupURL(id), t)
 	if err != nil {
 		return err

accounts/pkg/storage/disk.go

@@ -70,7 +70,7 @@ func (r DiskRepo) LoadAccount(ctx context.Context, id string, a *proto.Account)
 	return json.Unmarshal(data, a)
 }
 
-// LoadAccounts loads all the accounts from the local filesystem. If ids are given, the result set will be filtered.
+// LoadAccounts loads all the accounts from the local filesystem
 func (r DiskRepo) LoadAccounts(ctx context.Context, a *Accounts) (err error) {
 	root := filepath.Join(r.cfg.Repo.Disk.Path, accountsFolder)
 	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
@@ -132,6 +132,20 @@ func (r DiskRepo) LoadGroup(ctx context.Context, id string, g *proto.Group) (err
 	return json.Unmarshal(data, g)
 }
 
+// LoadGroups loads all the groups from the local filesystem
+func (r DiskRepo) LoadGroups(ctx context.Context, g *Groups) (err error) {
+	root := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder)
+	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
+		grp := &proto.Group{}
+		if err = r.LoadGroup(ctx, filepath.Base(path), grp); err != nil {
+			r.log.Err(err).Msg("could not load group")
+			return nil
+		}
+		g.Groups = append(g.Groups, grp)
+		return nil
+	})
+}
+
 // DeleteGroup from the local filesystem
 func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) {
 	path := filepath.Join(r.cfg.Repo.Disk.Path, groupsFolder, id)
@@ -142,9 +156,6 @@ func (r DiskRepo) DeleteGroup(ctx context.Context, id string) (err error) {
 	}
 
 	return
-
-	//r.log.Error().Err(err).Str("id", id).Str("path", path).Msg("could not remove group")
-	//return merrors.InternalServerError(r.serviceID, "could not remove group: %v", err.Error())
 }
 
 // deflateMemberOf replaces the groups of A user with an instance that only contains the id

accounts/pkg/storage/repo.go

@@ -15,6 +15,10 @@ type Accounts struct {
 	Accounts []*proto.Account
 }
 
+type Groups struct {
+	Groups []*proto.Group
+}
+
 // Repo defines the storage operations
 type Repo interface {
 	WriteAccount(ctx context.Context, a *proto.Account) (err error)
@@ -23,5 +27,6 @@ type Repo interface {
 	DeleteAccount(ctx context.Context, id string) (err error)
 	WriteGroup(ctx context.Context, g *proto.Group) (err error)
 	LoadGroup(ctx context.Context, id string, g *proto.Group) (err error)
+	LoadGroups(ctx context.Context, g *Groups) (err error)
 	DeleteGroup(ctx context.Context, id string) (err error)
 }