Arjun Patel
136 lines
3.2 KiB
Go
136 lines
3.2 KiB
Go
// memberreconciler reconciles the Firestore membership mirror
|
|
// (humans/{humanId}.networks) against the authoritative Postgres
|
|
// network_members table. Safe to run on a cron — only humans whose mirrored
|
|
// set differs from Postgres are written, so a steady-state run is nearly free.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"slices"
|
|
"time"
|
|
|
|
"github.com/flowy-live/llink/internal/utils/flog"
|
|
|
|
"cloud.google.com/go/firestore"
|
|
"google.golang.org/api/iterator"
|
|
|
|
"github.com/flowy-live/llink/internal/db"
|
|
"github.com/flowy-live/llink/internal/human"
|
|
"github.com/flowy-live/llink/internal/network"
|
|
"github.com/flowy-live/llink/internal/utils"
|
|
)
|
|
|
|
func main() {
|
|
ctx := context.Background()
|
|
|
|
db.Init()
|
|
defer db.Cleanup()
|
|
|
|
gcpProject := utils.MustGetEnv("GCP_PROJECT")
|
|
fs, err := firestore.NewClient(ctx, gcpProject)
|
|
if err != nil {
|
|
flog.Error("failed to create Firestore client", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
defer fs.Close()
|
|
|
|
humanSvc := human.NewService(db.Pool())
|
|
networkSvc := network.NewReader(db.Pool())
|
|
|
|
started := time.Now()
|
|
written, scanned, err := reconcile(ctx, fs, humanSvc, networkSvc)
|
|
if err != nil {
|
|
flog.Error("reconciliation failed", "error", err, "elapsed", time.Since(started))
|
|
os.Exit(1)
|
|
}
|
|
flog.Info("reconciliation complete",
|
|
"humans_scanned", scanned,
|
|
"humans_written", written,
|
|
"elapsed", time.Since(started),
|
|
)
|
|
}
|
|
|
|
func reconcile(
|
|
ctx context.Context,
|
|
fs *firestore.Client,
|
|
humanSvc human.Service,
|
|
networkReader network.Reader,
|
|
) (written, scanned int, err error) {
|
|
humans, err := humanSvc.ListAll(ctx)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
|
|
memberships, err := networkReader.ListAllMemberships(ctx)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
|
|
current, err := snapshotMirror(ctx, fs)
|
|
if err != nil {
|
|
return 0, 0, err
|
|
}
|
|
|
|
bulk := fs.BulkWriter(ctx)
|
|
for _, h := range humans {
|
|
scanned++
|
|
desired := memberships[h.ID]
|
|
if desired == nil {
|
|
desired = []string{}
|
|
}
|
|
if sameSet(current[h.ID], desired) {
|
|
continue
|
|
}
|
|
if _, err := bulk.Set(fs.Collection("humans").Doc(h.ID), map[string]any{
|
|
"networks": desired,
|
|
"updated_at": firestore.ServerTimestamp,
|
|
}, firestore.MergeAll); err != nil {
|
|
return written, scanned, err
|
|
}
|
|
written++
|
|
}
|
|
bulk.End()
|
|
return written, scanned, nil
|
|
}
|
|
|
|
// One iterator, N billed reads — returns humanId → mirrored networks.
|
|
func snapshotMirror(ctx context.Context, fs *firestore.Client) (map[string][]string, error) {
|
|
out := map[string][]string{}
|
|
iter := fs.Collection("humans").Documents(ctx)
|
|
defer iter.Stop()
|
|
for {
|
|
doc, err := iter.Next()
|
|
if err == iterator.Done {
|
|
break
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var data struct {
|
|
Networks []string `firestore:"networks"`
|
|
}
|
|
if err := doc.DataTo(&data); err != nil {
|
|
flog.Warn("skipping malformed mirror doc", "id", doc.Ref.ID, "error", err)
|
|
continue
|
|
}
|
|
out[doc.Ref.ID] = data.Networks
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// Set equality (order- and duplicate-insensitive); Firestore array ops don't
|
|
// preserve order.
|
|
func sameSet(a, b []string) bool {
|
|
if len(a) == 0 && len(b) == 0 {
|
|
return true
|
|
}
|
|
ac := slices.Clone(a)
|
|
bc := slices.Clone(b)
|
|
slices.Sort(ac)
|
|
slices.Sort(bc)
|
|
ac = slices.Compact(ac)
|
|
bc = slices.Compact(bc)
|
|
return slices.Equal(ac, bc)
|
|
}
|