refactor: agentic comment cleanup

This commit is contained in:
talksik
2026-05-18 10:52:37 -07:00
parent 6f476fe773
commit 4421b8e832
45 changed files with 269 additions and 545 deletions
+6 -31
View File
@@ -20,22 +20,17 @@ import (
)
const (
// Only notify about streams with activity in the last 24 hours
maxActivityAge = 24 * time.Hour
// Minimum time a message must be unread before we consider notifying
unreadThreshold = 10 * time.Minute
// Minimum time between emails to the same user
emailCooldown = 12 * time.Hour
maxActivityAge = 24 * time.Hour // ignore streams idle longer than this
unreadThreshold = 10 * time.Minute // grace window before a message is "unread"
emailCooldown = 12 * time.Hour // min gap between emails to the same user
)
func main() {
ctx := context.Background()
// Initialize Postgres
db.Init()
defer db.Cleanup()
// Initialize Firestore
gcpProject := utils.MustGetEnv("GCP_PROJECT")
firestoreClient, err := firestore.NewClient(ctx, gcpProject)
if err != nil {
@@ -44,7 +39,6 @@ func main() {
}
defer firestoreClient.Close()
// Initialize aero (email) gRPC client
aeroAddr := utils.MustGetEnv("AERO_ADDR")
aeroConn, err := grpc.NewClient(aeroAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
@@ -54,7 +48,6 @@ func main() {
defer aeroConn.Close()
aeroSvc := pbaero.NewPrimaryClient(aeroConn)
// Initialize pusher gRPC client
pusherAddr := utils.MustGetEnv("PUSHER_GRPC_ADDR")
pusherConn, err := grpc.NewClient(pusherAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
@@ -64,7 +57,6 @@ func main() {
defer pusherConn.Close()
pusherSvc := pbpusher.NewPusherServiceClient(pusherConn)
// Initialize services
humanSvc := human.NewService(db.Pool())
networkSvc := network.NewReader(db.Pool())
@@ -86,13 +78,11 @@ func runNotificationCycle(
) error {
now := time.Now()
// Load all networks
networks, err := networkReader.ListAll(ctx)
if err != nil {
return fmt.Errorf("listing networks: %w", err)
}
// Load all humans into a lookup map
allHumans, err := humanSvc.ListAll(ctx)
if err != nil {
return fmt.Errorf("listing humans: %w", err)
@@ -102,11 +92,9 @@ func runNotificationCycle(
humansById[h.ID] = h
}
// Track which streams each human is behind on, and the latest activity across those streams
behindCounts := map[string]int{}
latestActivity := map[string]time.Time{}
// Get all currently connected humans
allOnline := map[string]bool{}
onlineResp, err := pusherSvc.GetOnlineHumanIds(ctx, &pbpusher.GetOnlineHumanIdsRequest{})
if err != nil {
@@ -118,35 +106,29 @@ func runNotificationCycle(
}
for _, net := range networks {
// Query Firestore for open streams in this network
streams, err := getOpenStreams(ctx, fsClient, net.ID)
if err != nil {
slog.Error("failed to query streams", "networkId", net.ID, "error", err)
continue
}
// net.MemberHumanIds already contains the admin (Create() adds them and
// RemoveMemberFromNetwork won't drop them) — no need to union separately.
// net.MemberHumanIds already includes the admin.
for _, stream := range streams {
if stream.LastChildCreatedAt == nil {
continue
}
// Skip streams with no recent activity
if now.Sub(*stream.LastChildCreatedAt) > maxActivityAge {
continue
}
// Skip if the latest message is too fresh (within threshold)
if now.Sub(*stream.LastChildCreatedAt) < unreadThreshold {
continue
}
// Resolve members from visible_to using the shared helper
for _, humanId := range network.ResolveVisibility(stream.VisibleTo, net.MemberHumanIds) {
marker, hasMarker := stream.PlaybackMarkers[humanId]
if hasMarker && !marker.Before(*stream.LastChildCreatedAt) {
continue // up to date
continue
}
// No marker or marker is behind → this human is behind on this stream
behindCounts[humanId]++
if stream.LastChildCreatedAt.After(latestActivity[humanId]) {
latestActivity[humanId] = *stream.LastChildCreatedAt
@@ -156,10 +138,8 @@ func runNotificationCycle(
}
// Send notifications
sentCount := 0
for humanId, count := range behindCounts {
// Skip online users
if allOnline[humanId] {
slog.Info("human online...skipping email", "humanId", humanId)
continue
@@ -170,28 +150,24 @@ func runNotificationCycle(
continue
}
// Skip if notifications disabled
if !h.EmailNotificationsEnabled {
continue
}
// Skip if no new activity since last notification
// Skip if nothing new since the previous email.
if h.LastEmailNotificationSentAt != nil && !latestActivity[humanId].After(*h.LastEmailNotificationSentAt) {
continue
}
// Enforce cooldown between emails to the same user
if h.LastEmailNotificationSentAt != nil && now.Sub(*h.LastEmailNotificationSentAt) < emailCooldown {
continue
}
// Send email
if err := sendNotificationEmail(ctx, aeroSvc, h, count); err != nil {
slog.Error("failed to send email", "humanId", humanId, "error", err)
continue
}
// Update last sent timestamp
if err := humanSvc.UpdateLastEmailNotificationSentAt(ctx, humanId, now); err != nil {
slog.Error("failed to update last_email_notification_sent_at", "humanId", humanId, "error", err)
}
@@ -207,7 +183,6 @@ func runNotificationCycle(
return nil
}
// getOpenStreams queries Firestore for all open stream particles in a network.
func getOpenStreams(ctx context.Context, client *firestore.Client, networkId string) ([]particle.FirestoreStreamParticle, error) {
collPath := fmt.Sprintf("networks/%s/children", networkId)
docs, err := client.Collection(collPath).
+7 -15
View File
@@ -1,13 +1,7 @@
// memberreconciler is a one-shot job (also safe to run on a cron) that makes
// the Firestore membership mirror (humans/{humanId}.networks) match the
// authoritative Postgres network_members table.
//
// Run on a cron to heal any drift from a dropped mirror
// write in network.Service.
//
// The reconciler reads the current Firestore state and only writes humans
// whose mirrored networks differ from Postgres. Writes cost ~3x reads, and in
// steady state drift is rare, so read-first keeps the cron nearly free.
// memberreconciler reconciles the Firestore membership mirror
// (humans/{humanId}.networks) against the authoritative Postgres
// network_members table. Safe to run on a cron — only humans whose mirrored
// set differs from Postgres are written, so a steady-state run is nearly free.
package main
import (
@@ -98,8 +92,7 @@ func reconcile(
return written, scanned, nil
}
// snapshotMirror streams the humans collection once and returns a map of
// humanId -> current networks array. One iterator, N billed reads.
// One iterator, N billed reads — returns humanId → mirrored networks.
func snapshotMirror(ctx context.Context, fs *firestore.Client) (map[string][]string, error) {
out := map[string][]string{}
iter := fs.Collection("humans").Documents(ctx)
@@ -124,9 +117,8 @@ func snapshotMirror(ctx context.Context, fs *firestore.Client) (map[string][]str
return out, nil
}
// sameSet reports whether a and b contain the same elements, ignoring order
// and duplicates. Firestore array ops don't preserve order, so set equality is
// the right comparison for the networks array.
// Set equality (order- and duplicate-insensitive); Firestore array ops don't
// preserve order.
func sameSet(a, b []string) bool {
if len(a) == 0 && len(b) == 0 {
return true
+1 -2
View File
@@ -185,8 +185,7 @@ func main() {
mux.Handle("GET /waitlist/{email}", withAuth(h.GetWaitlistEntry))
mux.Handle("POST /waitlist/invite", withAuth(h.InviteWaitlistEntrant))
// Apply middleware
// nil allows all origins (required for electron app)
// nil = allow all origins (Electron app needs it).
muxWithCors := middleware.CORS(nil)(mux)
addr := fmt.Sprintf("0.0.0.0:%s", port)
+21 -35
View File
@@ -38,13 +38,9 @@ func createClient(ctx context.Context) *firestore.Client {
return client
}
// The purpose of the particle processor worker is to listen for new particles
// across all streams and perform side effects such as
// - generate transcript if the particle is of type media
// - send mobile notifications if a client is offline
// - update the parent stream's `last_child_created_at`
// - generate vector embedding
// - synthesize and decide whether ai should generate a particle as a response
// Listens for new particles and runs per-particle side effects: transcripts,
// transcode, parent stream's last_child_created_at, freemium usage, and push
// notifications for offline recipients.
func main() {
ctx := context.Background()
@@ -80,9 +76,8 @@ func main() {
humanSvc := human.NewService(db.Pool())
networkReader := network.NewReader(db.Pool())
pushTokenSvc := pushnotify.NewService(db.Pool())
// EXPO_ACCESS_TOKEN is required: with Enhanced Security enabled on the Expo
// project, sends without it fail; without it, anyone holding one of our
// push tokens could spam our users via the public Expo endpoint.
// EXPO_ACCESS_TOKEN is required: Enhanced Security is on for our Expo
// project (otherwise anyone holding one of our push tokens could spam users).
expoClient := pushnotify.NewExpoClient(utils.MustGetEnv("EXPO_ACCESS_TOKEN"))
notifier := pushnotify.NewNotifier(networkReader, pushTokenSvc, pusherClient, expoClient)
@@ -127,8 +122,8 @@ func main() {
slog.Debug("processing particle", "particleID", particleID, "data", change.Doc.Data())
// --- Perform side effects ---
// All of them do not stop us from marking the particle as processed.
// Side effects below are best-effort — failures don't prevent
// marking the particle as processed.
parentDoc := loadParentParticle(ctx, change.Doc)
updateParentLastChildCreatedAt(ctx, change.Doc, parentDoc)
transcript := transcribeMediaParticle(ctx, depotSvc, speechSvc, change.Doc)
@@ -143,9 +138,8 @@ func main() {
}
}
// transcribeMediaParticle transcribes a media particle, writes the structured
// transcript to Firestore, and returns the raw transcript text. Returns "" for
// non-media particles or on any error (errors are logged internally).
// Writes the structured transcript to Firestore and returns the raw text;
// returns "" for non-media particles or on any error (logged internally).
func transcribeMediaParticle(ctx context.Context, depotSvc depot.Service, speechSvc speech.SpeechService, doc *firestore.DocumentSnapshot) string {
var mediaParticle particle.FirestoreMediaParticle
err := doc.DataTo(&mediaParticle)
@@ -227,10 +221,9 @@ func toFirestoreTranscript(result *speech.TranscriptResult) particle.FirestoreTr
}
}
// recordFreemiumUsage bumps the network's daily message counter for non-container
// particles. Idempotent via the surrounding processed_particles guard: the worker
// only reaches this path on first-seen particles, so a crash/restart won't
// double-count.
// Bumps the network's daily message counter for non-container particles.
// The surrounding processed_particles guard keeps this idempotent across
// crashes/restarts.
func recordFreemiumUsage(ctx context.Context, billingSvc billing.Service, doc *firestore.DocumentSnapshot) {
rawType, err := doc.DataAt("type")
if err != nil {
@@ -247,7 +240,7 @@ func recordFreemiumUsage(ctx context.Context, billingSvc billing.Service, doc *f
slog.Error("invalid particle type", "error", err, "particleID", doc.Ref.ID)
return
}
// Containers (stream/folder) don't count as "messages" for the daily cap.
// Containers don't count toward the daily message cap.
if particleType == particle.TypeStream || particleType == particle.TypeFolder {
return
}
@@ -263,8 +256,7 @@ func recordFreemiumUsage(ctx context.Context, billingSvc billing.Service, doc *f
}
}
// loadParentParticle fetches the immediate parent particle doc for `doc`.
// Returns nil (and logs) if the path doesn't have a parent or the read fails.
// Returns nil (and logs) if the path has no parent or the read fails.
func loadParentParticle(ctx context.Context, doc *firestore.DocumentSnapshot) *firestore.DocumentSnapshot {
parentChildrenCollectionRef := doc.Ref.Parent
if parentChildrenCollectionRef == nil {
@@ -283,9 +275,8 @@ func loadParentParticle(ctx context.Context, doc *firestore.DocumentSnapshot) *f
return parentParticleDoc
}
// updateParentLastChildCreatedAt updates the parent stream's last_child_created_at
// to the child's actual created_at timestamp, so it stays directly comparable with
// playback markers (which also store child created_at values).
// Sets last_child_created_at to the child's created_at so it stays directly
// comparable with playback markers (which also store child created_at values).
func updateParentLastChildCreatedAt(ctx context.Context, doc *firestore.DocumentSnapshot, parent *firestore.DocumentSnapshot) {
if parent == nil {
return
@@ -307,7 +298,6 @@ func updateParentLastChildCreatedAt(ctx context.Context, doc *firestore.Document
return
}
// Read the child's created_at — this is the same value that playback markers store
childCreatedAt, err := doc.DataAt("created_at")
if err != nil {
slog.Error("failed to read child created_at", "error", err, "particleID", doc.Ref.ID)
@@ -325,10 +315,9 @@ func updateParentLastChildCreatedAt(ctx context.Context, doc *firestore.Document
}
}
// notifyForParticle dispatches a push notification for a newly-created particle.
// Skips containers (streams/folders) and particles whose parent isn't a stream
// (notifications are only sent for stream messages today). The transcript arg
// is used as the preview body for media particles when available.
// Skips containers and particles whose parent isn't a stream — notifications
// are scoped to stream messages today. The transcript arg becomes the preview
// body for media particles when available.
func notifyForParticle(
ctx context.Context,
notifier *pushnotify.Notifier,
@@ -405,11 +394,8 @@ func notifyForParticle(
}
}
// previewForParticle builds the visible notification body. Kept short — push
// previews truncate aggressively on lockscreens. For media, prefers the
// transcript text (already computed by transcribeMediaParticle in the same
// processing step) and falls back to the generic "Sent a ..." line if speech
// recognition produced nothing.
// Builds the notification body. Kept short — lockscreens truncate aggressively.
// Media prefers transcript text and falls back to a generic "Sent a …" line.
func previewForParticle(pType particle.ParticleType, doc *firestore.DocumentSnapshot, transcript string) string {
switch pType {
case particle.TypeText:
+7 -19
View File
@@ -25,29 +25,22 @@ func main() {
port := utils.MustGetEnv("PORT")
grpcPort := utils.MustGetEnv("GRPC_PORT")
// Initialize database (for network membership checks)
db.Init()
defer db.Cleanup()
// Redis for auth session validation (same DB as orion)
authRedis := internal.ConnectAndTestRedis(db.RedisDBAuth)
authRedis := internal.ConnectAndTestRedis(db.RedisDBAuth) // shared with orion
pusherRedis := internal.ConnectAndTestRedis(db.RedisDBPusher) // presence + pub/sub
// Redis for pusher state (presence hashes, pub/sub)
pusherRedis := internal.ConnectAndTestRedis(db.RedisDBPusher)
sessionReader := auth.NewSessionReader(authRedis)
networkReader := network.NewReader(db.Pool())
// Services
sessionReader := auth.NewSessionReader(authRedis) // pusher only validates sessions
networkReader := network.NewReader(db.Pool()) // pusher only checks membership
// Pod identity (use hostname in k8s, which is the pod name)
// Hostname is the k8s pod name.
podID, err := os.Hostname()
if err != nil {
podID = fmt.Sprintf("pod-%d", os.Getpid())
}
// Pusher core
bridge := pusher.NewRedisBridge(pusherRedis, podID)
// Context for graceful shutdown
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
@@ -56,16 +49,11 @@ func main() {
bridge.SetHub(hub)
server := pusher.NewServer(ctx, hub, bridge, sessionReader)
// Start hub event loop
go hub.Run(ctx)
// Start Redis Pub/Sub listener
go bridge.Listen(ctx)
// Start pod heartbeat + stale pod cleanup
go bridge.Heartbeat(ctx)
// --- gRPC server (internal, for presence queries) ---
// --- gRPC server (internal presence queries) ---
grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%s", grpcPort))
if err != nil {
slog.Error("failed to listen for gRPC", "port", grpcPort, "error", err)
@@ -104,7 +92,7 @@ func main() {
<-sigCh
slog.Info("shutting down...")
cancel() // stops hub, bridge listener, heartbeat
cancel()
grpcServer.GracefulStop()
httpServer.Shutdown(context.Background())
+4 -12
View File
@@ -1,12 +1,6 @@
// transcodebackfill is a one-shot job that scans every doc under the
// "children" Firestore collection group, identifies media particles missing a
// transcoded variant, and re-runs the transcode + upload + Firestore-update
// flow against the configured GCS bucket. Idempotent — re-running the Job
// after a partial failure picks up where it left off via the existing
// `transcoded_object_id != ""` short-circuit inside particle.Transcode.
//
// Intended to back-fill the production iOS playback backlog created before we
// launched mobile / particle processor worker only transcodes new media.
// transcodebackfill walks every doc under the "children" collection group and
// re-runs transcode for media particles missing a transcoded variant.
// Idempotent: particle.Transcode short-circuits on transcoded_object_id != "".
package main
import (
@@ -104,9 +98,7 @@ func run(ctx context.Context, fs *firestore.Client, depotSvc depot.Service) (sta
s.scanned++
// Cheap pre-filter: most docs under the "children" collection group
// are not media particles. DataAt avoids unmarshalling the full
// document for those.
// DataAt avoids unmarshalling the full doc; most children aren't media.
rawType, err := doc.DataAt("type")
if err != nil {
s.skippedNonMedia++