refactor: agentic comment cleanup
This commit is contained in:
@@ -20,22 +20,17 @@ import (
|
||||
)
|
||||
|
||||
const (
|
||||
// Only notify about streams with activity in the last 24 hours
|
||||
maxActivityAge = 24 * time.Hour
|
||||
// Minimum time a message must be unread before we consider notifying
|
||||
unreadThreshold = 10 * time.Minute
|
||||
// Minimum time between emails to the same user
|
||||
emailCooldown = 12 * time.Hour
|
||||
maxActivityAge = 24 * time.Hour // ignore streams idle longer than this
|
||||
unreadThreshold = 10 * time.Minute // grace window before a message is "unread"
|
||||
emailCooldown = 12 * time.Hour // min gap between emails to the same user
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
// Initialize Postgres
|
||||
db.Init()
|
||||
defer db.Cleanup()
|
||||
|
||||
// Initialize Firestore
|
||||
gcpProject := utils.MustGetEnv("GCP_PROJECT")
|
||||
firestoreClient, err := firestore.NewClient(ctx, gcpProject)
|
||||
if err != nil {
|
||||
@@ -44,7 +39,6 @@ func main() {
|
||||
}
|
||||
defer firestoreClient.Close()
|
||||
|
||||
// Initialize aero (email) gRPC client
|
||||
aeroAddr := utils.MustGetEnv("AERO_ADDR")
|
||||
aeroConn, err := grpc.NewClient(aeroAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
@@ -54,7 +48,6 @@ func main() {
|
||||
defer aeroConn.Close()
|
||||
aeroSvc := pbaero.NewPrimaryClient(aeroConn)
|
||||
|
||||
// Initialize pusher gRPC client
|
||||
pusherAddr := utils.MustGetEnv("PUSHER_GRPC_ADDR")
|
||||
pusherConn, err := grpc.NewClient(pusherAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
|
||||
if err != nil {
|
||||
@@ -64,7 +57,6 @@ func main() {
|
||||
defer pusherConn.Close()
|
||||
pusherSvc := pbpusher.NewPusherServiceClient(pusherConn)
|
||||
|
||||
// Initialize services
|
||||
humanSvc := human.NewService(db.Pool())
|
||||
networkSvc := network.NewReader(db.Pool())
|
||||
|
||||
@@ -86,13 +78,11 @@ func runNotificationCycle(
|
||||
) error {
|
||||
now := time.Now()
|
||||
|
||||
// Load all networks
|
||||
networks, err := networkReader.ListAll(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing networks: %w", err)
|
||||
}
|
||||
|
||||
// Load all humans into a lookup map
|
||||
allHumans, err := humanSvc.ListAll(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("listing humans: %w", err)
|
||||
@@ -102,11 +92,9 @@ func runNotificationCycle(
|
||||
humansById[h.ID] = h
|
||||
}
|
||||
|
||||
// Track which streams each human is behind on, and the latest activity across those streams
|
||||
behindCounts := map[string]int{}
|
||||
latestActivity := map[string]time.Time{}
|
||||
|
||||
// Get all currently connected humans
|
||||
allOnline := map[string]bool{}
|
||||
onlineResp, err := pusherSvc.GetOnlineHumanIds(ctx, &pbpusher.GetOnlineHumanIdsRequest{})
|
||||
if err != nil {
|
||||
@@ -118,35 +106,29 @@ func runNotificationCycle(
|
||||
}
|
||||
|
||||
for _, net := range networks {
|
||||
// Query Firestore for open streams in this network
|
||||
streams, err := getOpenStreams(ctx, fsClient, net.ID)
|
||||
if err != nil {
|
||||
slog.Error("failed to query streams", "networkId", net.ID, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// net.MemberHumanIds already contains the admin (Create() adds them and
|
||||
// RemoveMemberFromNetwork won't drop them) — no need to union separately.
|
||||
// net.MemberHumanIds already includes the admin.
|
||||
for _, stream := range streams {
|
||||
if stream.LastChildCreatedAt == nil {
|
||||
continue
|
||||
}
|
||||
// Skip streams with no recent activity
|
||||
if now.Sub(*stream.LastChildCreatedAt) > maxActivityAge {
|
||||
continue
|
||||
}
|
||||
// Skip if the latest message is too fresh (within threshold)
|
||||
if now.Sub(*stream.LastChildCreatedAt) < unreadThreshold {
|
||||
continue
|
||||
}
|
||||
|
||||
// Resolve members from visible_to using the shared helper
|
||||
for _, humanId := range network.ResolveVisibility(stream.VisibleTo, net.MemberHumanIds) {
|
||||
marker, hasMarker := stream.PlaybackMarkers[humanId]
|
||||
if hasMarker && !marker.Before(*stream.LastChildCreatedAt) {
|
||||
continue // up to date
|
||||
continue
|
||||
}
|
||||
// No marker or marker is behind → this human is behind on this stream
|
||||
behindCounts[humanId]++
|
||||
if stream.LastChildCreatedAt.After(latestActivity[humanId]) {
|
||||
latestActivity[humanId] = *stream.LastChildCreatedAt
|
||||
@@ -156,10 +138,8 @@ func runNotificationCycle(
|
||||
|
||||
}
|
||||
|
||||
// Send notifications
|
||||
sentCount := 0
|
||||
for humanId, count := range behindCounts {
|
||||
// Skip online users
|
||||
if allOnline[humanId] {
|
||||
slog.Info("human online...skipping email", "humanId", humanId)
|
||||
continue
|
||||
@@ -170,28 +150,24 @@ func runNotificationCycle(
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip if notifications disabled
|
||||
if !h.EmailNotificationsEnabled {
|
||||
continue
|
||||
}
|
||||
|
||||
// Skip if no new activity since last notification
|
||||
// Skip if nothing new since the previous email.
|
||||
if h.LastEmailNotificationSentAt != nil && !latestActivity[humanId].After(*h.LastEmailNotificationSentAt) {
|
||||
continue
|
||||
}
|
||||
|
||||
// Enforce cooldown between emails to the same user
|
||||
if h.LastEmailNotificationSentAt != nil && now.Sub(*h.LastEmailNotificationSentAt) < emailCooldown {
|
||||
continue
|
||||
}
|
||||
|
||||
// Send email
|
||||
if err := sendNotificationEmail(ctx, aeroSvc, h, count); err != nil {
|
||||
slog.Error("failed to send email", "humanId", humanId, "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Update last sent timestamp
|
||||
if err := humanSvc.UpdateLastEmailNotificationSentAt(ctx, humanId, now); err != nil {
|
||||
slog.Error("failed to update last_email_notification_sent_at", "humanId", humanId, "error", err)
|
||||
}
|
||||
@@ -207,7 +183,6 @@ func runNotificationCycle(
|
||||
return nil
|
||||
}
|
||||
|
||||
// getOpenStreams queries Firestore for all open stream particles in a network.
|
||||
func getOpenStreams(ctx context.Context, client *firestore.Client, networkId string) ([]particle.FirestoreStreamParticle, error) {
|
||||
collPath := fmt.Sprintf("networks/%s/children", networkId)
|
||||
docs, err := client.Collection(collPath).
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
// memberreconciler is a one-shot job (also safe to run on a cron) that makes
|
||||
// the Firestore membership mirror (humans/{humanId}.networks) match the
|
||||
// authoritative Postgres network_members table.
|
||||
//
|
||||
// Run on a cron to heal any drift from a dropped mirror
|
||||
// write in network.Service.
|
||||
//
|
||||
// The reconciler reads the current Firestore state and only writes humans
|
||||
// whose mirrored networks differ from Postgres. Writes cost ~3x reads, and in
|
||||
// steady state drift is rare, so read-first keeps the cron nearly free.
|
||||
// memberreconciler reconciles the Firestore membership mirror
|
||||
// (humans/{humanId}.networks) against the authoritative Postgres
|
||||
// network_members table. Safe to run on a cron — only humans whose mirrored
|
||||
// set differs from Postgres are written, so a steady-state run is nearly free.
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -98,8 +92,7 @@ func reconcile(
|
||||
return written, scanned, nil
|
||||
}
|
||||
|
||||
// snapshotMirror streams the humans collection once and returns a map of
|
||||
// humanId -> current networks array. One iterator, N billed reads.
|
||||
// One iterator, N billed reads — returns humanId → mirrored networks.
|
||||
func snapshotMirror(ctx context.Context, fs *firestore.Client) (map[string][]string, error) {
|
||||
out := map[string][]string{}
|
||||
iter := fs.Collection("humans").Documents(ctx)
|
||||
@@ -124,9 +117,8 @@ func snapshotMirror(ctx context.Context, fs *firestore.Client) (map[string][]str
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// sameSet reports whether a and b contain the same elements, ignoring order
|
||||
// and duplicates. Firestore array ops don't preserve order, so set equality is
|
||||
// the right comparison for the networks array.
|
||||
// Set equality (order- and duplicate-insensitive); Firestore array ops don't
|
||||
// preserve order.
|
||||
func sameSet(a, b []string) bool {
|
||||
if len(a) == 0 && len(b) == 0 {
|
||||
return true
|
||||
|
||||
@@ -185,8 +185,7 @@ func main() {
|
||||
mux.Handle("GET /waitlist/{email}", withAuth(h.GetWaitlistEntry))
|
||||
mux.Handle("POST /waitlist/invite", withAuth(h.InviteWaitlistEntrant))
|
||||
|
||||
// Apply middleware
|
||||
// nil allows all origins (required for electron app)
|
||||
// nil = allow all origins (Electron app needs it).
|
||||
muxWithCors := middleware.CORS(nil)(mux)
|
||||
|
||||
addr := fmt.Sprintf("0.0.0.0:%s", port)
|
||||
|
||||
@@ -38,13 +38,9 @@ func createClient(ctx context.Context) *firestore.Client {
|
||||
return client
|
||||
}
|
||||
|
||||
// The purpose of the particle processor worker is to listen for new particles
|
||||
// across all streams and perform side effects such as
|
||||
// - generate transcript if the particle is of type media
|
||||
// - send mobile notifications if a client is offline
|
||||
// - update the parent stream's `last_child_created_at`
|
||||
// - generate vector embedding
|
||||
// - synthesize and decide whether ai should generate a particle as a response
|
||||
// Listens for new particles and runs per-particle side effects: transcripts,
|
||||
// transcode, parent stream's last_child_created_at, freemium usage, and push
|
||||
// notifications for offline recipients.
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
@@ -80,9 +76,8 @@ func main() {
|
||||
humanSvc := human.NewService(db.Pool())
|
||||
networkReader := network.NewReader(db.Pool())
|
||||
pushTokenSvc := pushnotify.NewService(db.Pool())
|
||||
// EXPO_ACCESS_TOKEN is required: with Enhanced Security enabled on the Expo
|
||||
// project, sends without it fail; without it, anyone holding one of our
|
||||
// push tokens could spam our users via the public Expo endpoint.
|
||||
// EXPO_ACCESS_TOKEN is required: Enhanced Security is on for our Expo
|
||||
// project (otherwise anyone holding one of our push tokens could spam users).
|
||||
expoClient := pushnotify.NewExpoClient(utils.MustGetEnv("EXPO_ACCESS_TOKEN"))
|
||||
notifier := pushnotify.NewNotifier(networkReader, pushTokenSvc, pusherClient, expoClient)
|
||||
|
||||
@@ -127,8 +122,8 @@ func main() {
|
||||
|
||||
slog.Debug("processing particle", "particleID", particleID, "data", change.Doc.Data())
|
||||
|
||||
// --- Perform side effects ---
|
||||
// All of them do not stop us from marking the particle as processed.
|
||||
// Side effects below are best-effort — failures don't prevent
|
||||
// marking the particle as processed.
|
||||
parentDoc := loadParentParticle(ctx, change.Doc)
|
||||
updateParentLastChildCreatedAt(ctx, change.Doc, parentDoc)
|
||||
transcript := transcribeMediaParticle(ctx, depotSvc, speechSvc, change.Doc)
|
||||
@@ -143,9 +138,8 @@ func main() {
|
||||
}
|
||||
}
|
||||
|
||||
// transcribeMediaParticle transcribes a media particle, writes the structured
|
||||
// transcript to Firestore, and returns the raw transcript text. Returns "" for
|
||||
// non-media particles or on any error (errors are logged internally).
|
||||
// Writes the structured transcript to Firestore and returns the raw text;
|
||||
// returns "" for non-media particles or on any error (logged internally).
|
||||
func transcribeMediaParticle(ctx context.Context, depotSvc depot.Service, speechSvc speech.SpeechService, doc *firestore.DocumentSnapshot) string {
|
||||
var mediaParticle particle.FirestoreMediaParticle
|
||||
err := doc.DataTo(&mediaParticle)
|
||||
@@ -227,10 +221,9 @@ func toFirestoreTranscript(result *speech.TranscriptResult) particle.FirestoreTr
|
||||
}
|
||||
}
|
||||
|
||||
// recordFreemiumUsage bumps the network's daily message counter for non-container
|
||||
// particles. Idempotent via the surrounding processed_particles guard: the worker
|
||||
// only reaches this path on first-seen particles, so a crash/restart won't
|
||||
// double-count.
|
||||
// Bumps the network's daily message counter for non-container particles.
|
||||
// The surrounding processed_particles guard keeps this idempotent across
|
||||
// crashes/restarts.
|
||||
func recordFreemiumUsage(ctx context.Context, billingSvc billing.Service, doc *firestore.DocumentSnapshot) {
|
||||
rawType, err := doc.DataAt("type")
|
||||
if err != nil {
|
||||
@@ -247,7 +240,7 @@ func recordFreemiumUsage(ctx context.Context, billingSvc billing.Service, doc *f
|
||||
slog.Error("invalid particle type", "error", err, "particleID", doc.Ref.ID)
|
||||
return
|
||||
}
|
||||
// Containers (stream/folder) don't count as "messages" for the daily cap.
|
||||
// Containers don't count toward the daily message cap.
|
||||
if particleType == particle.TypeStream || particleType == particle.TypeFolder {
|
||||
return
|
||||
}
|
||||
@@ -263,8 +256,7 @@ func recordFreemiumUsage(ctx context.Context, billingSvc billing.Service, doc *f
|
||||
}
|
||||
}
|
||||
|
||||
// loadParentParticle fetches the immediate parent particle doc for `doc`.
|
||||
// Returns nil (and logs) if the path doesn't have a parent or the read fails.
|
||||
// Returns nil (and logs) if the path has no parent or the read fails.
|
||||
func loadParentParticle(ctx context.Context, doc *firestore.DocumentSnapshot) *firestore.DocumentSnapshot {
|
||||
parentChildrenCollectionRef := doc.Ref.Parent
|
||||
if parentChildrenCollectionRef == nil {
|
||||
@@ -283,9 +275,8 @@ func loadParentParticle(ctx context.Context, doc *firestore.DocumentSnapshot) *f
|
||||
return parentParticleDoc
|
||||
}
|
||||
|
||||
// updateParentLastChildCreatedAt updates the parent stream's last_child_created_at
|
||||
// to the child's actual created_at timestamp, so it stays directly comparable with
|
||||
// playback markers (which also store child created_at values).
|
||||
// Sets last_child_created_at to the child's created_at so it stays directly
|
||||
// comparable with playback markers (which also store child created_at values).
|
||||
func updateParentLastChildCreatedAt(ctx context.Context, doc *firestore.DocumentSnapshot, parent *firestore.DocumentSnapshot) {
|
||||
if parent == nil {
|
||||
return
|
||||
@@ -307,7 +298,6 @@ func updateParentLastChildCreatedAt(ctx context.Context, doc *firestore.Document
|
||||
return
|
||||
}
|
||||
|
||||
// Read the child's created_at — this is the same value that playback markers store
|
||||
childCreatedAt, err := doc.DataAt("created_at")
|
||||
if err != nil {
|
||||
slog.Error("failed to read child created_at", "error", err, "particleID", doc.Ref.ID)
|
||||
@@ -325,10 +315,9 @@ func updateParentLastChildCreatedAt(ctx context.Context, doc *firestore.Document
|
||||
}
|
||||
}
|
||||
|
||||
// notifyForParticle dispatches a push notification for a newly-created particle.
|
||||
// Skips containers (streams/folders) and particles whose parent isn't a stream
|
||||
// (notifications are only sent for stream messages today). The transcript arg
|
||||
// is used as the preview body for media particles when available.
|
||||
// Skips containers and particles whose parent isn't a stream — notifications
|
||||
// are scoped to stream messages today. The transcript arg becomes the preview
|
||||
// body for media particles when available.
|
||||
func notifyForParticle(
|
||||
ctx context.Context,
|
||||
notifier *pushnotify.Notifier,
|
||||
@@ -405,11 +394,8 @@ func notifyForParticle(
|
||||
}
|
||||
}
|
||||
|
||||
// previewForParticle builds the visible notification body. Kept short — push
|
||||
// previews truncate aggressively on lockscreens. For media, prefers the
|
||||
// transcript text (already computed by transcribeMediaParticle in the same
|
||||
// processing step) and falls back to the generic "Sent a ..." line if speech
|
||||
// recognition produced nothing.
|
||||
// Builds the notification body. Kept short — lockscreens truncate aggressively.
|
||||
// Media prefers transcript text and falls back to a generic "Sent a …" line.
|
||||
func previewForParticle(pType particle.ParticleType, doc *firestore.DocumentSnapshot, transcript string) string {
|
||||
switch pType {
|
||||
case particle.TypeText:
|
||||
|
||||
@@ -25,29 +25,22 @@ func main() {
|
||||
port := utils.MustGetEnv("PORT")
|
||||
grpcPort := utils.MustGetEnv("GRPC_PORT")
|
||||
|
||||
// Initialize database (for network membership checks)
|
||||
db.Init()
|
||||
defer db.Cleanup()
|
||||
|
||||
// Redis for auth session validation (same DB as orion)
|
||||
authRedis := internal.ConnectAndTestRedis(db.RedisDBAuth)
|
||||
authRedis := internal.ConnectAndTestRedis(db.RedisDBAuth) // shared with orion
|
||||
pusherRedis := internal.ConnectAndTestRedis(db.RedisDBPusher) // presence + pub/sub
|
||||
|
||||
// Redis for pusher state (presence hashes, pub/sub)
|
||||
pusherRedis := internal.ConnectAndTestRedis(db.RedisDBPusher)
|
||||
sessionReader := auth.NewSessionReader(authRedis)
|
||||
networkReader := network.NewReader(db.Pool())
|
||||
|
||||
// Services
|
||||
sessionReader := auth.NewSessionReader(authRedis) // pusher only validates sessions
|
||||
networkReader := network.NewReader(db.Pool()) // pusher only checks membership
|
||||
|
||||
// Pod identity (use hostname in k8s, which is the pod name)
|
||||
// Hostname is the k8s pod name.
|
||||
podID, err := os.Hostname()
|
||||
if err != nil {
|
||||
podID = fmt.Sprintf("pod-%d", os.Getpid())
|
||||
}
|
||||
|
||||
// Pusher core
|
||||
bridge := pusher.NewRedisBridge(pusherRedis, podID)
|
||||
// Context for graceful shutdown
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
@@ -56,16 +49,11 @@ func main() {
|
||||
bridge.SetHub(hub)
|
||||
server := pusher.NewServer(ctx, hub, bridge, sessionReader)
|
||||
|
||||
// Start hub event loop
|
||||
go hub.Run(ctx)
|
||||
|
||||
// Start Redis Pub/Sub listener
|
||||
go bridge.Listen(ctx)
|
||||
|
||||
// Start pod heartbeat + stale pod cleanup
|
||||
go bridge.Heartbeat(ctx)
|
||||
|
||||
// --- gRPC server (internal, for presence queries) ---
|
||||
// --- gRPC server (internal presence queries) ---
|
||||
grpcListener, err := net.Listen("tcp", fmt.Sprintf("0.0.0.0:%s", grpcPort))
|
||||
if err != nil {
|
||||
slog.Error("failed to listen for gRPC", "port", grpcPort, "error", err)
|
||||
@@ -104,7 +92,7 @@ func main() {
|
||||
<-sigCh
|
||||
|
||||
slog.Info("shutting down...")
|
||||
cancel() // stops hub, bridge listener, heartbeat
|
||||
cancel()
|
||||
|
||||
grpcServer.GracefulStop()
|
||||
httpServer.Shutdown(context.Background())
|
||||
|
||||
@@ -1,12 +1,6 @@
|
||||
// transcodebackfill is a one-shot job that scans every doc under the
|
||||
// "children" Firestore collection group, identifies media particles missing a
|
||||
// transcoded variant, and re-runs the transcode + upload + Firestore-update
|
||||
// flow against the configured GCS bucket. Idempotent — re-running the Job
|
||||
// after a partial failure picks up where it left off via the existing
|
||||
// `transcoded_object_id != ""` short-circuit inside particle.Transcode.
|
||||
//
|
||||
// Intended to back-fill the production iOS playback backlog created before we
|
||||
// launched mobile / particle processor worker only transcodes new media.
|
||||
// transcodebackfill walks every doc under the "children" collection group and
|
||||
// re-runs transcode for media particles missing a transcoded variant.
|
||||
// Idempotent: particle.Transcode short-circuits on transcoded_object_id != "".
|
||||
package main
|
||||
|
||||
import (
|
||||
@@ -104,9 +98,7 @@ func run(ctx context.Context, fs *firestore.Client, depotSvc depot.Service) (sta
|
||||
|
||||
s.scanned++
|
||||
|
||||
// Cheap pre-filter: most docs under the "children" collection group
|
||||
// are not media particles. DataAt avoids unmarshalling the full
|
||||
// document for those.
|
||||
// DataAt avoids unmarshalling the full doc; most children aren't media.
|
||||
rawType, err := doc.DataAt("type")
|
||||
if err != nil {
|
||||
s.skippedNonMedia++
|
||||
|
||||
Reference in New Issue
Block a user