// transcodebackfill walks every doc under the "children" collection group and // re-runs transcode for media particles missing a transcoded variant. // Idempotent: particle.Transcode short-circuits on transcoded_object_id != "". package main import ( "context" "os" "time" "github.com/flowy-live/llink/internal/utils/flog" "cloud.google.com/go/firestore" "cloud.google.com/go/storage" "google.golang.org/api/iterator" "github.com/flowy-live/llink/internal/db" "github.com/flowy-live/llink/internal/depot" "github.com/flowy-live/llink/internal/particle" "github.com/flowy-live/llink/internal/utils" ) func main() { ctx := context.Background() db.Init() defer db.Cleanup() storageClient, err := storage.NewClient(ctx) if err != nil { flog.Error("failed to create GCS client", "error", err) os.Exit(1) } defer storageClient.Close() depotSvc := depot.NewService(db.Pool(), storageClient, depot.Config{ GoogleServiceAccountEmail: utils.MustGetEnv("GOOGLE_SERVICE_ACCOUNT_EMAIL"), BucketName: utils.MustGetEnv("GCS_BUCKET"), }) gcpProject := utils.MustGetEnv("GCP_PROJECT") fs, err := firestore.NewClient(ctx, gcpProject) if err != nil { flog.Error("failed to create Firestore client", "error", err) os.Exit(1) } defer fs.Close() started := time.Now() stats, err := run(ctx, fs, depotSvc) elapsed := time.Since(started) flog.Info("transcode_backfill_summary", "scanned", stats.scanned, "media", stats.media, "transcoded", stats.transcoded, "skippedAlreadyDone", stats.skippedAlreadyDone, "skippedIOSPlayable", stats.skippedIOSPlayable, "skippedNonMedia", stats.skippedNonMedia, "failures", stats.failures, "elapsed_seconds", elapsed.Seconds(), ) if err != nil { flog.Error("transcode backfill aborted", "error", err) os.Exit(1) } if stats.failures > 0 { os.Exit(1) } } type stats struct { scanned int media int transcoded int skippedAlreadyDone int skippedIOSPlayable int skippedNonMedia int failures int } func run(ctx context.Context, fs *firestore.Client, depotSvc depot.Service) (stats, error) { var s stats it := fs.CollectionGroup("children"). OrderBy(firestore.DocumentID, firestore.Asc). Documents(ctx) defer it.Stop() for { doc, err := it.Next() if err == iterator.Done { return s, nil } if err != nil { return s, err } s.scanned++ // DataAt avoids unmarshalling the full doc; most children aren't media. rawType, err := doc.DataAt("type") if err != nil { s.skippedNonMedia++ continue } typeStr, ok := rawType.(string) if !ok || typeStr != string(particle.TypeMedia) { s.skippedNonMedia++ continue } s.media++ result := particle.Transcode(ctx, depotSvc, doc) switch { case result.Err != nil: s.failures++ flog.Error("transcode_backfill_failure", "particleID", doc.Ref.ID, "path", doc.Ref.Path, "error", result.Err, ) case result.Skipped && result.SkipReason == particle.SkipReasonAlreadyTranscoded: s.skippedAlreadyDone++ case result.Skipped && result.SkipReason == particle.SkipReasonIOSPlayable: s.skippedIOSPlayable++ case result.Skipped: s.skippedNonMedia++ default: s.transcoded++ flog.Info("transcode_backfill_progress", "particleID", doc.Ref.ID, "transcodedObjectID", result.TranscodedObjectID, "outputMime", result.OutputMimeType, "idx", s.transcoded, ) } } }