142 lines
3.4 KiB
Go
142 lines
3.4 KiB
Go
// transcodebackfill walks every doc under the "children" collection group and
|
|
// re-runs transcode for media particles missing a transcoded variant.
|
|
// Idempotent: particle.Transcode short-circuits on transcoded_object_id != "".
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"os"
|
|
"time"
|
|
|
|
"github.com/flowy-live/llink/internal/utils/flog"
|
|
|
|
"cloud.google.com/go/firestore"
|
|
"cloud.google.com/go/storage"
|
|
"google.golang.org/api/iterator"
|
|
|
|
"github.com/flowy-live/llink/internal/db"
|
|
"github.com/flowy-live/llink/internal/depot"
|
|
"github.com/flowy-live/llink/internal/particle"
|
|
"github.com/flowy-live/llink/internal/utils"
|
|
)
|
|
|
|
func main() {
|
|
ctx := context.Background()
|
|
|
|
db.Init()
|
|
defer db.Cleanup()
|
|
|
|
storageClient, err := storage.NewClient(ctx)
|
|
if err != nil {
|
|
flog.Error("failed to create GCS client", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
defer storageClient.Close()
|
|
|
|
depotSvc := depot.NewService(db.Pool(), storageClient, depot.Config{
|
|
GoogleServiceAccountEmail: utils.MustGetEnv("GOOGLE_SERVICE_ACCOUNT_EMAIL"),
|
|
BucketName: utils.MustGetEnv("GCS_BUCKET"),
|
|
})
|
|
|
|
gcpProject := utils.MustGetEnv("GCP_PROJECT")
|
|
fs, err := firestore.NewClient(ctx, gcpProject)
|
|
if err != nil {
|
|
flog.Error("failed to create Firestore client", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
defer fs.Close()
|
|
|
|
started := time.Now()
|
|
stats, err := run(ctx, fs, depotSvc)
|
|
elapsed := time.Since(started)
|
|
|
|
flog.Info("transcode_backfill_summary",
|
|
"scanned", stats.scanned,
|
|
"media", stats.media,
|
|
"transcoded", stats.transcoded,
|
|
"skippedAlreadyDone", stats.skippedAlreadyDone,
|
|
"skippedIOSPlayable", stats.skippedIOSPlayable,
|
|
"skippedNonMedia", stats.skippedNonMedia,
|
|
"failures", stats.failures,
|
|
"elapsed_seconds", elapsed.Seconds(),
|
|
)
|
|
|
|
if err != nil {
|
|
flog.Error("transcode backfill aborted", "error", err)
|
|
os.Exit(1)
|
|
}
|
|
if stats.failures > 0 {
|
|
os.Exit(1)
|
|
}
|
|
}
|
|
|
|
type stats struct {
|
|
scanned int
|
|
media int
|
|
transcoded int
|
|
skippedAlreadyDone int
|
|
skippedIOSPlayable int
|
|
skippedNonMedia int
|
|
failures int
|
|
}
|
|
|
|
func run(ctx context.Context, fs *firestore.Client, depotSvc depot.Service) (stats, error) {
|
|
var s stats
|
|
|
|
it := fs.CollectionGroup("children").
|
|
OrderBy(firestore.DocumentID, firestore.Asc).
|
|
Documents(ctx)
|
|
defer it.Stop()
|
|
|
|
for {
|
|
doc, err := it.Next()
|
|
if err == iterator.Done {
|
|
return s, nil
|
|
}
|
|
if err != nil {
|
|
return s, err
|
|
}
|
|
|
|
s.scanned++
|
|
|
|
// DataAt avoids unmarshalling the full doc; most children aren't media.
|
|
rawType, err := doc.DataAt("type")
|
|
if err != nil {
|
|
s.skippedNonMedia++
|
|
continue
|
|
}
|
|
typeStr, ok := rawType.(string)
|
|
if !ok || typeStr != string(particle.TypeMedia) {
|
|
s.skippedNonMedia++
|
|
continue
|
|
}
|
|
|
|
s.media++
|
|
|
|
result := particle.Transcode(ctx, depotSvc, doc)
|
|
switch {
|
|
case result.Err != nil:
|
|
s.failures++
|
|
flog.Error("transcode_backfill_failure",
|
|
"particleID", doc.Ref.ID,
|
|
"path", doc.Ref.Path,
|
|
"error", result.Err,
|
|
)
|
|
case result.Skipped && result.SkipReason == particle.SkipReasonAlreadyTranscoded:
|
|
s.skippedAlreadyDone++
|
|
case result.Skipped && result.SkipReason == particle.SkipReasonIOSPlayable:
|
|
s.skippedIOSPlayable++
|
|
case result.Skipped:
|
|
s.skippedNonMedia++
|
|
default:
|
|
s.transcoded++
|
|
flog.Info("transcode_backfill_progress",
|
|
"particleID", doc.Ref.ID,
|
|
"transcodedObjectID", result.TranscodedObjectID,
|
|
"outputMime", result.OutputMimeType,
|
|
"idx", s.transcoded,
|
|
)
|
|
}
|
|
}
|
|
}
|