mobile v0.1 with deployment for ios (#191)

* stage 1: project init * stage 2: skeleton with navigation * step 2.5: streams list * step 4: stream playback experience * step 5-6: compose experience * fix: broken record * transcode media particles to mp4 * build: reproducible go generate * build: rename skaffold module for particle processor worker * infra: increase particle processor worker resources Was dealing with OOM errors * tweaks to mobile * log transcode work * view on desktop placeholder * tweak padding * cap video resolution to save on memory * infra: bump memory limits as insurance * ux improvements * update bundle id for mobile * config for mobile
2026-04-29 17:39:11 -07:00
parent 3a11a82cd3
commit e3461dd5cd
110 changed files with 14682 additions and 22 deletions
@@ -1,11 +1,13 @@
 package main

 import (
+	"bytes"
 	"context"
 	"fmt"
 	"log"
 	"log/slog"
 	"os"
+	"os/exec"
 	"strings"
 	"time"

@@ -107,6 +109,7 @@ func main() {

 			updateParentLastChildCreatedAt(ctx, change.Doc)
 			transcribeMediaParticle(ctx, depotSvc, speechSvc, change.Doc)
+			transcodeMediaParticle(ctx, depotSvc, change.Doc)
 			recordFreemiumUsage(ctx, billingSvc, change.Doc)

 			if err := processingRepo.MarkProcessed(ctx, particleID); err != nil {
@@ -162,6 +165,152 @@ func transcribeMediaParticle(ctx context.Context, depotSvc depot.Service, speech
 	slog.Info("transcribed media particle", "particleID", doc.Ref.ID)
 }

+// transcodeMediaParticle produces an iOS-playable MP4/m4a derivative for media
+// particles whose original mime type AVPlayer can't decode (notably the WebM
+// the desktop recorder emits today). Skips when the source is already in an
+// iOS-playable family or when a transcoded variant has already been written.
+//
+// ffmpeg reads directly from the GCS signed URL and writes to a local temp
+// file — `+faststart` requires seekable output, so a stdout pipe wouldn't work.
+// The temp file is then streamed to GCS via depotSvc.CreateFromReader (no
+// presigned-PUT round-trip — the worker has direct SDK access).
+func transcodeMediaParticle(ctx context.Context, depotSvc depot.Service, doc *firestore.DocumentSnapshot) {
+	var mediaParticle particle.FirestoreMediaParticle
+	if err := doc.DataTo(&mediaParticle); err != nil {
+		slog.Error("transcode: unable to marshal particle data", "error", err)
+		return
+	}
+
+	particleType, err := particle.ParseParticleType(mediaParticle.Type)
+	if err != nil {
+		slog.Error("transcode: invalid particle type", "error", err)
+		return
+	}
+	if particleType != particle.TypeMedia {
+    slog.Info("transcode: particle is not of type media")
+		return
+	}
+
+	// Already transcoded — re-delivery within the 5-min Firestore window.
+	if mediaParticle.Properties.TranscodedObjectId != "" {
+		return
+	}
+
+	// Source is already iOS-playable; nothing to do.
+	if isIOSPlayableMime(mediaParticle.Properties.MimeType) {
+    slog.Info("transcode: skipping because already playable on ios")
+		return
+	}
+
+	sourceURL, err := depotSvc.GetDownloadURL(ctx, mediaParticle.Properties.ObjectId)
+	if err != nil {
+		slog.Error("transcode: failed to get download URL", "error", err, "object_id", mediaParticle.Properties.ObjectId)
+		return
+	}
+
+	isAudio := strings.HasPrefix(mediaParticle.Properties.MimeType, "audio/")
+	var outputExt, outputMime string
+	if isAudio {
+		outputExt = ".m4a"
+		outputMime = "audio/mp4"
+	} else {
+		outputExt = ".mp4"
+		outputMime = "video/mp4"
+	}
+
+	tmp, err := os.CreateTemp("", "transcode-*"+outputExt)
+	if err != nil {
+		slog.Error("transcode: failed to create temp file", "error", err)
+		return
+	}
+	tmpPath := tmp.Name()
+	tmp.Close()
+	defer os.Remove(tmpPath)
+
+	transcodeCtx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+
+	var args []string
+	if isAudio {
+		args = []string{
+			"-y", "-i", sourceURL,
+			"-vn",
+			"-c:a", "aac", "-b:a", "128k",
+			"-movflags", "+faststart",
+			tmpPath,
+		}
+	} else {
+		// Cap encoder parallelism and lookahead to keep memory bounded — screen
+		// recordings come in at native display resolution (often 1440p–4K) and
+		// libx264's per-thread lookahead/reference buffers blow past the worker's
+		// memory limit otherwise. Output is also downscaled to 1080p max, which
+		// mobile playback won't notice; the original WebM stays in GCS untouched.
+		args = []string{
+			"-y", "-i", sourceURL,
+			"-vf", "scale='min(1920,iw)':-2:flags=lanczos",
+			"-c:v", "libx264", "-preset", "veryfast", "-crf", "23",
+			"-pix_fmt", "yuv420p", "-profile:v", "baseline", "-level", "3.1",
+			"-x264-params", "rc-lookahead=20:ref=2",
+			"-threads", "2", "-filter_threads", "2",
+			"-c:a", "aac", "-b:a", "128k",
+			"-movflags", "+faststart",
+			tmpPath,
+		}
+	}
+
+	cmd := exec.CommandContext(transcodeCtx, "ffmpeg", args...)
+	var stderr bytes.Buffer
+	cmd.Stderr = &stderr
+	if err := cmd.Run(); err != nil {
+		slog.Error("transcode: ffmpeg failed", "error", err, "stderr", stderr.String(), "particleID", doc.Ref.ID)
+		return
+	}
+
+	networkID, err := networkIDFromParticlePath(doc.Ref.Path)
+	if err != nil {
+		slog.Error("transcode: failed to derive network id", "error", err, "path", doc.Ref.Path)
+		return
+	}
+
+	f, err := os.Open(tmpPath)
+	if err != nil {
+		slog.Error("transcode: failed to open transcoded file", "error", err, "path", tmpPath)
+		return
+	}
+	defer f.Close()
+
+	newObj, err := depotSvc.CreateFromReader(ctx, depot.CreateFromReaderInput{
+		Prefix:      networkID,
+		Name:        "transcoded" + outputExt,
+		ContentType: outputMime,
+	}, f)
+	if err != nil {
+		slog.Error("transcode: failed to upload transcoded object", "error", err, "particleID", doc.Ref.ID)
+		return
+	}
+
+	_, err = doc.Ref.Set(ctx, map[string]interface{}{
+		"properties": map[string]interface{}{
+			"transcoded_object_id": newObj.ID,
+			"transcoded_mime_type": outputMime,
+		},
+	}, firestore.MergeAll)
+	if err != nil {
+		slog.Error("transcode: failed to update particle in firestore", "error", err, "particleID", doc.Ref.ID)
+		return
+	}
+
+	slog.Info("transcoded media particle", "particleID", doc.Ref.ID, "transcoded_object_id", newObj.ID, "mime", outputMime)
+}
+
+func isIOSPlayableMime(mime string) bool {
+	switch mime {
+	case "video/mp4", "video/quicktime", "audio/mp4", "audio/aac", "audio/x-m4a", "audio/mpeg":
+		return true
+	}
+	return false
+}
+
 func toFirestoreTranscript(result *speech.TranscriptResult) particle.FirestoreTranscript {
 	words := make([]particle.FirestoreTranscriptWord, len(result.Words))
 	for i, w := range result.Words {