feat: generate transcript and event-driven particle processing
This generates the transcript and shows the caption experience on the client side for media particles. It also simplifies other side effects that we must perform such as updating the `last_child_created_at` field for stream and container particles.
This commit is contained in:
@@ -5,12 +5,19 @@ import (
|
||||
"fmt"
|
||||
"log"
|
||||
"log/slog"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/flowy-live/llink/internal/db"
|
||||
"github.com/flowy-live/llink/internal/depot"
|
||||
"github.com/flowy-live/llink/internal/particle"
|
||||
"github.com/flowy-live/llink/internal/speech"
|
||||
"github.com/flowy-live/llink/internal/utils"
|
||||
"google.golang.org/grpc/codes"
|
||||
"google.golang.org/grpc/status"
|
||||
|
||||
"cloud.google.com/go/firestore"
|
||||
"cloud.google.com/go/storage"
|
||||
)
|
||||
|
||||
func createClient(ctx context.Context) *firestore.Client {
|
||||
@@ -28,13 +35,38 @@ func createClient(ctx context.Context) *firestore.Client {
|
||||
// - generate transcript if the particle is of type media
|
||||
// - send mobile notifications if a client is offline
|
||||
// - update the parent stream's `last_child_created_at`
|
||||
// - generate vector embedding
|
||||
// - synthesize and decide whether ai should generate a particle as a response
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
|
||||
db.Init()
|
||||
defer db.Cleanup()
|
||||
processingRepo := particle.NewProcessingRepository(db.Pool())
|
||||
|
||||
storageClient, err := storage.NewClient(ctx)
|
||||
if err != nil {
|
||||
slog.Error("failed to create GCS client", "error", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
defer storageClient.Close()
|
||||
|
||||
gcsBucket := utils.MustGetEnv("GCS_BUCKET")
|
||||
depotSvc := depot.NewService(db.Pool(), storageClient, depot.Config{
|
||||
GoogleServiceAccountEmail: utils.MustGetEnv("GOOGLE_SERVICE_ACCOUNT_EMAIL"),
|
||||
BucketName: gcsBucket,
|
||||
})
|
||||
|
||||
speechSvc := speech.NewSpeechService(ctx)
|
||||
|
||||
client := createClient(ctx)
|
||||
defer client.Close()
|
||||
|
||||
it := client.CollectionGroup("children").Snapshots(ctx)
|
||||
var initialLoad = true
|
||||
cutoff := time.Now().Add(-5 * time.Minute)
|
||||
it := client.CollectionGroup("children").
|
||||
Where("created_at", ">", cutoff).
|
||||
Snapshots(ctx)
|
||||
|
||||
for {
|
||||
snap, err := it.Next()
|
||||
if e := status.Code(err); e == codes.DeadlineExceeded || e == codes.Canceled {
|
||||
@@ -42,26 +74,167 @@ func main() {
|
||||
}
|
||||
if err != nil {
|
||||
slog.Error("error in processing snapshot", "error", err)
|
||||
continue
|
||||
}
|
||||
|
||||
if snap != nil {
|
||||
if initialLoad {
|
||||
slog.Info("initial load", "changeCount", len(snap.Changes))
|
||||
} else {
|
||||
for _, change := range snap.Changes {
|
||||
switch change.Kind {
|
||||
case firestore.DocumentAdded:
|
||||
slog.Info("document added: ")
|
||||
case firestore.DocumentModified:
|
||||
slog.Info("document modified")
|
||||
case firestore.DocumentRemoved:
|
||||
slog.Info("document removed")
|
||||
}
|
||||
slog.Info("received document snapshot", "data", change.Doc.Data())
|
||||
}
|
||||
if snap == nil {
|
||||
continue
|
||||
}
|
||||
|
||||
for _, change := range snap.Changes {
|
||||
if change.Kind != firestore.DocumentAdded {
|
||||
continue
|
||||
}
|
||||
|
||||
particleID := change.Doc.Ref.ID
|
||||
|
||||
processed, err := processingRepo.IsProcessed(ctx, particleID)
|
||||
if err != nil {
|
||||
slog.Error("failed to check processing status", "particleID", particleID, "error", err)
|
||||
continue
|
||||
}
|
||||
if processed {
|
||||
slog.Debug("skipping already processed particle", "particleID", particleID)
|
||||
continue
|
||||
}
|
||||
|
||||
slog.Debug("processing particle", "particleID", particleID, "data", change.Doc.Data())
|
||||
|
||||
// --- Perform side effects ---
|
||||
|
||||
updateParentLastChildCreatedAt(ctx, change.Doc.Ref)
|
||||
transcribeMediaParticle(ctx, depotSvc, speechSvc, change.Doc)
|
||||
|
||||
if err := processingRepo.MarkProcessed(ctx, particleID); err != nil {
|
||||
slog.Error("failed to mark particle as processed", "particleID", particleID, "error", err)
|
||||
}
|
||||
}
|
||||
|
||||
initialLoad = false
|
||||
}
|
||||
}
|
||||
|
||||
func transcribeMediaParticle(ctx context.Context, depotSvc depot.Service, speechSvc speech.SpeechService, doc *firestore.DocumentSnapshot) {
|
||||
var mediaParticle particle.FirestoreMediaParticle
|
||||
err := doc.DataTo(&mediaParticle)
|
||||
if err != nil {
|
||||
slog.Error("unable to marshal particle data", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
particleType, err := particle.ParseParticleType(mediaParticle.Type)
|
||||
if err != nil {
|
||||
slog.Error("invalid particle type", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if particleType != particle.TypeMedia {
|
||||
slog.Info("received a particle of type", "particle type", particleType)
|
||||
return
|
||||
}
|
||||
|
||||
downloadURL, err := depotSvc.GetDownloadURL(ctx, mediaParticle.Properties.ObjectId)
|
||||
if err != nil {
|
||||
slog.Error("failed to get download URL", "error", err, "object_id", mediaParticle.Properties.ObjectId)
|
||||
return
|
||||
}
|
||||
|
||||
result, err := speechSvc.Transcribe(ctx, downloadURL)
|
||||
if err != nil {
|
||||
slog.Error("failed to transcribe media", "error", err, "particleID", doc.Ref.ID)
|
||||
return
|
||||
}
|
||||
|
||||
transcript := toFirestoreTranscript(result)
|
||||
|
||||
_, err = doc.Ref.Set(ctx, map[string]interface{}{
|
||||
"properties": map[string]interface{}{
|
||||
"transcript": transcript,
|
||||
},
|
||||
}, firestore.MergeAll)
|
||||
if err != nil {
|
||||
slog.Error("failed to update transcript in firestore", "error", err, "particleID", doc.Ref.ID)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("transcribed media particle", "particleID", doc.Ref.ID)
|
||||
}
|
||||
|
||||
func toFirestoreTranscript(result *speech.TranscriptResult) particle.FirestoreTranscript {
|
||||
words := make([]particle.FirestoreTranscriptWord, len(result.Words))
|
||||
for i, w := range result.Words {
|
||||
words[i] = particle.FirestoreTranscriptWord{
|
||||
Word: w.Word,
|
||||
Start: w.Start,
|
||||
End: w.End,
|
||||
}
|
||||
}
|
||||
|
||||
paragraphs := make([]particle.FirestoreTranscriptParagraph, len(result.Paragraphs))
|
||||
for i, p := range result.Paragraphs {
|
||||
sentences := make([]particle.FirestoreTranscriptSentence, len(p.Sentences))
|
||||
for j, s := range p.Sentences {
|
||||
sentences[j] = particle.FirestoreTranscriptSentence{
|
||||
Text: s.Text,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
}
|
||||
}
|
||||
paragraphs[i] = particle.FirestoreTranscriptParagraph{
|
||||
Sentences: sentences,
|
||||
Start: p.Start,
|
||||
End: p.End,
|
||||
}
|
||||
}
|
||||
|
||||
return particle.FirestoreTranscript{
|
||||
Transcript: result.Transcript,
|
||||
Words: words,
|
||||
Paragraphs: paragraphs,
|
||||
}
|
||||
}
|
||||
|
||||
// updateParentLastChildCreatedAt updates the parent particle's field only if it's a particle of type stream
|
||||
func updateParentLastChildCreatedAt(ctx context.Context, docRef *firestore.DocumentRef) {
|
||||
parentChildrenCollectionRef := docRef.Parent
|
||||
if parentChildrenCollectionRef == nil {
|
||||
return
|
||||
}
|
||||
|
||||
parentParticleDocRef := parentChildrenCollectionRef.Parent
|
||||
if parentParticleDocRef == nil {
|
||||
slog.Error("particle has no parent document", "particleID", docRef.ID)
|
||||
return
|
||||
}
|
||||
|
||||
parentParticleDoc, err := parentParticleDocRef.Get(ctx)
|
||||
if err != nil {
|
||||
slog.Error("failed to get parent particle", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
slog.Info("parent particle is", "parent particle id", parentParticleDoc.Ref.ID)
|
||||
|
||||
var streamParticle particle.FirestoreStreamParticle
|
||||
if err := parentParticleDoc.DataTo(&streamParticle); err != nil {
|
||||
slog.Error("failed to parse stream particle", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
particleType, err := particle.ParseParticleType(streamParticle.Type)
|
||||
if err != nil {
|
||||
slog.Error("invalid particle type", "error", err)
|
||||
return
|
||||
}
|
||||
|
||||
if particleType == particle.TypeStream {
|
||||
slog.Info("going to update the last_child_created_at for parent particle")
|
||||
_, err = parentParticleDocRef.Update(ctx, []firestore.Update{
|
||||
{
|
||||
Path: "last_child_created_at",
|
||||
Value: firestore.ServerTimestamp,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
slog.Error("unable to update parent particle `last_child_created_at`")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -42,14 +42,18 @@ require (
|
||||
github.com/containerd/platforms v0.2.1 // indirect
|
||||
github.com/cpuguy83/dockercfg v0.3.2 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/deepgram/deepgram-go-sdk v1.9.0 // indirect
|
||||
github.com/deepgram/deepgram-go-sdk/v3 v3.5.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/distribution/reference v0.6.0 // indirect
|
||||
github.com/docker/docker v28.5.1+incompatible // indirect
|
||||
github.com/docker/go-connections v0.6.0 // indirect
|
||||
github.com/docker/go-units v0.5.0 // indirect
|
||||
github.com/dvonthenen/websocket v1.5.1-dyv.2 // indirect
|
||||
github.com/ebitengine/purego v0.8.4 // indirect
|
||||
github.com/envoyproxy/go-control-plane/envoy v1.35.0 // indirect
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect
|
||||
github.com/fatih/color v1.15.0 // indirect
|
||||
github.com/felixge/httpsnoop v1.0.4 // indirect
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 // indirect
|
||||
github.com/go-logr/logr v1.4.3 // indirect
|
||||
@@ -59,7 +63,9 @@ require (
|
||||
github.com/google/s2a-go v0.1.9 // indirect
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7 // indirect
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 // indirect
|
||||
github.com/gorilla/schema v1.3.0 // indirect
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 // indirect
|
||||
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
@@ -67,6 +73,8 @@ require (
|
||||
github.com/lib/pq v1.10.9 // indirect
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
|
||||
github.com/magiconair/properties v1.8.10 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.17 // indirect
|
||||
github.com/moby/docker-image-spec v1.3.1 // indirect
|
||||
github.com/moby/go-archive v0.1.0 // indirect
|
||||
github.com/moby/patternmatcher v0.6.0 // indirect
|
||||
@@ -107,4 +115,5 @@ require (
|
||||
google.golang.org/genproto/googleapis/api v0.0.0-20251222181119-0a764e51fe1b // indirect
|
||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20251222181119-0a764e51fe1b // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
k8s.io/klog/v2 v2.110.1 // indirect
|
||||
)
|
||||
|
||||
@@ -64,6 +64,10 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/deepgram/deepgram-go-sdk v1.9.0 h1:FlJ1iJ//+Cz0goWGWU/Ms2jjM/wMBqyNAk+5bcAsptU=
|
||||
github.com/deepgram/deepgram-go-sdk v1.9.0/go.mod h1:il+6HLmvxa47EG12LG6VwzaHcyI8Lo+yfBsOcDq3R8s=
|
||||
github.com/deepgram/deepgram-go-sdk/v3 v3.5.0 h1:ug48j1DVNRKrkXti18/aFT3NP5HV2Q2CN3QMwTvHmy4=
|
||||
github.com/deepgram/deepgram-go-sdk/v3 v3.5.0/go.mod h1:wVr0PDvlJFWVLUmf65u+K80SJVf/PUWvkFFubGPW/As=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
|
||||
@@ -76,6 +80,8 @@ github.com/docker/go-connections v0.6.0 h1:LlMG9azAe1TqfR7sO+NJttz1gy6KO7VJBh+pM
|
||||
github.com/docker/go-connections v0.6.0/go.mod h1:AahvXYshr6JgfUJGdDCs2b5EZG/vmaMAntpSFH5BFKE=
|
||||
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
|
||||
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
|
||||
github.com/dvonthenen/websocket v1.5.1-dyv.2 h1:OXlWJJkeHt8k4+MEI0Y8SQjY2ihHYD2z/tI7sZZfsnA=
|
||||
github.com/dvonthenen/websocket v1.5.1-dyv.2/go.mod h1:q2GbopbpFJvBP4iqVvqwwahVmvu2HnCfdqCWDoQVKMM=
|
||||
github.com/ebitengine/purego v0.8.4 h1:CF7LEKg5FFOsASUj0+QwaXf8Ht6TlFxg09+S9wz0omw=
|
||||
github.com/ebitengine/purego v0.8.4/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ=
|
||||
github.com/envoyproxy/go-control-plane v0.13.5-0.20251024222203-75eaa193e329 h1:K+fnvUM0VZ7ZFJf0n4L/BRlnsb9pL/GuDG6FqaH+PwM=
|
||||
@@ -86,11 +92,14 @@ github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an
|
||||
github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8=
|
||||
github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU=
|
||||
github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs=
|
||||
github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw=
|
||||
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
|
||||
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3 h1:CVLmWDhDVRa6Mi/IgCgaopNosCaHz7zrMeF9MlZRkrs=
|
||||
github.com/go-jose/go-jose/v4 v4.1.3/go.mod h1:x4oUasVrzR7071A4TnHLGSPpNOm2a21K9Kf04k1rs08=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
|
||||
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
@@ -116,8 +125,12 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.7 h1:zrn2Ee/nWmHulBx5sAV
|
||||
github.com/googleapis/enterprise-certificate-proxy v0.3.7/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0 h1:SyjDc1mGgZU5LncH8gimWo9lW1DtIfPibOG81vgd/bo=
|
||||
github.com/googleapis/gax-go/v2 v2.15.0/go.mod h1:zVVkkxAQHa1RQpg9z2AUCMnKhi0Qld9rcmyfL1OZhoc=
|
||||
github.com/gorilla/schema v1.3.0 h1:rbciOzXAx3IB8stEFnfTwO3sYa6EWlQk79XdyustPDA=
|
||||
github.com/gorilla/schema v1.3.0/go.mod h1:Dg5SSm5PV60mhF2NFaTV1xuYYj8tV8NOPRo4FggUMnM=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4 h1:kEISI/Gx67NzH3nJxAmY/dGac80kKZgZt134u7Y/k1s=
|
||||
github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.4/go.mod h1:6Nz966r3vQYCqIzWsuEl9d7cf7mRhtDmm++sOxlnfxI=
|
||||
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f h1:7LYC+Yfkj3CTRcShK0KOL/w6iTiKyqqBA9a41Wnggw8=
|
||||
github.com/hokaccha/go-prettyjson v0.0.0-20211117102719-0474bc63780f/go.mod h1:pFlLw2CfqZiIBOx6BuCeRLCrfxBJipTY0nIOF/VbGcI=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -138,6 +151,11 @@ github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ
|
||||
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
|
||||
github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE=
|
||||
github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
|
||||
github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mdelapenya/tlscert v0.2.0 h1:7H81W6Z/4weDvZBNOfQte5GpIMo0lGYEeWbkGp5LJHI=
|
||||
github.com/mdelapenya/tlscert v0.2.0/go.mod h1:O4njj3ELLnJjGdkN7M/vIVCpZ+Cf0L6muqOG4tLSl8o=
|
||||
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
|
||||
@@ -238,6 +256,7 @@ golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
@@ -273,3 +292,5 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q=
|
||||
gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA=
|
||||
k8s.io/klog/v2 v2.110.1 h1:U/Af64HJf7FcwMcXyKm2RPM22WZzyR7OSpYj5tg3cL0=
|
||||
k8s.io/klog/v2 v2.110.1/go.mod h1:YGtd1984u+GgbuZ7e08/yBuAfKLSO0+uR1Fhi6ExXjo=
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
package particle
|
||||
|
||||
import "time"
|
||||
|
||||
type FirestoreMediaParticle struct {
|
||||
CreatedByHumanId string `firestore:"created_by_human_id"`
|
||||
Type string `firestore:"type"`
|
||||
Properties FirestoreMediaParticleProperties `firestore:"properties"`
|
||||
CreatedAt time.Time `firestore:"created_at,serverTimestamp"`
|
||||
UpdatedAt *time.Time `firestore:"updated_at,omitempty"`
|
||||
}
|
||||
|
||||
type FirestoreTranscriptWord struct {
|
||||
Word string `firestore:"word"`
|
||||
Start float64 `firestore:"start"`
|
||||
End float64 `firestore:"end"`
|
||||
}
|
||||
|
||||
type FirestoreTranscriptSentence struct {
|
||||
Text string `firestore:"text"`
|
||||
Start float64 `firestore:"start"`
|
||||
End float64 `firestore:"end"`
|
||||
}
|
||||
|
||||
type FirestoreTranscriptParagraph struct {
|
||||
Sentences []FirestoreTranscriptSentence `firestore:"sentences"`
|
||||
Start float64 `firestore:"start"`
|
||||
End float64 `firestore:"end"`
|
||||
}
|
||||
|
||||
type FirestoreTranscript struct {
|
||||
Transcript string `firestore:"transcript"`
|
||||
Words []FirestoreTranscriptWord `firestore:"words"`
|
||||
Paragraphs []FirestoreTranscriptParagraph `firestore:"paragraphs"`
|
||||
}
|
||||
|
||||
type FirestoreMediaParticleProperties struct {
|
||||
ObjectId string `firestore:"object_id"`
|
||||
MimeType string `firestore:"mime_type"`
|
||||
DurationMs int `firestore:"duration_ms"`
|
||||
SizeBytes int `firestore:"size_bytes"`
|
||||
Transcript *FirestoreTranscript `firestore:"transcript,omitempty"`
|
||||
}
|
||||
|
||||
type FirestoreStreamParticle struct {
|
||||
CreatedByHumanId string `firestore:"created_by_human_id"`
|
||||
Type string `firestore:"type"`
|
||||
// Properties FirestoreStreamParticleProperties `firestore:"properties"`
|
||||
CreatedAt time.Time `firestore:"created_at,serverTimestamp"`
|
||||
LastChildCreatedAt *time.Time `firestore:"last_child_created_at,omitempty"`
|
||||
VisibleTo []string `firestore:"visible_to"`
|
||||
UpdatedAt *time.Time `firestore:"updated_at,omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
package particle
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/jackc/pgx/v5/pgxpool"
|
||||
)
|
||||
|
||||
type ProcessingRepository interface {
|
||||
IsProcessed(ctx context.Context, particleID string) (bool, error)
|
||||
MarkProcessed(ctx context.Context, particleID string) error
|
||||
}
|
||||
|
||||
type processingRepositoryImpl struct {
|
||||
pool *pgxpool.Pool
|
||||
}
|
||||
|
||||
func NewProcessingRepository(pool *pgxpool.Pool) ProcessingRepository {
|
||||
return &processingRepositoryImpl{pool: pool}
|
||||
}
|
||||
|
||||
func (r *processingRepositoryImpl) IsProcessed(ctx context.Context, particleID string) (bool, error) {
|
||||
var exists bool
|
||||
err := r.pool.QueryRow(ctx,
|
||||
`SELECT EXISTS(SELECT 1 FROM processed_particles WHERE particle_id = $1)`,
|
||||
particleID,
|
||||
).Scan(&exists)
|
||||
return exists, err
|
||||
}
|
||||
|
||||
func (r *processingRepositoryImpl) MarkProcessed(ctx context.Context, particleID string) error {
|
||||
_, err := r.pool.Exec(ctx,
|
||||
`INSERT INTO processed_particles (particle_id) VALUES ($1) ON CONFLICT (particle_id) DO NOTHING`,
|
||||
particleID,
|
||||
)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package speech
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
|
||||
dgapi "github.com/deepgram/deepgram-go-sdk/v3/pkg/api/listen/v1/rest"
|
||||
interfaces "github.com/deepgram/deepgram-go-sdk/v3/pkg/client/interfaces"
|
||||
client "github.com/deepgram/deepgram-go-sdk/v3/pkg/client/listen"
|
||||
)
|
||||
|
||||
type TranscriptWord struct {
|
||||
Word string
|
||||
Start float64
|
||||
End float64
|
||||
}
|
||||
|
||||
type TranscriptSentence struct {
|
||||
Text string
|
||||
Start float64
|
||||
End float64
|
||||
}
|
||||
|
||||
type TranscriptParagraph struct {
|
||||
Sentences []TranscriptSentence
|
||||
Start float64
|
||||
End float64
|
||||
}
|
||||
|
||||
type TranscriptResult struct {
|
||||
Transcript string
|
||||
Words []TranscriptWord
|
||||
Paragraphs []TranscriptParagraph
|
||||
}
|
||||
|
||||
type SpeechService interface {
|
||||
Transcribe(ctx context.Context, mediaUrl string) (*TranscriptResult, error)
|
||||
}
|
||||
|
||||
type speechServiceImpl struct {
|
||||
deepgramClient *dgapi.Client
|
||||
}
|
||||
|
||||
func NewSpeechService(ctx context.Context) SpeechService {
|
||||
return &speechServiceImpl{
|
||||
deepgramClient: dgapi.New(client.NewRESTWithDefaults()),
|
||||
}
|
||||
}
|
||||
|
||||
func (s *speechServiceImpl) Transcribe(ctx context.Context, mediaUrl string) (*TranscriptResult, error) {
|
||||
options := &interfaces.PreRecordedTranscriptionOptions{
|
||||
Model: "nova-3",
|
||||
SmartFormat: true,
|
||||
Paragraphs: true,
|
||||
}
|
||||
|
||||
response, err := s.deepgramClient.FromURL(ctx, mediaUrl, options)
|
||||
if err != nil {
|
||||
slog.Error("failed to transcribe prerecorded media", "error", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
alt := response.Results.Channels[0].Alternatives[0]
|
||||
|
||||
words := make([]TranscriptWord, len(alt.Words))
|
||||
for i, w := range alt.Words {
|
||||
words[i] = TranscriptWord{
|
||||
Word: w.PunctuatedWord,
|
||||
Start: w.Start,
|
||||
End: w.End,
|
||||
}
|
||||
}
|
||||
|
||||
var paragraphs []TranscriptParagraph
|
||||
if alt.Paragraphs != nil {
|
||||
paragraphs = make([]TranscriptParagraph, len(alt.Paragraphs.Paragraphs))
|
||||
for i, p := range alt.Paragraphs.Paragraphs {
|
||||
sentences := make([]TranscriptSentence, len(p.Sentences))
|
||||
for j, s := range p.Sentences {
|
||||
sentences[j] = TranscriptSentence{
|
||||
Text: s.Text,
|
||||
Start: s.Start,
|
||||
End: s.End,
|
||||
}
|
||||
}
|
||||
paragraphs[i] = TranscriptParagraph{
|
||||
Sentences: sentences,
|
||||
Start: p.Start,
|
||||
End: p.End,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return &TranscriptResult{
|
||||
Transcript: alt.Transcript,
|
||||
Words: words,
|
||||
Paragraphs: paragraphs,
|
||||
}, nil
|
||||
}
|
||||
@@ -33,15 +33,10 @@ spec:
|
||||
value: "8080"
|
||||
- name: "AERO_ADDR"
|
||||
value: "aero:50051"
|
||||
- name: "IAM_FOR_GKE_SERVICE_ACCOUNT"
|
||||
value: "iam-for-gke-sa@flowy-dev-440017.iam.gserviceaccount.com"
|
||||
- name: "GCS_BUCKET"
|
||||
value: "flowy-llink-bucket"
|
||||
- name: "LLINK_POSTGRES_CONNECTION_URL"
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shared-secrets
|
||||
key: LLINK_POSTGRES_CONNECTION_URL
|
||||
value: "postgresql://neondb_owner:npg_ysiLkYo3Ez9G@ep-aged-feather-amfusyhe-pooler.c-5.us-east-1.aws.neon.tech/neondb?sslmode=require&channel_binding=require"
|
||||
- name: "DEEPGRAM_SECRET"
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
|
||||
@@ -29,3 +29,14 @@ spec:
|
||||
env:
|
||||
- name: "GCP_PROJECT"
|
||||
value: "flowy-dev-440017"
|
||||
- name: "GCS_BUCKET"
|
||||
value: "flowy-llink-bucket"
|
||||
- name: "GOOGLE_SERVICE_ACCOUNT_EMAIL"
|
||||
value: "iam-for-gke-sa@flowy-dev-440017.iam.gserviceaccount.com"
|
||||
- name: "LLINK_POSTGRES_CONNECTION_URL"
|
||||
value: "postgresql://neondb_owner:npg_ysiLkYo3Ez9G@ep-aged-feather-amfusyhe-pooler.c-5.us-east-1.aws.neon.tech/neondb?sslmode=require&channel_binding=require"
|
||||
- name: "DEEPGRAM_API_KEY"
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shared-secrets
|
||||
key: DEEPGRAM_SECRET
|
||||
|
||||
@@ -12,9 +12,6 @@ spec:
|
||||
args: ["-path", "/migrations", "-database", "$(LLINK_POSTGRES_CONNECTION_URL)", "up"]
|
||||
env:
|
||||
- name: "LLINK_POSTGRES_CONNECTION_URL"
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: shared-secrets
|
||||
key: LLINK_POSTGRES_CONNECTION_URL
|
||||
value: "postgresql://neondb_owner:npg_ysiLkYo3Ez9G@ep-aged-feather-amfusyhe-pooler.c-5.us-east-1.aws.neon.tech/neondb?sslmode=require&channel_binding=require"
|
||||
restartPolicy: Never
|
||||
backoffLimit: 0
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
DROP TABLE IF EXISTS processed_particles;
|
||||
@@ -0,0 +1,4 @@
|
||||
CREATE TABLE processed_particles (
|
||||
particle_id TEXT PRIMARY KEY,
|
||||
processed_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
|
||||
);
|
||||
@@ -91,11 +91,37 @@ export const FolderPropertiesSchema = z.object({
|
||||
});
|
||||
export type FolderProperties = z.infer<typeof FolderPropertiesSchema>;
|
||||
|
||||
const TranscriptWordSchema = z.object({
|
||||
word: z.string(),
|
||||
start: z.number(),
|
||||
end: z.number(),
|
||||
});
|
||||
|
||||
const TranscriptSentenceSchema = z.object({
|
||||
text: z.string(),
|
||||
start: z.number(),
|
||||
end: z.number(),
|
||||
});
|
||||
|
||||
const TranscriptParagraphSchema = z.object({
|
||||
sentences: z.array(TranscriptSentenceSchema),
|
||||
start: z.number(),
|
||||
end: z.number(),
|
||||
});
|
||||
|
||||
export const TranscriptSchema = z.object({
|
||||
transcript: z.string(),
|
||||
words: z.array(TranscriptWordSchema),
|
||||
paragraphs: z.array(TranscriptParagraphSchema),
|
||||
});
|
||||
export type Transcript = z.infer<typeof TranscriptSchema>;
|
||||
|
||||
export const MediaPropertiesSchema = z.object({
|
||||
object_id: z.string(),
|
||||
mime_type: z.string(),
|
||||
duration_ms: z.number(),
|
||||
size_bytes: z.number(),
|
||||
transcript: TranscriptSchema.optional(),
|
||||
});
|
||||
export type MediaProperties = z.infer<typeof MediaPropertiesSchema>;
|
||||
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import { useEffect, useRef, useState } from "react";
|
||||
import type { Particle } from "@/api/types";
|
||||
import { useDownloadUrl } from "@/hooks/use-download-url";
|
||||
import { useTranscriptPlayback } from "@/hooks/use-transcript-playback";
|
||||
import { TranscriptOverlay } from "@/features/particles/transcript-overlay";
|
||||
import { Skeleton } from "@/components/ui/skeleton";
|
||||
import { AudioLevelBars } from "@/components/audio/audio-level-bars";
|
||||
import { useAudioSource } from "@/components/audio/use-audio-source";
|
||||
@@ -25,6 +27,13 @@ export function MediaParticleView({
|
||||
const videoRef = useRef<HTMLVideoElement>(null);
|
||||
const audioRef = useRef<HTMLAudioElement>(null);
|
||||
const isAudio = particle.properties.mime_type?.startsWith("audio/");
|
||||
const [currentTime, setCurrentTime] = useState(0);
|
||||
|
||||
const transcript = particle.properties.transcript;
|
||||
const { activeParagraph, activeWordIndex } = useTranscriptPlayback(
|
||||
transcript,
|
||||
currentTime,
|
||||
);
|
||||
|
||||
// WORKAROUND: useAudioSource needs an audio element, but audioRef is only set after mount
|
||||
const [audioEl, setAudioEl] = useState<HTMLAudioElement | null>(null);
|
||||
@@ -55,6 +64,20 @@ export function MediaParticleView({
|
||||
return <Skeleton className="h-full w-full rounded-none" />;
|
||||
}
|
||||
|
||||
const handleTimeUpdate = (e: React.SyntheticEvent<HTMLAudioElement | HTMLVideoElement>) => {
|
||||
const { currentTime: time, duration } = e.currentTarget;
|
||||
setCurrentTime(time);
|
||||
if (duration > 0) onProgress?.(time / duration);
|
||||
};
|
||||
|
||||
const captionOverlay = transcript ? (
|
||||
<TranscriptOverlay
|
||||
transcript={transcript}
|
||||
activeParagraph={activeParagraph}
|
||||
activeWordIndex={activeWordIndex}
|
||||
/>
|
||||
) : null;
|
||||
|
||||
if (isAudio) {
|
||||
return (
|
||||
<div className="relative flex h-full w-full items-center justify-center bg-black/90">
|
||||
@@ -67,10 +90,7 @@ export function MediaParticleView({
|
||||
src={url}
|
||||
autoPlay
|
||||
onEnded={onEnded}
|
||||
onTimeUpdate={(e) => {
|
||||
const { currentTime, duration } = e.currentTarget;
|
||||
if (duration > 0) onProgress?.(currentTime / duration);
|
||||
}}
|
||||
onTimeUpdate={handleTimeUpdate}
|
||||
/>
|
||||
|
||||
{audioSource && (
|
||||
@@ -78,6 +98,8 @@ export function MediaParticleView({
|
||||
<AudioLevelBars sourceNode={audioSource.sourceNode} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{captionOverlay}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -90,12 +112,11 @@ export function MediaParticleView({
|
||||
autoPlay
|
||||
playsInline
|
||||
onEnded={onEnded}
|
||||
onTimeUpdate={(e) => {
|
||||
const { currentTime, duration } = e.currentTarget;
|
||||
if (duration > 0) onProgress?.(currentTime / duration);
|
||||
}}
|
||||
onTimeUpdate={handleTimeUpdate}
|
||||
className="h-full w-full object-cover"
|
||||
/>
|
||||
|
||||
{captionOverlay}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
import { useMemo } from "react";
|
||||
import type { Transcript } from "@/api/types";
|
||||
|
||||
interface TranscriptOverlayProps {
|
||||
transcript: Transcript;
|
||||
activeParagraph: Transcript["paragraphs"][number] | null;
|
||||
activeWordIndex: number | null;
|
||||
}
|
||||
|
||||
export function TranscriptOverlay({
|
||||
transcript,
|
||||
activeParagraph,
|
||||
activeWordIndex,
|
||||
}: TranscriptOverlayProps) {
|
||||
// Find the words that belong to the active paragraph by time range
|
||||
const paragraphWords = useMemo(() => {
|
||||
if (!activeParagraph) return [];
|
||||
return transcript.words.filter(
|
||||
(w) => w.start >= activeParagraph.start && w.end <= activeParagraph.end,
|
||||
);
|
||||
}, [transcript.words, activeParagraph]);
|
||||
|
||||
if (!activeParagraph || paragraphWords.length === 0) return null;
|
||||
|
||||
// The active word from the flat array — find it by index to compare
|
||||
const activeWord =
|
||||
activeWordIndex !== null ? transcript.words[activeWordIndex] : null;
|
||||
|
||||
return (
|
||||
<div className="absolute bottom-10 left-0 right-0 flex justify-center px-6 py-4 max-w-md">
|
||||
<p className="rounded-lg bg-black/20 px-5 py-3 text-base leading-relaxed backdrop-blur-sm text-lg text-left">
|
||||
{paragraphWords.map((word, i) => {
|
||||
const isSpoken =
|
||||
activeWord !== null && word.start <= activeWord.end;
|
||||
|
||||
return (
|
||||
<span
|
||||
key={`${word.start}-${i}`}
|
||||
className={
|
||||
isSpoken
|
||||
? "text-white font-medium transition-colors duration-150"
|
||||
: "text-white/40 transition-colors duration-150"
|
||||
}
|
||||
>
|
||||
{i > 0 ? " " : ""}
|
||||
{word.word}
|
||||
</span>
|
||||
);
|
||||
})}
|
||||
</p>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useMutation } from "@tanstack/react-query";
|
||||
import { createParticle, updateStreamLastChildAt } from "@/lib/firestore-particles";
|
||||
import { createParticle } from "@/lib/firestore-particles";
|
||||
import type { ParticleType, ParticlePropertiesMap } from "@/api/types";
|
||||
import { particlePath, ParticlePath, toFirestoreChildrenPath, toFirestoreDocPath } from "@/lib/particle-path";
|
||||
|
||||
@@ -15,16 +15,12 @@ export function useCreateParticle() {
|
||||
return useMutation({
|
||||
mutationFn: async (params: CreateParticleParams) => {
|
||||
const collectionPath = toFirestoreChildrenPath(params.path);
|
||||
const result = await createParticle(
|
||||
return await createParticle(
|
||||
collectionPath,
|
||||
params.type,
|
||||
params.properties,
|
||||
params.createdByHumanId,
|
||||
);
|
||||
|
||||
const streamDocPath = toFirestoreDocPath(params.path);
|
||||
await updateStreamLastChildAt(streamDocPath);
|
||||
return result;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
import { useMemo } from "react";
|
||||
import type { Transcript } from "@/api/types";
|
||||
|
||||
interface TranscriptPlaybackState {
|
||||
/** The paragraph currently being spoken, or null if before/after speech */
|
||||
activeParagraph: Transcript["paragraphs"][number] | null;
|
||||
/** Index of the active word within the transcript's flat words array */
|
||||
activeWordIndex: number | null;
|
||||
}
|
||||
|
||||
export function useTranscriptPlayback(
|
||||
transcript: Transcript | undefined,
|
||||
currentTime: number,
|
||||
): TranscriptPlaybackState {
|
||||
return useMemo(() => {
|
||||
if (!transcript) return { activeParagraph: null, activeWordIndex: null };
|
||||
|
||||
const activeParagraph =
|
||||
transcript.paragraphs.find(
|
||||
(p) => currentTime >= p.start && currentTime <= p.end,
|
||||
) ?? null;
|
||||
|
||||
// Binary search for active word
|
||||
const words = transcript.words;
|
||||
let activeWordIndex: number | null = null;
|
||||
let lo = 0;
|
||||
let hi = words.length - 1;
|
||||
while (lo <= hi) {
|
||||
const mid = (lo + hi) >>> 1;
|
||||
if (currentTime < words[mid].start) hi = mid - 1;
|
||||
else if (currentTime > words[mid].end) lo = mid + 1;
|
||||
else {
|
||||
activeWordIndex = mid;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return { activeParagraph, activeWordIndex };
|
||||
}, [transcript, currentTime]);
|
||||
}
|
||||
@@ -263,16 +263,6 @@ export async function updateParticle(
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateStreamLastChildAt(
|
||||
docPath: string,
|
||||
): Promise<void> {
|
||||
const particleRef = typedDoc(docPath);
|
||||
await updateDoc(particleRef, {
|
||||
last_child_created_at: serverTimestamp(),
|
||||
updated_at: serverTimestamp(),
|
||||
});
|
||||
}
|
||||
|
||||
export async function updateStreamPlaybackMarker(
|
||||
docPath: string,
|
||||
humanId: string,
|
||||
|
||||
Reference in New Issue
Block a user