migrate orion repo into monorepo structure

go/internal/depot/errors.go

@@ -0,0 +1,8 @@
+package depot
+
+import "errors"
+
+var (
+	ErrNotFound     = errors.New("object not found")
+	ErrInvalidInput = errors.New("invalid input")
+)

go/internal/depot/models.go

@@ -0,0 +1,38 @@
+package depot
+
+import "time"
+
+// Object represents a stored object in the depot
+type Object struct {
+	ID              string
+	Name            string
+	ContentType     string
+	ContentLength   int64
+	BucketName      string
+	ObjectKey       string
+	ContainsContent bool
+	CreatedAt       time.Time
+}
+
+// PrepareUploadInput represents the input for preparing an upload
+type PrepareUploadInput struct {
+	Prefix        string // Optional prefix for organizing objects (e.g., network_id)
+	Name          string
+	ContentType   string
+	ContentLength int64
+}
+
+// PrepareUploadResult represents the result of preparing an upload
+type PrepareUploadResult struct {
+	ObjectID      string
+	UploadURL     string
+	UploadHeaders map[string]string
+}
+
+// Config holds configuration for the depot service
+type Config struct {
+	GoogleServiceAccountEmail string
+	BucketName                string
+	UploadURLExpiry           time.Duration
+	DownloadURLExpiry         time.Duration
+}

go/internal/depot/repository.go

@@ -0,0 +1,112 @@
+package depot
+
+import (
+	"context"
+	"errors"
+
+	"github.com/jackc/pgx/v5"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"go.jetify.com/typeid"
+)
+
+var errNotFound = errors.New("not found")
+
+type depotIDPrefix struct{}
+
+func (depotIDPrefix) Prefix() string { return "dpo" }
+
+type depotID struct {
+	typeid.TypeID[depotIDPrefix]
+}
+
+func newDepotID() (depotID, error) {
+	return typeid.New[depotID]()
+}
+
+type repository interface {
+	create(ctx context.Context, obj *Object) (*Object, error)
+	getByID(ctx context.Context, id string) (*Object, error)
+	setContainsContent(ctx context.Context, id string, containsContent bool) error
+	delete(ctx context.Context, id string) error
+	exists(ctx context.Context, id string) (bool, error)
+}
+
+type repositoryImpl struct {
+	pool *pgxpool.Pool
+}
+
+func newRepository(pool *pgxpool.Pool) repository {
+	return &repositoryImpl{pool: pool}
+}
+
+func (r *repositoryImpl) create(ctx context.Context, obj *Object) (*Object, error) {
+	id, err := newDepotID()
+	if err != nil {
+		return nil, err
+	}
+
+	var result Object
+	err = r.pool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id, name, content_type, content_length, bucket_name, object_key, contains_content, created_at`,
+		id.String(), obj.Name, obj.ContentType, obj.ContentLength, obj.BucketName, obj.ObjectKey, obj.ContainsContent,
+	).Scan(&result.ID, &result.Name, &result.ContentType, &result.ContentLength,
+		&result.BucketName, &result.ObjectKey, &result.ContainsContent, &result.CreatedAt)
+	if err != nil {
+		return nil, err
+	}
+
+	return &result, nil
+}
+
+func (r *repositoryImpl) getByID(ctx context.Context, id string) (*Object, error) {
+	var obj Object
+	err := r.pool.QueryRow(ctx,
+		`SELECT id, name, content_type, content_length, bucket_name, object_key, contains_content, created_at
+		 FROM depot_objects WHERE id = $1`,
+		id,
+	).Scan(&obj.ID, &obj.Name, &obj.ContentType, &obj.ContentLength,
+		&obj.BucketName, &obj.ObjectKey, &obj.ContainsContent, &obj.CreatedAt)
+	if err != nil {
+		if errors.Is(err, pgx.ErrNoRows) {
+			return nil, errNotFound
+		}
+		return nil, err
+	}
+	return &obj, nil
+}
+
+func (r *repositoryImpl) setContainsContent(ctx context.Context, id string, containsContent bool) error {
+	result, err := r.pool.Exec(ctx,
+		`UPDATE depot_objects SET contains_content = $1 WHERE id = $2`,
+		containsContent, id,
+	)
+	if err != nil {
+		return err
+	}
+	if result.RowsAffected() == 0 {
+		return errNotFound
+	}
+	return nil
+}
+
+func (r *repositoryImpl) delete(ctx context.Context, id string) error {
+	result, err := r.pool.Exec(ctx, `DELETE FROM depot_objects WHERE id = $1`, id)
+	if err != nil {
+		return err
+	}
+	if result.RowsAffected() == 0 {
+		return errNotFound
+	}
+	return nil
+}
+
+func (r *repositoryImpl) exists(ctx context.Context, id string) (bool, error) {
+	var exists bool
+	err := r.pool.QueryRow(ctx,
+		`SELECT EXISTS(SELECT 1 FROM depot_objects WHERE id = $1)`,
+		id,
+	).Scan(&exists)
+	return exists, err
+}

go/internal/depot/service.go

@@ -0,0 +1,221 @@
+package depot
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log/slog"
+	"time"
+
+	"cloud.google.com/go/storage"
+	"github.com/google/uuid"
+	"github.com/jackc/pgx/v5/pgxpool"
+)
+
+const (
+	defaultUploadURLExpiry   = 15 * time.Minute
+	defaultDownloadURLExpiry = 24 * time.Hour
+)
+
+type Service interface {
+	PrepareUpload(ctx context.Context, input PrepareUploadInput) (*PrepareUploadResult, error)
+	ConfirmUpload(ctx context.Context, objectID string) (*Object, error)
+	GetByID(ctx context.Context, objectID string) (*Object, error)
+	GetDownloadURL(ctx context.Context, objectID string) (string, error)
+	Delete(ctx context.Context, objectID string) error
+	Exists(ctx context.Context, objectID string) (bool, error)
+}
+
+type serviceImpl struct {
+	repo                      repository
+	storageClient             *storage.Client
+	bucketName                string
+	uploadURLExpiry           time.Duration
+	downloadURLExpiry         time.Duration
+	googleServiceAccountEmail string
+}
+
+func NewService(pool *pgxpool.Pool, storageClient *storage.Client, config Config) Service {
+	uploadExpiry := config.UploadURLExpiry
+	if uploadExpiry == 0 {
+		uploadExpiry = defaultUploadURLExpiry
+	}
+
+	downloadExpiry := config.DownloadURLExpiry
+	if downloadExpiry == 0 {
+		downloadExpiry = defaultDownloadURLExpiry
+	}
+
+	if config.GoogleServiceAccountEmail == "" {
+		slog.Error("GoogleServiceAccountEmail is not set in config. Signed URLs may not work if the storage client is not properly authenticated with a service account.")
+		panic("GoogleServiceAccountEmail is required for signed URL generation")
+	}
+
+	return &serviceImpl{
+		repo:                      newRepository(pool),
+		storageClient:             storageClient,
+		bucketName:                config.BucketName,
+		uploadURLExpiry:           uploadExpiry,
+		downloadURLExpiry:         downloadExpiry,
+		googleServiceAccountEmail: config.GoogleServiceAccountEmail,
+	}
+}
+
+func (s *serviceImpl) PrepareUpload(ctx context.Context, input PrepareUploadInput) (*PrepareUploadResult, error) {
+	if input.Name == "" {
+		return nil, errors.Join(ErrInvalidInput, errors.New("name is required"))
+	}
+	if input.ContentType == "" {
+		return nil, errors.Join(ErrInvalidInput, errors.New("content_type is required"))
+	}
+	if input.ContentLength <= 0 {
+		return nil, errors.Join(ErrInvalidInput, errors.New("content_length must be positive"))
+	}
+
+	// Generate object key: {prefix}/{uuid}/{filename}
+	objectKey := fmt.Sprintf("%s/%s/%s", input.Prefix, uuid.New().String(), input.Name)
+
+	// Create the database record (contains_content = false initially)
+	obj := &Object{
+		Name:            input.Name,
+		ContentType:     input.ContentType,
+		ContentLength:   input.ContentLength,
+		BucketName:      s.bucketName,
+		ObjectKey:       objectKey,
+		ContainsContent: false,
+	}
+
+	created, err := s.repo.create(ctx, obj)
+	if err != nil {
+		return nil, err
+	}
+
+	// Generate a signed URL for uploading with Content-Length enforcement
+	// The Headers field specifies headers that MUST be included in the upload request
+	contentLengthHeader := fmt.Sprintf("Content-Length:%d", input.ContentLength)
+	uploadURL, err := s.storageClient.Bucket(s.bucketName).SignedURL(objectKey, &storage.SignedURLOptions{
+		GoogleAccessID: s.googleServiceAccountEmail,
+		Method:         "PUT",
+		Expires:        time.Now().Add(s.uploadURLExpiry),
+		ContentType:    input.ContentType,
+		Headers:        []string{contentLengthHeader},
+	})
+	if err != nil {
+		slog.Error("failed to generate signed upload URL", "error", err, "bucket", s.bucketName, "object_key", objectKey)
+		// Clean up the database record if we can't generate the URL
+		if delErr := s.repo.delete(ctx, created.ID); delErr != nil {
+			slog.Warn("failed to cleanup db record after signed URL failure", "error", delErr, "object_id", created.ID)
+		}
+		return nil, err
+	}
+
+	return &PrepareUploadResult{
+		ObjectID:  created.ID,
+		UploadURL: uploadURL,
+		UploadHeaders: map[string]string{
+			"Content-Type":   input.ContentType,
+			"Content-Length": fmt.Sprintf("%d", input.ContentLength),
+		},
+	}, nil
+}
+
+func (s *serviceImpl) ConfirmUpload(ctx context.Context, objectID string) (*Object, error) {
+	obj, err := s.repo.getByID(ctx, objectID)
+	if err != nil {
+		if errors.Is(err, errNotFound) {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	// Verify the object exists in GCS and check its size matches expected
+	attrs, err := s.storageClient.Bucket(obj.BucketName).Object(obj.ObjectKey).Attrs(ctx)
+	if err != nil {
+		if errors.Is(err, storage.ErrObjectNotExist) {
+			return nil, errors.Join(ErrNotFound, errors.New("object not found in storage"))
+		}
+		slog.Error("failed to get GCS object attrs", "error", err, "bucket", obj.BucketName, "object_key", obj.ObjectKey)
+		return nil, err
+	}
+
+	// Verify content length matches what was declared
+	if attrs.Size != obj.ContentLength {
+		return nil, errors.Join(ErrInvalidInput, fmt.Errorf("content length mismatch: expected %d, got %d", obj.ContentLength, attrs.Size))
+	}
+
+	// Mark as containing content
+	if err := s.repo.setContainsContent(ctx, objectID, true); err != nil {
+		if errors.Is(err, errNotFound) {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	// Fetch and return the updated object
+	return s.repo.getByID(ctx, objectID)
+}
+
+func (s *serviceImpl) GetByID(ctx context.Context, objectID string) (*Object, error) {
+	obj, err := s.repo.getByID(ctx, objectID)
+	if err != nil {
+		if errors.Is(err, errNotFound) {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+	return obj, nil
+}
+
+func (s *serviceImpl) GetDownloadURL(ctx context.Context, objectID string) (string, error) {
+	obj, err := s.repo.getByID(ctx, objectID)
+	if err != nil {
+		if errors.Is(err, errNotFound) {
+			return "", ErrNotFound
+		}
+		return "", err
+	}
+
+	// Generate a signed URL for downloading
+	downloadURL, err := s.storageClient.Bucket(obj.BucketName).SignedURL(obj.ObjectKey, &storage.SignedURLOptions{
+		GoogleAccessID: s.googleServiceAccountEmail,
+		Method:         "GET",
+		Expires:        time.Now().Add(s.downloadURLExpiry),
+	})
+	if err != nil {
+		slog.Error("failed to generate signed download URL", "error", err, "bucket", obj.BucketName, "object_key", obj.ObjectKey)
+		return "", err
+	}
+
+	return downloadURL, nil
+}
+
+func (s *serviceImpl) Delete(ctx context.Context, objectID string) error {
+	obj, err := s.repo.getByID(ctx, objectID)
+	if err != nil {
+		if errors.Is(err, errNotFound) {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	// Delete from GCS (ignore not found errors)
+	gcsErr := s.storageClient.Bucket(obj.BucketName).Object(obj.ObjectKey).Delete(ctx)
+	if gcsErr != nil && !errors.Is(gcsErr, storage.ErrObjectNotExist) {
+		slog.Error("failed to delete object from GCS", "error", gcsErr, "bucket", obj.BucketName, "object_key", obj.ObjectKey)
+		return gcsErr
+	}
+
+	// Delete from database
+	if err := s.repo.delete(ctx, objectID); err != nil {
+		if errors.Is(err, errNotFound) {
+			return ErrNotFound
+		}
+		return err
+	}
+
+	return nil
+}
+
+func (s *serviceImpl) Exists(ctx context.Context, objectID string) (bool, error) {
+	return s.repo.exists(ctx, objectID)
+}

go/internal/depot/service_test.go

@@ -0,0 +1,250 @@
+package depot_test
+
+import (
+	"context"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/flowy-live/llink/internal/depot"
+	"github.com/flowy-live/llink/internal/testhelper"
+	"github.com/jackc/pgx/v5/pgxpool"
+	"github.com/stretchr/testify/assert"
+)
+
+var dbPool *pgxpool.Pool
+
+func TestMain(m *testing.M) {
+	dbPool = testhelper.SetupTestDB()
+	defer testhelper.TeardownTestDB()
+
+	ret := m.Run()
+	os.Exit(ret)
+}
+
+// TestDepotRepository tests the repository layer directly
+// These tests can run without GCS since they only test database operations
+func TestDepotRepository_CreateAndGet(t *testing.T) {
+	ctx := context.Background()
+
+	// Create a depot object directly in the database for testing
+	var id string
+	err := dbPool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id`,
+		"dpo_test123", "test-file.txt", "text/plain", int64(1024), "test-bucket", "test-key-123", false,
+	).Scan(&id)
+	assert.NoError(t, err)
+	assert.Equal(t, "dpo_test123", id)
+
+	// Query the object back
+	var obj struct {
+		ID              string
+		Name            string
+		ContentType     string
+		ContentLength   int64
+		BucketName      string
+		ObjectKey       string
+		ContainsContent bool
+		CreatedAt       time.Time
+	}
+	err = dbPool.QueryRow(ctx,
+		`SELECT id, name, content_type, content_length, bucket_name, object_key, contains_content, created_at
+		 FROM depot_objects WHERE id = $1`,
+		id,
+	).Scan(&obj.ID, &obj.Name, &obj.ContentType, &obj.ContentLength,
+		&obj.BucketName, &obj.ObjectKey, &obj.ContainsContent, &obj.CreatedAt)
+	assert.NoError(t, err)
+	assert.Equal(t, "test-file.txt", obj.Name)
+	assert.Equal(t, "text/plain", obj.ContentType)
+	assert.Equal(t, int64(1024), obj.ContentLength)
+	assert.Equal(t, "test-bucket", obj.BucketName)
+	assert.Equal(t, "test-key-123", obj.ObjectKey)
+	assert.False(t, obj.ContainsContent)
+	assert.False(t, obj.CreatedAt.IsZero())
+
+	// Clean up
+	_, err = dbPool.Exec(ctx, `DELETE FROM depot_objects WHERE id = $1`, id)
+	assert.NoError(t, err)
+}
+
+func TestDepotRepository_ConfirmUpload(t *testing.T) {
+	ctx := context.Background()
+
+	// Create a depot object
+	var id string
+	err := dbPool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id`,
+		"dpo_confirm123", "confirm-file.txt", "text/plain", int64(2048), "test-bucket", "confirm-key", false,
+	).Scan(&id)
+	assert.NoError(t, err)
+
+	// Verify it starts with contains_content = false
+	var containsContent bool
+	err = dbPool.QueryRow(ctx,
+		`SELECT contains_content FROM depot_objects WHERE id = $1`,
+		id,
+	).Scan(&containsContent)
+	assert.NoError(t, err)
+	assert.False(t, containsContent)
+
+	// Confirm upload
+	_, err = dbPool.Exec(ctx,
+		`UPDATE depot_objects SET contains_content = TRUE WHERE id = $1`,
+		id,
+	)
+	assert.NoError(t, err)
+
+	// Verify it's now true
+	err = dbPool.QueryRow(ctx,
+		`SELECT contains_content FROM depot_objects WHERE id = $1`,
+		id,
+	).Scan(&containsContent)
+	assert.NoError(t, err)
+	assert.True(t, containsContent)
+
+	// Clean up
+	_, err = dbPool.Exec(ctx, `DELETE FROM depot_objects WHERE id = $1`, id)
+	assert.NoError(t, err)
+}
+
+func TestDepotRepository_Delete(t *testing.T) {
+	ctx := context.Background()
+
+	// Create a depot object
+	var id string
+	err := dbPool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id`,
+		"dpo_delete123", "delete-file.txt", "text/plain", int64(512), "test-bucket", "delete-key", false,
+	).Scan(&id)
+	assert.NoError(t, err)
+
+	// Delete it
+	result, err := dbPool.Exec(ctx, `DELETE FROM depot_objects WHERE id = $1`, id)
+	assert.NoError(t, err)
+	assert.Equal(t, int64(1), result.RowsAffected())
+
+	// Verify it's gone
+	var count int
+	err = dbPool.QueryRow(ctx,
+		`SELECT COUNT(*) FROM depot_objects WHERE id = $1`,
+		id,
+	).Scan(&count)
+	assert.NoError(t, err)
+	assert.Equal(t, 0, count)
+}
+
+func TestDepotRepository_Exists(t *testing.T) {
+	ctx := context.Background()
+
+	// Create a depot object
+	var id string
+	err := dbPool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id`,
+		"dpo_exists123", "exists-file.txt", "text/plain", int64(256), "test-bucket", "exists-key", true,
+	).Scan(&id)
+	assert.NoError(t, err)
+
+	// Check exists
+	var exists bool
+	err = dbPool.QueryRow(ctx,
+		`SELECT EXISTS(SELECT 1 FROM depot_objects WHERE id = $1)`,
+		id,
+	).Scan(&exists)
+	assert.NoError(t, err)
+	assert.True(t, exists)
+
+	// Check non-existent
+	err = dbPool.QueryRow(ctx,
+		`SELECT EXISTS(SELECT 1 FROM depot_objects WHERE id = $1)`,
+		"dpo_nonexistent",
+	).Scan(&exists)
+	assert.NoError(t, err)
+	assert.False(t, exists)
+
+	// Clean up
+	_, err = dbPool.Exec(ctx, `DELETE FROM depot_objects WHERE id = $1`, id)
+	assert.NoError(t, err)
+}
+
+func TestDepotRepository_OrphanedIndex(t *testing.T) {
+	ctx := context.Background()
+
+	// Create an orphaned object (contains_content = false)
+	var orphanID string
+	err := dbPool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id`,
+		"dpo_orphan123", "orphan-file.txt", "text/plain", int64(128), "test-bucket", "orphan-key", false,
+	).Scan(&orphanID)
+	assert.NoError(t, err)
+
+	// Create a confirmed object (contains_content = true)
+	var confirmedID string
+	err = dbPool.QueryRow(ctx,
+		`INSERT INTO depot_objects (id, name, content_type, content_length, bucket_name, object_key, contains_content)
+		 VALUES ($1, $2, $3, $4, $5, $6, $7)
+		 RETURNING id`,
+		"dpo_confirmed123", "confirmed-file.txt", "text/plain", int64(128), "test-bucket", "confirmed-key", true,
+	).Scan(&confirmedID)
+	assert.NoError(t, err)
+
+	// Query orphaned objects using the index
+	rows, err := dbPool.Query(ctx,
+		`SELECT id FROM depot_objects WHERE contains_content = FALSE`)
+	assert.NoError(t, err)
+	defer rows.Close()
+
+	var orphanedIDs []string
+	for rows.Next() {
+		var id string
+		err := rows.Scan(&id)
+		assert.NoError(t, err)
+		orphanedIDs = append(orphanedIDs, id)
+	}
+
+	// Our orphan should be in the list
+	assert.Contains(t, orphanedIDs, orphanID)
+	assert.NotContains(t, orphanedIDs, confirmedID)
+
+	// Clean up
+	_, err = dbPool.Exec(ctx, `DELETE FROM depot_objects WHERE id IN ($1, $2)`, orphanID, confirmedID)
+	assert.NoError(t, err)
+}
+
+// TestDepotModels tests the model structures
+func TestDepotModels(t *testing.T) {
+	// Test Config defaults
+	config := depot.Config{
+		BucketName: "test-bucket",
+	}
+	assert.Equal(t, "test-bucket", config.BucketName)
+	assert.Equal(t, time.Duration(0), config.UploadURLExpiry)
+	assert.Equal(t, time.Duration(0), config.DownloadURLExpiry)
+
+	// Test with explicit values
+	config = depot.Config{
+		BucketName:        "custom-bucket",
+		UploadURLExpiry:   10 * time.Minute,
+		DownloadURLExpiry: 12 * time.Hour,
+	}
+	assert.Equal(t, "custom-bucket", config.BucketName)
+	assert.Equal(t, 10*time.Minute, config.UploadURLExpiry)
+	assert.Equal(t, 12*time.Hour, config.DownloadURLExpiry)
+}
+
+// TestDepotErrors tests the error definitions
+func TestDepotErrors(t *testing.T) {
+	assert.Error(t, depot.ErrNotFound)
+	assert.Error(t, depot.ErrInvalidInput)
+	assert.Equal(t, "object not found", depot.ErrNotFound.Error())
+	assert.Equal(t, "invalid input", depot.ErrInvalidInput.Error())
+}