adding more functions to media managament system

internal/store/postgres/media_hard_retention_integration_test.go

@@ -0,0 +1,105 @@
+package postgres
+
+import (
+	"context"
+	"strconv"
+	"testing"
+	"time"
+
+	"telesrv/internal/domain"
+)
+
+// TestHardRetentionPurgesBlobBytesButKeepsMetadataRow exercises the "hard"
+// storage retention mode's store methods end-to-end against a real
+// Postgres: a document old enough (by created_at, not orphaned_at) is a
+// candidate for ListDocumentIDsForHardRetentionOlderThan REGARDLESS of
+// still having a live media_references row, DeleteFileBlobsForDocument
+// removes only its file_blobs row (never the documents row itself), and a
+// second sweep pass no longer finds it a candidate since it no longer owns
+// any file_blobs row. This is the correctness property the whole "hard"
+// mode design hinges on: a message must still be able to render its media
+// placeholder after the bytes are gone.
+func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
+	pool := testPool(t)
+	ctx := context.Background()
+	media := NewMediaStore(pool)
+
+	docID := time.Now().UnixNano()
+	locationKey := "doc:" + strconv.FormatInt(docID, 10)
+
+	if err := media.PutDocument(ctx, domain.Document{
+		ID:       docID,
+		MimeType: "application/octet-stream",
+		Size:     1024,
+	}); err != nil {
+		t.Fatalf("PutDocument: %v", err)
+	}
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
+		_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
+	})
+
+	// Backdate created_at well past any plausible test cutoff -- PutDocument
+	// always stamps now() and has no created_at parameter.
+	if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days' WHERE id = $1", docID); err != nil {
+		t.Fatalf("backdate document: %v", err)
+	}
+
+	blob := postgresTestBlob(locationKey, "hard-retention-doc", 1024, "application/octet-stream")
+	if err := media.PutFileBlob(ctx, blob); err != nil {
+		t.Fatalf("PutFileBlob: %v", err)
+	}
+
+	// A LIVE reference (as if a message still embeds this document) must not
+	// exempt it from "hard" mode -- that's the entire point of the mode,
+	// unlike the orphan-only sweep.
+	if err := addMediaReferencesTx(ctx, pool, &domain.MessageMedia{
+		Kind:     domain.MessageMediaKindDocument,
+		Document: &domain.Document{ID: docID},
+	}, domain.MediaRefKindMessageBox, "hard-retention-test:1"); err != nil {
+		t.Fatalf("add media reference: %v", err)
+	}
+	t.Cleanup(func() {
+		_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE ref_key = 'hard-retention-test:1'")
+	})
+
+	cutoff := time.Now().Add(-24 * time.Hour)
+	ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
+	if err != nil {
+		t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
+	}
+	if !containsInt64(ids, docID) {
+		t.Fatalf("hard retention candidates = %v, want to include still-referenced but old document %d", ids, docID)
+	}
+
+	blobs, err := media.DeleteFileBlobsForDocument(ctx, docID)
+	if err != nil {
+		t.Fatalf("DeleteFileBlobsForDocument: %v", err)
+	}
+	if len(blobs) != 1 || blobs[0].LocationKey != locationKey {
+		t.Fatalf("deleted blobs = %+v, want exactly one for %q", blobs, locationKey)
+	}
+
+	// The documents row itself must survive -- a message referencing it
+	// still needs to render its placeholder (mime type, size, filename).
+	if _, found, err := media.GetDocument(ctx, docID); err != nil || !found {
+		t.Fatalf("document metadata row missing after hard blob purge: found=%v err=%v", found, err)
+	}
+
+	// The file_blobs row is gone: a subsequent download attempt resolves via
+	// GetFileBlob (files.Service.GetFile's path) to not-found, which the rpc
+	// layer already maps to LOCATION_INVALID.
+	if _, found, err := media.GetFileBlob(ctx, locationKey); err != nil || found {
+		t.Fatalf("file_blobs row still present after hard purge: found=%v err=%v", found, err)
+	}
+
+	// Idempotent / self-terminating: a second sweep pass no longer selects
+	// this document, since it no longer owns any file_blobs row.
+	ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
+	if err != nil {
+		t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
+	}
+	if containsInt64(ids, docID) {
+		t.Fatalf("hard retention re-selected already-purged document %d", docID)
+	}
+}

internal/store/postgres/media_refs.go

@@ -197,6 +197,101 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
 	return blobs, nil
 }
 
+// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
+// cutoff (by upload/created_at) that still own at least one file_blobs row,
+// oldest first, up to limit -- the "hard" retention sweep's candidate list.
+// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
+// entirely: a document still referenced by a live message is exactly as
+// eligible as an orphaned one.
+func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
+	if limit <= 0 {
+		return nil, nil
+	}
+	return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
+		Cutoff:     pgtype.Timestamptz{Time: cutoff, Valid: true},
+		BatchLimit: int32(limit),
+	})
+}
+
+// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
+// ListDocumentIDsForHardRetentionOlderThan.
+func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
+	if limit <= 0 {
+		return nil, nil
+	}
+	return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
+		Cutoff:     pgtype.Timestamptz{Time: cutoff, Valid: true},
+		BatchLimit: int32(limit),
+	})
+}
+
+// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
+// (main body + thumbnail variants), returning what was deleted so the caller
+// can physically remove each object from its backend once confirming (via
+// CountFileBlobRefs, after this call) no other row still needs it. Unlike
+// DeleteDocumentAndBlobs, this deliberately does NOT delete the documents
+// row itself -- "hard" retention mode keeps the metadata (dimensions, mime
+// type, filename) so a message can still render "this document is no longer
+// available" instead of disappearing outright. A subsequent
+// upload.getFile/GetFileBlob lookup for a location key this call removed
+// correctly finds nothing and reports not-found.
+func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) {
+	var blobs []domain.FileBlob
+	err := withTx(ctx, s.db, "delete document blob bytes (hard retention)", func(tx pgx.Tx) error {
+		qtx := s.q.WithTx(tx)
+		rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
+			ExactKey:      fmt.Sprintf("doc:%d", id),
+			PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
+		})
+		if err != nil {
+			return fmt.Errorf("list document blobs: %w", err)
+		}
+		for _, r := range rows {
+			blobs = append(blobs, domain.FileBlob{
+				LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
+			})
+			if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
+				return fmt.Errorf("delete file blob row: %w", err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return blobs, nil
+}
+
+// DeleteFileBlobsForPhoto is the photo counterpart of
+// DeleteFileBlobsForDocument -- see its doc comment. Deliberately does not
+// delete the photos row.
+func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) {
+	var blobs []domain.FileBlob
+	err := withTx(ctx, s.db, "delete photo blob bytes (hard retention)", func(tx pgx.Tx) error {
+		qtx := s.q.WithTx(tx)
+		rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
+			ExactKey:      fmt.Sprintf("photo:%d", id),
+			PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
+		})
+		if err != nil {
+			return fmt.Errorf("list photo blobs: %w", err)
+		}
+		for _, r := range rows {
+			blobs = append(blobs, domain.FileBlob{
+				LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
+			})
+			if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
+				return fmt.Errorf("delete file blob row: %w", err)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+	return blobs, nil
+}
+
 // DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
 // (one per rendition size), returning what was deleted so the caller can
 // physically remove each object from its backend once confirming (via

internal/store/postgres/queries/media.sql

@@ -237,6 +237,37 @@ WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
 ORDER BY orphaned_at ASC
 LIMIT sqlc.arg(batch_limit)::int;
 
+-- name: ListDocumentIDsForHardRetentionOlderThan :many
+-- "Hard" retention mode: candidates are documents older than cutoff (by
+-- upload/created_at, NOT orphaned_at -- a live reference does not exempt
+-- them) that still own at least one file_blobs row. The EXISTS check is what
+-- keeps this sweep from re-selecting the same document forever: once its
+-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
+-- rows but deliberately leaves this documents row in place), it naturally
+-- drops out of this query on the next pass.
+SELECT d.id FROM documents d
+WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
+  AND EXISTS (
+    SELECT 1 FROM file_blobs fb
+    WHERE fb.location_key = 'doc:' || d.id::text
+       OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
+  )
+ORDER BY d.created_at ASC
+LIMIT sqlc.arg(batch_limit)::int;
+
+-- name: ListPhotoIDsForHardRetentionOlderThan :many
+-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
+-- candidate selection, for photos.
+SELECT p.id FROM photos p
+WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
+  AND EXISTS (
+    SELECT 1 FROM file_blobs fb
+    WHERE fb.location_key = 'photo:' || p.id::text
+       OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
+  )
+ORDER BY p.created_at ASC
+LIMIT sqlc.arg(batch_limit)::int;
+
 -- name: CountFileBlobRefs :one
 SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
 

internal/store/postgres/sqlcgen/media.sql.go

@@ -825,6 +825,50 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
 	return items, nil
 }
 
+const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
+SELECT d.id FROM documents d
+WHERE d.created_at < $1::timestamptz
+  AND EXISTS (
+    SELECT 1 FROM file_blobs fb
+    WHERE fb.location_key = 'doc:' || d.id::text
+       OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
+  )
+ORDER BY d.created_at ASC
+LIMIT $2::int
+`
+
+type ListDocumentIDsForHardRetentionOlderThanParams struct {
+	Cutoff     pgtype.Timestamptz
+	BatchLimit int32
+}
+
+// "Hard" retention mode: candidates are documents older than cutoff (by
+// upload/created_at, NOT orphaned_at -- a live reference does not exempt
+// them) that still own at least one file_blobs row. The EXISTS check is what
+// keeps this sweep from re-selecting the same document forever: once its
+// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
+// rows but deliberately leaves this documents row in place), it naturally
+// drops out of this query on the next pass.
+func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
+	rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		items = append(items, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
 SELECT location_key, backend, object_key, size
 FROM file_blobs
@@ -937,6 +981,45 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
 	return items, nil
 }
 
+const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
+SELECT p.id FROM photos p
+WHERE p.created_at < $1::timestamptz
+  AND EXISTS (
+    SELECT 1 FROM file_blobs fb
+    WHERE fb.location_key = 'photo:' || p.id::text
+       OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
+  )
+ORDER BY p.created_at ASC
+LIMIT $2::int
+`
+
+type ListPhotoIDsForHardRetentionOlderThanParams struct {
+	Cutoff     pgtype.Timestamptz
+	BatchLimit int32
+}
+
+// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
+// candidate selection, for photos.
+func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
+	rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var items []int64
+	for rows.Next() {
+		var id int64
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		items = append(items, id)
+	}
+	if err := rows.Err(); err != nil {
+		return nil, err
+	}
+	return items, nil
+}
+
 const listProfilePhotos = `-- name: ListProfilePhotos :many
 SELECT photo_id
 FROM profile_photos