owpengram-server/internal/app/files/retention.go

package files

import (
	"context"
	"fmt"
	"sort"
	"time"

	"go.uber.org/zap"

	"telesrv/internal/domain"
)

// hardRetentionDocumentCategories enumerates every documents.category bucket
// the per-category retention sweep loops over. MediaCategoryNone covers
// unclassified documents (stickers and anything else classifyDocumentCategory
// doesn't tag) and always uses the shared global age -- there is no
// per-category override for that bucket. Photo/Avatar are not in this list:
// photos have no category column and are instead split by a profile_photos
// join (see categoryRetentionAge/avatarRetentionAge below and the dedicated
// photo/avatar query variants).
var hardRetentionDocumentCategories = []domain.MediaCategory{
	domain.MediaCategoryNone,
	domain.MediaCategoryVideo,
	domain.MediaCategoryGif,
	domain.MediaCategoryFile,
	domain.MediaCategoryMusic,
	domain.MediaCategoryVoice,
	domain.MediaCategoryRoundVideo,
}

// categoryRetentionAge returns the effective retention age for a document
// media category: its configured override if positive, otherwise the shared
// global age.
func (s *Service) categoryRetentionAge(category domain.MediaCategory) time.Duration {
	if age, ok := s.storageRetentionCategoryAges[category]; ok && age > 0 {
		return age
	}
	return s.storageRetentionGlobalMaxAge
}

// avatarRetentionAge is the Photo category's counterpart for photos currently
// active as someone's avatar -- see categoryRetentionAge.
func (s *Service) avatarRetentionAge() time.Duration {
	if s.storageRetentionAvatarMaxAge > 0 {
		return s.storageRetentionAvatarMaxAge
	}
	return s.storageRetentionGlobalMaxAge
}

// mediaRetentionStore is implemented by store.MediaStore backends that
// support the storage retention sweep (currently only the Postgres store).
// A type assertion, not a MediaStore interface method, keeps these
// admin/maintenance-only queries out of the hot RPC-facing interface --
// same convention as photoBatchStore above.
type mediaRetentionStore interface {
	ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
	ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
	ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
	CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
	DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
	DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
	// OrphanDocumentIfUnreferenced is the immediate (no grace period)
	// counterpart to the age-based sweep above -- see its doc comment.
	OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)

	// -- "hard" retention mode (TELESRV_STORAGE_RETENTION_MODE=hard) --
	// Candidate selection ignores media_references entirely: a document/
	// photo still referenced by a live message is exactly as eligible as an
	// orphaned one once it's old enough. The delete methods physically
	// remove only the file_blobs row(s)/bytes, never the document/photo
	// metadata row -- see DeleteFileBlobsForDocument's doc comment.
	ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
	ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
	ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
	DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
	DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)

	// -- active eviction (TELESRV_STORAGE_EVICTION_ENABLE) --
	SumFileBlobBytes(ctx context.Context) (int64, error)
	ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error)

	// -- retention purge notice (see retention_purge.go) --
	ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error)
}

// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
// permanently deletes documents/photos that have had no live reference
// (message/profile-photo/sticker-set, see media_references) since at least
// the per-category cutoff derived from now, along with their blob(s) -- but
// only physically removes bytes from the backend once confirming no other
// file_blobs row still needs the object, since content-addressed storage
// means the same bytes can be shared across documents/photos. Loops one
// query per document category (each with its own effective age, see
// categoryRetentionAge) plus a regular/avatar split for photos; limit applies
// per category per tick, not as one shared budget across all of them --
// simplest correct behavior, revisit only if one category starves another in
// practice.
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
	store, ok := s.media.(mediaRetentionStore)
	if !ok || limit <= 0 {
		return 0, nil
	}
	deleted := 0
	for _, cat := range hardRetentionDocumentCategories {
		age := s.categoryRetentionAge(cat)
		if age <= 0 {
			// Global age is 0 (retention only enabled for other, explicitly
			// overridden categories) and this category has no override of its
			// own -- skip it entirely rather than treating age<=0 as an
			// immediate "everything is older than now" cutoff.
			continue
		}
		cutoff := now.Add(-age)
		docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cat, cutoff, limit)
		if err != nil {
			return deleted, fmt.Errorf("list orphaned documents (category %d): %w", cat, err)
		}
		for _, id := range docIDs {
			blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
			if err != nil {
				s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
				continue
			}
			s.deleteOrphanedBlobs(ctx, store, blobs)
			deleted++
		}
	}
	if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
		photoCutoff := now.Add(-age)
		photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, photoCutoff, limit)
		if err != nil {
			return deleted, fmt.Errorf("list orphaned photos: %w", err)
		}
		for _, id := range photoIDs {
			blobs, err := store.DeletePhotoAndBlobs(ctx, id)
			if err != nil {
				s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
				continue
			}
			s.deleteOrphanedBlobs(ctx, store, blobs)
			deleted++
		}
	}
	if age := s.avatarRetentionAge(); age > 0 {
		avatarCutoff := now.Add(-age)
		avatarIDs, err := store.ListAvatarOrphanedPhotoIDsOlderThan(ctx, avatarCutoff, limit)
		if err != nil {
			return deleted, fmt.Errorf("list orphaned avatar photos: %w", err)
		}
		for _, id := range avatarIDs {
			blobs, err := store.DeletePhotoAndBlobs(ctx, id)
			if err != nil {
				s.log.Warn("delete orphaned avatar photo failed", zap.Int64("photo_id", id), zap.Error(err))
				continue
			}
			s.deleteOrphanedBlobs(ctx, store, blobs)
			deleted++
		}
	}
	return deleted, nil
}

// DeleteBlobBytesForMediaOlderThan implements
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
// documents/photos whose upload/created_at is older than the per-category
// cutoff derived from now (see categoryRetentionAge/avatarRetentionAge),
// physically deletes their blob bytes (main body + thumbnail/rendition
// variants) from the backend and removes their file_blobs rows --
// REGARDLESS of whether a live message/profile-photo/sticker-set still
// references them. It deliberately never touches the documents/photos
// metadata row itself: a message must still be able to render "here was a
// photo/document" (dimensions, mime type, filename) after its bytes are
// gone, rather than the message breaking outright. A subsequent
// upload.getFile for the same location key finds no file_blobs row and
// returns LOCATION_INVALID, which stock clients already render as a "media
// unavailable" placeholder -- and every message that still embeds the purged
// media additionally gets turned into a visible retention-purge notice (see
// notifyRetentionPurge in retention_purge.go).
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
	store, ok := s.media.(mediaRetentionStore)
	if !ok || limit <= 0 {
		return 0, nil
	}
	purged := 0
	for _, cat := range hardRetentionDocumentCategories {
		age := s.categoryRetentionAge(cat)
		if age <= 0 {
			// See DeleteOrphanedOlderThan's identical guard: age<=0 means this
			// category has no override and the global age is 0 (retention
			// enabled only for other categories) -- skip, don't treat it as
			// "everything is older than now".
			continue
		}
		cutoff := now.Add(-age)
		docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit)
		if err != nil {
			return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
		}
		for _, id := range docIDs {
			blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
			if err != nil {
				s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
				continue
			}
			if len(blobs) == 0 {
				continue
			}
			s.deleteOrphanedBlobs(ctx, store, blobs)
			s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
			purged++
		}
	}
	if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
		photoCutoff := now.Add(-age)
		photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit)
		if err != nil {
			return purged, fmt.Errorf("list photos for hard retention: %w", err)
		}
		for _, id := range photoIDs {
			blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
			if err != nil {
				s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
				continue
			}
			if len(blobs) == 0 {
				continue
			}
			s.deleteOrphanedBlobs(ctx, store, blobs)
			s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
			purged++
		}
	}
	if age := s.avatarRetentionAge(); age > 0 {
		avatarCutoff := now.Add(-age)
		avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit)
		if err != nil {
			return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
		}
		for _, id := range avatarIDs {
			blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
			if err != nil {
				s.log.Warn("hard-delete avatar photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
				continue
			}
			if len(blobs) == 0 {
				continue
			}
			s.deleteOrphanedBlobs(ctx, store, blobs)
			s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
			purged++
		}
	}
	return purged, nil
}

// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the
// oldest documents/photos overall -- interleaved by created_at across both
// tables, regardless of category or age -- reusing the exact same blob-purge
// (DeleteFileBlobsForDocument/DeleteFileBlobsForPhoto) and retention-purge
// notice primitive as "hard" mode. Stops once the running total (tracked
// locally from each purge's returned blob sizes, avoiding a re-query per
// item) is back under budget, or limit purges have happened this tick,
// whichever comes first -- bounding how much one tick can reclaim at once.
func (s *Service) EvictOldestMediaOverBudget(ctx context.Context, limit int) (int, error) {
	store, ok := s.media.(mediaRetentionStore)
	if !ok || limit <= 0 || s.storageMaxTotalBytes <= 0 {
		return 0, nil
	}
	total, err := store.SumFileBlobBytes(ctx)
	if err != nil {
		return 0, fmt.Errorf("sum file blob bytes: %w", err)
	}
	if total <= s.storageMaxTotalBytes {
		return 0, nil
	}
	candidates, err := store.ListOldestMediaForEviction(ctx, limit)
	if err != nil {
		return 0, fmt.Errorf("list oldest media for eviction: %w", err)
	}
	sort.Slice(candidates, func(i, j int) bool { return candidates[i].CreatedAt.Before(candidates[j].CreatedAt) })
	evicted := 0
	for _, c := range candidates {
		if evicted >= limit || total <= s.storageMaxTotalBytes {
			break
		}
		var blobs []domain.FileBlob
		var delErr error
		switch c.Kind {
		case domain.MediaKindDocument:
			blobs, delErr = store.DeleteFileBlobsForDocument(ctx, c.MediaID)
		case domain.MediaKindPhoto:
			blobs, delErr = store.DeleteFileBlobsForPhoto(ctx, c.MediaID)
		default:
			continue
		}
		if delErr != nil {
			s.log.Warn("active storage eviction blob purge failed",
				zap.String("media_kind", string(c.Kind)), zap.Int64("media_id", c.MediaID), zap.Error(delErr))
			continue
		}
		if len(blobs) == 0 {
			continue
		}
		s.deleteOrphanedBlobs(ctx, store, blobs)
		for _, b := range blobs {
			total -= b.Size
		}
		s.notifyRetentionPurge(ctx, c.Kind, c.MediaID)
		evicted++
	}
	return evicted, nil
}

// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
// id right now (skipping the grace period) and, only if that succeeds --
// i.e. nothing else currently references it -- physically deletes it and
// its blobs immediately. Returns whether it was actually deleted; false
// (with no error) means something still references the document, so it and
// its blob(s) were deliberately left alone.
func (s *Service) deleteDocumentNowIfUnreferenced(ctx context.Context, id int64) (bool, error) {
	store, ok := s.media.(mediaRetentionStore)
	if !ok {
		return false, nil
	}
	orphaned, err := store.OrphanDocumentIfUnreferenced(ctx, id)
	if err != nil {
		return false, fmt.Errorf("orphan document: %w", err)
	}
	if !orphaned {
		return false, nil
	}
	blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
	if err != nil {
		return false, fmt.Errorf("delete document: %w", err)
	}
	s.deleteOrphanedBlobs(ctx, store, blobs)
	return true, nil
}

// deleteOrphanedBlobs removes each blob from its backend once confirming
// (via CountFileBlobRefs) no other file_blobs row still references
// (backend, object_key). Resolves the correct backend per blob via
// backendFor (not just the currently active one) -- a blob written before a
// TELESRV_BLOB_BACKEND switch still needs deleting from wherever it
// actually lives. If that backend is no longer configured (its credentials
// were removed after switching away from it), the blob is logged and
// skipped rather than silently dropped, since there's nothing reachable to
// delete it from.
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
	for _, b := range blobs {
		// The file_blobs row for this exact location_key is already gone
		// (the caller deleted it in the same transaction that produced this
		// blob list) -- so any cached "found" metadata for it is now wrong
		// regardless of whether the underlying bytes turn out to still be
		// shared by another row below. Without this, a hot GetFile path that
		// had this location_key's metadata cached would keep trying to read
		// bytes that may no longer exist (hard retention mode purges blobs
		// for actively-referenced, potentially still-hot media), producing
		// an internal error instead of the graceful LOCATION_INVALID a stock
		// client knows how to render.
		s.blobCache.delete(b.LocationKey)
		refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
		if err != nil {
			s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
			continue
		}
		if refs > 0 {
			continue
		}
		backend, err := s.backendFor(b.Backend)
		if err != nil {
			s.log.Warn("orphaned blob's backend is not configured, skipping physical delete",
				zap.String("backend", string(b.Backend)), zap.String("object_key", b.ObjectKey), zap.Error(err))
			continue
		}
		if err := backend.Delete(ctx, b.ObjectKey); err != nil {
			s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
		}
		s.byteCache.delete(b.ObjectKey)
	}
}