fix for retention logic
This commit is contained in:
onysd
parent
70c0ba44f0
commit
e6bfe2d444
35 changed files with 2108 additions and 132 deletions
|
|
@ -3,6 +3,7 @@ package files
|
|||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
|
@ -10,14 +11,52 @@ import (
|
|||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// hardRetentionDocumentCategories enumerates every documents.category bucket
|
||||
// the per-category retention sweep loops over. MediaCategoryNone covers
|
||||
// unclassified documents (stickers and anything else classifyDocumentCategory
|
||||
// doesn't tag) and always uses the shared global age -- there is no
|
||||
// per-category override for that bucket. Photo/Avatar are not in this list:
|
||||
// photos have no category column and are instead split by a profile_photos
|
||||
// join (see categoryRetentionAge/avatarRetentionAge below and the dedicated
|
||||
// photo/avatar query variants).
|
||||
var hardRetentionDocumentCategories = []domain.MediaCategory{
|
||||
domain.MediaCategoryNone,
|
||||
domain.MediaCategoryVideo,
|
||||
domain.MediaCategoryGif,
|
||||
domain.MediaCategoryFile,
|
||||
domain.MediaCategoryMusic,
|
||||
domain.MediaCategoryVoice,
|
||||
domain.MediaCategoryRoundVideo,
|
||||
}
|
||||
|
||||
// categoryRetentionAge returns the effective retention age for a document
|
||||
// media category: its configured override if positive, otherwise the shared
|
||||
// global age.
|
||||
func (s *Service) categoryRetentionAge(category domain.MediaCategory) time.Duration {
|
||||
if age, ok := s.storageRetentionCategoryAges[category]; ok && age > 0 {
|
||||
return age
|
||||
}
|
||||
return s.storageRetentionGlobalMaxAge
|
||||
}
|
||||
|
||||
// avatarRetentionAge is the Photo category's counterpart for photos currently
|
||||
// active as someone's avatar -- see categoryRetentionAge.
|
||||
func (s *Service) avatarRetentionAge() time.Duration {
|
||||
if s.storageRetentionAvatarMaxAge > 0 {
|
||||
return s.storageRetentionAvatarMaxAge
|
||||
}
|
||||
return s.storageRetentionGlobalMaxAge
|
||||
}
|
||||
|
||||
// mediaRetentionStore is implemented by store.MediaStore backends that
|
||||
// support the storage retention sweep (currently only the Postgres store).
|
||||
// A type assertion, not a MediaStore interface method, keeps these
|
||||
// admin/maintenance-only queries out of the hot RPC-facing interface --
|
||||
// same convention as photoBatchStore above.
|
||||
type mediaRetentionStore interface {
|
||||
ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
|
||||
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
|
|
@ -31,111 +70,250 @@ type mediaRetentionStore interface {
|
|||
// orphaned one once it's old enough. The delete methods physically
|
||||
// remove only the file_blobs row(s)/bytes, never the document/photo
|
||||
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
|
||||
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
|
||||
// -- active eviction (TELESRV_STORAGE_EVICTION_ENABLE) --
|
||||
SumFileBlobBytes(ctx context.Context) (int64, error)
|
||||
ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error)
|
||||
|
||||
// -- retention purge notice (see retention_purge.go) --
|
||||
ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error)
|
||||
}
|
||||
|
||||
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
||||
// permanently deletes documents/photos that have had no live reference
|
||||
// (message/profile-photo/sticker-set, see media_references) since at least
|
||||
// cutoff, along with their blob(s) -- but only physically removes bytes
|
||||
// from the backend once confirming no other file_blobs row still needs the
|
||||
// object, since content-addressed storage means the same bytes can be
|
||||
// shared across documents/photos.
|
||||
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
||||
// the per-category cutoff derived from now, along with their blob(s) -- but
|
||||
// only physically removes bytes from the backend once confirming no other
|
||||
// file_blobs row still needs the object, since content-addressed storage
|
||||
// means the same bytes can be shared across documents/photos. Loops one
|
||||
// query per document category (each with its own effective age, see
|
||||
// categoryRetentionAge) plus a regular/avatar split for photos; limit applies
|
||||
// per category per tick, not as one shared budget across all of them --
|
||||
// simplest correct behavior, revisit only if one category starves another in
|
||||
// practice.
|
||||
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
|
||||
store, ok := s.media.(mediaRetentionStore)
|
||||
if !ok || limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
deleted := 0
|
||||
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned documents: %w", err)
|
||||
}
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
for _, cat := range hardRetentionDocumentCategories {
|
||||
age := s.categoryRetentionAge(cat)
|
||||
if age <= 0 {
|
||||
// Global age is 0 (retention only enabled for other, explicitly
|
||||
// overridden categories) and this category has no override of its
|
||||
// own -- skip it entirely rather than treating age<=0 as an
|
||||
// immediate "everything is older than now" cutoff.
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned photos: %w", err)
|
||||
}
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
|
||||
cutoff := now.Add(-age)
|
||||
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cat, cutoff, limit)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
return deleted, fmt.Errorf("list orphaned documents (category %d): %w", cat, err)
|
||||
}
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
|
||||
photoCutoff := now.Add(-age)
|
||||
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, photoCutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned photos: %w", err)
|
||||
}
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
}
|
||||
if age := s.avatarRetentionAge(); age > 0 {
|
||||
avatarCutoff := now.Add(-age)
|
||||
avatarIDs, err := store.ListAvatarOrphanedPhotoIDsOlderThan(ctx, avatarCutoff, limit)
|
||||
if err != nil {
|
||||
return deleted, fmt.Errorf("list orphaned avatar photos: %w", err)
|
||||
}
|
||||
for _, id := range avatarIDs {
|
||||
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("delete orphaned avatar photo failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
deleted++
|
||||
}
|
||||
return deleted, nil
|
||||
}
|
||||
|
||||
// DeleteBlobBytesForMediaOlderThan implements
|
||||
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
|
||||
// documents/photos whose upload/created_at is older than cutoff, physically
|
||||
// deletes their blob bytes (main body + thumbnail/rendition variants) from
|
||||
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
|
||||
// live message/profile-photo/sticker-set still references them. It
|
||||
// deliberately never touches the documents/photos metadata row itself: a
|
||||
// message must still be able to render "here was a photo/document"
|
||||
// (dimensions, mime type, filename) after its bytes are gone, rather than
|
||||
// the message breaking outright. A subsequent upload.getFile for the same
|
||||
// location key finds no file_blobs row and returns LOCATION_INVALID, which
|
||||
// stock clients already render as a "media unavailable" placeholder.
|
||||
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
||||
// documents/photos whose upload/created_at is older than the per-category
|
||||
// cutoff derived from now (see categoryRetentionAge/avatarRetentionAge),
|
||||
// physically deletes their blob bytes (main body + thumbnail/rendition
|
||||
// variants) from the backend and removes their file_blobs rows --
|
||||
// REGARDLESS of whether a live message/profile-photo/sticker-set still
|
||||
// references them. It deliberately never touches the documents/photos
|
||||
// metadata row itself: a message must still be able to render "here was a
|
||||
// photo/document" (dimensions, mime type, filename) after its bytes are
|
||||
// gone, rather than the message breaking outright. A subsequent
|
||||
// upload.getFile for the same location key finds no file_blobs row and
|
||||
// returns LOCATION_INVALID, which stock clients already render as a "media
|
||||
// unavailable" placeholder -- and every message that still embeds the purged
|
||||
// media additionally gets turned into a visible retention-purge notice (see
|
||||
// notifyRetentionPurge in retention_purge.go).
|
||||
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
|
||||
store, ok := s.media.(mediaRetentionStore)
|
||||
if !ok || limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
purged := 0
|
||||
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return purged, fmt.Errorf("list documents for hard retention: %w", err)
|
||||
}
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
|
||||
for _, cat := range hardRetentionDocumentCategories {
|
||||
age := s.categoryRetentionAge(cat)
|
||||
if age <= 0 {
|
||||
// See DeleteOrphanedOlderThan's identical guard: age<=0 means this
|
||||
// category has no override and the global age is 0 (retention
|
||||
// enabled only for other categories) -- skip, don't treat it as
|
||||
// "everything is older than now".
|
||||
continue
|
||||
}
|
||||
cutoff := now.Add(-age)
|
||||
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
continue
|
||||
return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
|
||||
purged++
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
purged++
|
||||
}
|
||||
remaining := limit - len(docIDs)
|
||||
if remaining <= 0 {
|
||||
return purged, nil
|
||||
}
|
||||
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
|
||||
if err != nil {
|
||||
return purged, fmt.Errorf("list photos for hard retention: %w", err)
|
||||
}
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
|
||||
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
|
||||
photoCutoff := now.Add(-age)
|
||||
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
return purged, fmt.Errorf("list photos for hard retention: %w", err)
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
|
||||
purged++
|
||||
}
|
||||
}
|
||||
if age := s.avatarRetentionAge(); age > 0 {
|
||||
avatarCutoff := now.Add(-age)
|
||||
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit)
|
||||
if err != nil {
|
||||
return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
|
||||
}
|
||||
for _, id := range avatarIDs {
|
||||
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete avatar photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
|
||||
purged++
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
purged++
|
||||
}
|
||||
return purged, nil
|
||||
}
|
||||
|
||||
// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
|
||||
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
|
||||
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the
|
||||
// oldest documents/photos overall -- interleaved by created_at across both
|
||||
// tables, regardless of category or age -- reusing the exact same blob-purge
|
||||
// (DeleteFileBlobsForDocument/DeleteFileBlobsForPhoto) and retention-purge
|
||||
// notice primitive as "hard" mode. Stops once the running total (tracked
|
||||
// locally from each purge's returned blob sizes, avoiding a re-query per
|
||||
// item) is back under budget, or limit purges have happened this tick,
|
||||
// whichever comes first -- bounding how much one tick can reclaim at once.
|
||||
func (s *Service) EvictOldestMediaOverBudget(ctx context.Context, limit int) (int, error) {
|
||||
store, ok := s.media.(mediaRetentionStore)
|
||||
if !ok || limit <= 0 || s.storageMaxTotalBytes <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
total, err := store.SumFileBlobBytes(ctx)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("sum file blob bytes: %w", err)
|
||||
}
|
||||
if total <= s.storageMaxTotalBytes {
|
||||
return 0, nil
|
||||
}
|
||||
candidates, err := store.ListOldestMediaForEviction(ctx, limit)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("list oldest media for eviction: %w", err)
|
||||
}
|
||||
sort.Slice(candidates, func(i, j int) bool { return candidates[i].CreatedAt.Before(candidates[j].CreatedAt) })
|
||||
evicted := 0
|
||||
for _, c := range candidates {
|
||||
if evicted >= limit || total <= s.storageMaxTotalBytes {
|
||||
break
|
||||
}
|
||||
var blobs []domain.FileBlob
|
||||
var delErr error
|
||||
switch c.Kind {
|
||||
case domain.MediaKindDocument:
|
||||
blobs, delErr = store.DeleteFileBlobsForDocument(ctx, c.MediaID)
|
||||
case domain.MediaKindPhoto:
|
||||
blobs, delErr = store.DeleteFileBlobsForPhoto(ctx, c.MediaID)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
if delErr != nil {
|
||||
s.log.Warn("active storage eviction blob purge failed",
|
||||
zap.String("media_kind", string(c.Kind)), zap.Int64("media_id", c.MediaID), zap.Error(delErr))
|
||||
continue
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
for _, b := range blobs {
|
||||
total -= b.Size
|
||||
}
|
||||
s.notifyRetentionPurge(ctx, c.Kind, c.MediaID)
|
||||
evicted++
|
||||
}
|
||||
return evicted, nil
|
||||
}
|
||||
|
||||
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
|
||||
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
|
||||
// id right now (skipping the grace period) and, only if that succeeds --
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue