fixes for storage managament

This commit is contained in:
onysd 2026-09-03 08:33:06 +03:00
parent e6bfe2d444
commit 8ef2b58bf9
29 changed files with 1768 additions and 63 deletions

View file

@ -69,10 +69,19 @@ type mediaRetentionStore interface {
// photo still referenced by a live message is exactly as eligible as an
// orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
// metadata row -- see DeleteFileBlobsForDocument's doc comment. cutoff is
// a *time.Time so it can be nil ("no age filter at all") -- the automatic
// sweep below always passes a real cutoff; the manual purge admin action
// (see ManualPurge) is what actually uses nil.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error)
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff *time.Time, limit int) ([]int64, error)
// Count* are the exact-count counterparts of the List* queries just
// above, used by CountManualPurgeCandidates's dry-run preview: a
// LIMIT-capped list length is not a real total.
CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error)
CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error)
CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
@ -193,7 +202,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
continue
}
cutoff := now.Add(-age)
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit)
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, &cutoff, limit)
if err != nil {
return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
}
@ -213,7 +222,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
}
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
photoCutoff := now.Add(-age)
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit)
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, &photoCutoff, limit)
if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
@ -233,7 +242,7 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
}
if age := s.avatarRetentionAge(); age > 0 {
avatarCutoff := now.Add(-age)
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit)
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, &avatarCutoff, limit)
if err != nil {
return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
}
@ -254,6 +263,169 @@ func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time
return purged, nil
}
// manualPurgeCategories enumerates the domain.MediaCategory values the
// manual purge admin action (ManualPurge/CountManualPurgeCandidates) accepts.
// Unlike hardRetentionDocumentCategories (used by the automatic sweep),
// this includes MediaCategoryPhoto -- an operator explicitly choosing
// categories to purge expects "Photo" to be one of the choices, even though
// photos have no documents.category column and are resolved through the
// dedicated ListPhotoIDsForHardRetentionOlderThan query instead. Avatar is
// deliberately NOT a domain.MediaCategory member and is instead its own
// includeAvatars bool parameter, matching how the automatic sweep already
// splits avatar photos out via avatarRetentionAge/ListAvatarPhotoIDsForHardRetentionOlderThan.
var manualPurgeCategories = map[domain.MediaCategory]bool{
domain.MediaCategoryPhoto: true,
domain.MediaCategoryVideo: true,
domain.MediaCategoryRoundVideo: true,
domain.MediaCategoryGif: true,
domain.MediaCategoryMusic: true,
domain.MediaCategoryVoice: true,
domain.MediaCategoryFile: true,
}
// validateManualPurgeCategories rejects any category outside
// manualPurgeCategories (e.g. MediaCategoryNone or MediaCategoryURL, which
// are not meaningful purge targets) rather than silently ignoring it -- an
// operator who fat-fingers a category deserves an error, not a purge that
// quietly did less than they asked for.
func validateManualPurgeCategories(categories []domain.MediaCategory) error {
for _, cat := range categories {
if !manualPurgeCategories[cat] {
return fmt.Errorf("unsupported manual purge category: %d", cat)
}
}
return nil
}
// CountManualPurgeCandidates is the dry-run counterpart of ManualPurge: an
// exact count of how many documents/photos the given selection would purge,
// without deleting anything. before may be nil, meaning no age filter at all
// (every document/photo in the selected categories is a candidate,
// regardless of created_at) -- unlike the automatic sweep, which always
// filters by a configured age.
func (s *Service) CountManualPurgeCandidates(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time) (docs int, photos int, err error) {
store, ok := s.media.(mediaRetentionStore)
if !ok {
return 0, 0, nil
}
if err := validateManualPurgeCategories(categories); err != nil {
return 0, 0, err
}
for _, cat := range categories {
if cat == domain.MediaCategoryPhoto {
n, err := store.CountPhotosForHardRetention(ctx, before)
if err != nil {
return docs, photos, fmt.Errorf("count photos for manual purge: %w", err)
}
photos += n
continue
}
n, err := store.CountDocumentsForHardRetention(ctx, cat, before)
if err != nil {
return docs, photos, fmt.Errorf("count documents for manual purge (category %d): %w", cat, err)
}
docs += n
}
if includeAvatars {
n, err := store.CountAvatarPhotosForHardRetention(ctx, before)
if err != nil {
return docs, photos, fmt.Errorf("count avatar photos for manual purge: %w", err)
}
photos += n
}
return docs, photos, nil
}
// ManualPurge is the admin-triggered counterpart of DeleteBlobBytesForMediaOlderThan
// ("hard" retention mode's blob-purge primitive): instead of config-derived
// categories and a configured retention age, the operator explicitly chooses
// which categories to purge and an optional cutoff date. before == nil means
// no age filter at all -- everything matching the chosen categories is
// purged, regardless of how recently it was created. Deletion semantics are
// identical to the automatic sweep: only file_blobs bytes are removed, never
// the documents/photos metadata row, and notifyRetentionPurge still turns any
// message still displaying the purged media into the retention-purge notice.
// limit bounds how many documents/photos are purged per category/bucket in
// this single call (the admin action loops/paginates by calling again if the
// dry-run count exceeds one call's limit).
func (s *Service) ManualPurge(ctx context.Context, categories []domain.MediaCategory, includeAvatars bool, before *time.Time, limit int) (purgedDocs int, purgedPhotos int, bytesReclaimed int64, err error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, 0, 0, nil
}
if err := validateManualPurgeCategories(categories); err != nil {
return 0, 0, 0, err
}
for _, cat := range categories {
if cat == domain.MediaCategoryPhoto {
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list photos for manual purge: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("manual purge photo blob delete failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purgedPhotos++
}
continue
}
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list documents for manual purge (category %d): %w", cat, err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("manual purge document blob delete failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
purgedDocs++
}
}
if includeAvatars {
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, before, limit)
if err != nil {
return purgedDocs, purgedPhotos, bytesReclaimed, fmt.Errorf("list avatar photos for manual purge: %w", err)
}
for _, id := range avatarIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("manual purge avatar photo blob delete failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
bytesReclaimed += b.Size
}
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purgedPhotos++
}
}
return purgedDocs, purgedPhotos, bytesReclaimed, nil
}
// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the

View file

@ -24,21 +24,31 @@ type fakeCategorySweepStore struct {
queriedAvatars bool
}
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ *time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil
}
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
f.queriedPhotos = true
return nil, nil
}
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
f.queriedAvatars = true
return nil, nil
}
func (f *fakeCategorySweepStore) CountDocumentsForHardRetention(context.Context, domain.MediaCategory, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) CountPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) CountAvatarPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) ListOrphanedDocumentIDsOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil

View file

@ -0,0 +1,314 @@
package files
import (
"context"
"fmt"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeManualPurgeStore is a mediaRetentionStore fake that mimics the SQL
// nullable-cutoff contract in Go: a nil cutoff matches every candidate
// regardless of its createdAt, a non-nil cutoff only matches candidates
// strictly older than it -- same semantics as the real
// ListDocumentIDsForHardRetentionOlderThan/ListPhotoIDsForHardRetentionOlderThan
// queries (see internal/store/postgres/queries/media.sql). Embeds a nil
// store.MediaStore so it satisfies that (large) interface for free, same
// trick as fakeCategorySweepStore in retention_category_zero_test.go.
type fakeManualPurgeStore struct {
store.MediaStore
docsByCategory map[domain.MediaCategory][]int64
docCreatedAt map[int64]time.Time
photos []int64
photoCreatedAt map[int64]time.Time
avatarPhotos []int64
avatarCreated map[int64]time.Time
deletedDocs []int64
deletedPhotos []int64
}
func matchCutoff(createdAt time.Time, cutoff *time.Time) bool {
if cutoff == nil {
return true
}
return createdAt.Before(*cutoff)
}
func (f *fakeManualPurgeStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.docsByCategory[category] {
if !matchCutoff(f.docCreatedAt[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountDocumentsForHardRetention(ctx context.Context, category domain.MediaCategory, cutoff *time.Time) (int, error) {
ids, err := f.ListDocumentIDsForHardRetentionOlderThan(ctx, category, cutoff, len(f.docsByCategory[category])+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) ListPhotoIDsForHardRetentionOlderThan(_ context.Context, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.photos {
if !matchCutoff(f.photoCreatedAt[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
ids, err := f.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, len(f.photos)+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) ListAvatarPhotoIDsForHardRetentionOlderThan(_ context.Context, cutoff *time.Time, limit int) ([]int64, error) {
var out []int64
for _, id := range f.avatarPhotos {
if !matchCutoff(f.avatarCreated[id], cutoff) {
continue
}
out = append(out, id)
if len(out) >= limit {
break
}
}
return out, nil
}
func (f *fakeManualPurgeStore) CountAvatarPhotosForHardRetention(ctx context.Context, cutoff *time.Time) (int, error) {
ids, err := f.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, cutoff, len(f.avatarPhotos)+1)
return len(ids), err
}
func (f *fakeManualPurgeStore) DeleteFileBlobsForDocument(_ context.Context, id int64) ([]domain.FileBlob, error) {
f.deletedDocs = append(f.deletedDocs, id)
return []domain.FileBlob{{
LocationKey: fmt.Sprintf("doc:%d", id), Backend: domain.MediaBackendLocalFS,
ObjectKey: fmt.Sprintf("obj-doc-%d", id), Size: 100,
}}, nil
}
func (f *fakeManualPurgeStore) DeleteFileBlobsForPhoto(_ context.Context, id int64) ([]domain.FileBlob, error) {
f.deletedPhotos = append(f.deletedPhotos, id)
return []domain.FileBlob{{
LocationKey: fmt.Sprintf("photo:%d", id), Backend: domain.MediaBackendLocalFS,
ObjectKey: fmt.Sprintf("obj-photo-%d", id), Size: 50,
}}, nil
}
// The rest of mediaRetentionStore isn't exercised by ManualPurge/
// CountManualPurgeCandidates -- stub it out, same as
// fakeCategorySweepStore.
func (f *fakeManualPurgeStore) ListOrphanedDocumentIDsOlderThan(context.Context, domain.MediaCategory, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
// CountFileBlobRefs deliberately returns 1 (not 0): ManualPurge's deleted
// blobs must still be accounted for (deleteOrphanedBlobs's cache eviction),
// but this fake has no real BlobBackend wired up, so it reports the blob as
// still referenced elsewhere -- skipping the physical backend.Delete call
// that would otherwise need a working s.blobs.
func (f *fakeManualPurgeStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
return 1, nil
}
func (f *fakeManualPurgeStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeManualPurgeStore) SumFileBlobBytes(context.Context) (int64, error) { return 0, nil }
func (f *fakeManualPurgeStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeManualPurgeStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
func newManualPurgeTestService(fake *fakeManualPurgeStore) *Service {
return &Service{
media: fake,
log: zap.NewNop(),
blobCache: newBlobMetaCache(64),
byteCache: newBlobBytesCache(1 << 20),
}
}
// TestManualPurgeOnlyTouchesSelectedCategory guards ManualPurge's per-category
// scoping: purging just "Video" must never touch a Music document sitting in
// the same fake store, mirroring the real query's category filter (see
// TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory in
// internal/store/postgres).
func TestManualPurgeOnlyTouchesSelectedCategory(t *testing.T) {
fake := &fakeManualPurgeStore{
docsByCategory: map[domain.MediaCategory][]int64{
domain.MediaCategoryVideo: {1},
domain.MediaCategoryMusic: {2},
},
docCreatedAt: map[int64]time.Time{
1: time.Now().Add(-100 * 24 * time.Hour),
2: time.Now().Add(-100 * 24 * time.Hour),
},
}
s := newManualPurgeTestService(fake)
purgedDocs, purgedPhotos, bytes, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryVideo}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 1 || purgedPhotos != 0 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 1/0", purgedDocs, purgedPhotos)
}
if bytes != 100 {
t.Fatalf("bytesReclaimed = %d, want 100", bytes)
}
if len(fake.deletedDocs) != 1 || fake.deletedDocs[0] != 1 {
t.Fatalf("deletedDocs = %v, want only [1] (the Video document)", fake.deletedDocs)
}
}
// TestManualPurgeNilBeforeIgnoresAge is the unit-level counterpart of the
// Postgres integration test TestManualPurgeNilCutoffIgnoresAge: a document
// created "just now" must still be a purge candidate when before is nil,
// unlike the automatic sweep which always filters by a configured age.
func TestManualPurgeNilBeforeIgnoresAge(t *testing.T) {
fake := &fakeManualPurgeStore{
docsByCategory: map[domain.MediaCategory][]int64{
domain.MediaCategoryFile: {42},
},
docCreatedAt: map[int64]time.Time{
42: time.Now(), // created just now
},
}
s := newManualPurgeTestService(fake)
// Dry-run: nil before must count the fresh document.
docs, photos, err := s.CountManualPurgeCandidates(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, nil)
if err != nil {
t.Fatalf("CountManualPurgeCandidates(nil before): %v", err)
}
if docs != 1 || photos != 0 {
t.Fatalf("CountManualPurgeCandidates(nil before) = (%d, %d), want (1, 0)", docs, photos)
}
// A real cutoff strictly before "now" must exclude the fresh document.
past := time.Now().Add(-time.Hour)
docs, photos, err = s.CountManualPurgeCandidates(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, &past)
if err != nil {
t.Fatalf("CountManualPurgeCandidates(past before): %v", err)
}
if docs != 0 || photos != 0 {
t.Fatalf("CountManualPurgeCandidates(past before) = (%d, %d), want (0, 0)", docs, photos)
}
// Confirm: nil before must actually purge the fresh document.
purgedDocs, _, _, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryFile}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge(nil before): %v", err)
}
if purgedDocs != 1 {
t.Fatalf("ManualPurge(nil before) purgedDocs = %d, want 1", purgedDocs)
}
}
// TestManualPurgePhotoCategoryUsesPhotoQuery guards that selecting the
// "Photo" category (unlike the other categories, which are documents.category
// buckets) routes through the dedicated photo query rather than being
// silently dropped or misrouted to the document path.
func TestManualPurgePhotoCategoryUsesPhotoQuery(t *testing.T) {
fake := &fakeManualPurgeStore{
photos: []int64{7},
photoCreatedAt: map[int64]time.Time{7: time.Now()},
}
s := newManualPurgeTestService(fake)
purgedDocs, purgedPhotos, _, err := s.ManualPurge(context.Background(),
[]domain.MediaCategory{domain.MediaCategoryPhoto}, false, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 0 || purgedPhotos != 1 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 0/1", purgedDocs, purgedPhotos)
}
if len(fake.deletedPhotos) != 1 || fake.deletedPhotos[0] != 7 {
t.Fatalf("deletedPhotos = %v, want only [7]", fake.deletedPhotos)
}
}
// TestManualPurgeIncludeAvatarsIsIndependentOfCategories guards that
// includeAvatars purges avatar photos even when Photo is not among the
// selected categories (and vice versa: selecting Photo alone must not also
// sweep avatars) -- the same split the automatic sweep already has between
// categoryRetentionAge(Photo) and avatarRetentionAge().
func TestManualPurgeIncludeAvatarsIsIndependentOfCategories(t *testing.T) {
fake := &fakeManualPurgeStore{
avatarPhotos: []int64{9},
avatarCreated: map[int64]time.Time{9: time.Now()},
}
s := newManualPurgeTestService(fake)
// No categories selected, but includeAvatars=true must still purge it.
purgedDocs, purgedPhotos, _, err := s.ManualPurge(context.Background(), nil, true, nil, 100)
if err != nil {
t.Fatalf("ManualPurge: %v", err)
}
if purgedDocs != 0 || purgedPhotos != 1 {
t.Fatalf("purgedDocs=%d purgedPhotos=%d, want 0/1", purgedDocs, purgedPhotos)
}
if len(fake.deletedPhotos) != 1 || fake.deletedPhotos[0] != 9 {
t.Fatalf("deletedPhotos = %v, want only [9]", fake.deletedPhotos)
}
}
// TestManualPurgeRejectsUnknownCategory guards validateManualPurgeCategories:
// an operator who fat-fingers a category should get an error, not a purge
// that quietly did less than they asked for.
func TestManualPurgeRejectsUnknownCategory(t *testing.T) {
fake := &fakeManualPurgeStore{}
s := newManualPurgeTestService(fake)
invalid := domain.MediaCategory(99)
if _, _, _, err := s.ManualPurge(context.Background(), []domain.MediaCategory{invalid}, false, nil, 100); err == nil {
t.Fatal("ManualPurge with an unsupported category = nil error, want an error")
}
if _, _, err := s.CountManualPurgeCandidates(context.Background(), []domain.MediaCategory{invalid}, false, nil); err == nil {
t.Fatal("CountManualPurgeCandidates with an unsupported category = nil error, want an error")
}
// MediaCategoryNone is a real category, but not one ManualPurge accepts
// (see manualPurgeCategories's doc comment) -- it must be rejected too,
// not silently treated as "match nothing"/"match everything".
if _, _, _, err := s.ManualPurge(context.Background(), []domain.MediaCategory{domain.MediaCategoryNone}, false, nil, 100); err == nil {
t.Fatal("ManualPurge with MediaCategoryNone = nil error, want an error")
}
}

View file

@ -0,0 +1,52 @@
package files
import (
"context"
"fmt"
"go.uber.org/zap"
"telesrv/internal/domain"
)
// DeleteEncryptedFileBlob deletes a secret-chat encrypted file's blob bytes
// (location_key "enc:<id>") once the recipient has fully downloaded it --
// see internal/rpc/upload.go's onUploadGetFile, which calls this after a
// getFile response whose bytes reach the end of the file. Secret chats are
// single-device on both ends (no multi-device sync, unlike ordinary chats),
// so once the one recipient device that will ever ask for this ciphertext
// has it, the server has no further reason to keep it -- unlike ordinary
// message media, there is no "another device might still need to fetch
// this" case to protect against.
//
// No-op unless TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD is enabled
// (WithSecretChatDeleteFileAfterDownload, default true). Best-effort: the
// caller is a fire-and-forget goroutine off the download response, so
// errors are only logged, never propagated -- a failed cleanup here must
// never turn into a failed or delayed file download.
func (s *Service) DeleteEncryptedFileBlob(ctx context.Context, locationKey string) error {
if !s.secretChatDeleteFileAfterDownload {
return nil
}
store, ok := s.media.(mediaRetentionStore)
if !ok {
return nil
}
blob, found, err := s.media.GetFileBlob(ctx, locationKey)
if err != nil {
return fmt.Errorf("get encrypted file blob: %w", err)
}
if !found {
// Already deleted (e.g. a racing duplicate getFile request for the
// same final chunk) -- nothing left to do.
return nil
}
if err := s.media.DeleteFileBlobRow(ctx, locationKey); err != nil {
return fmt.Errorf("delete encrypted file blob row: %w", err)
}
s.blobCache.delete(locationKey)
s.deleteOrphanedBlobs(ctx, store, []domain.FileBlob{blob})
s.log.Info("secret chat file deleted after download",
zap.String("location_key", locationKey), zap.Int64("size", blob.Size))
return nil
}

View file

@ -0,0 +1,185 @@
package files
import (
"context"
"errors"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeEncryptedBlobStore is a minimal stateful store.MediaStore double for
// DeleteEncryptedFileBlob: a real in-memory map of location_key -> FileBlob
// (so GetFileBlob/DeleteFileBlobRow actually observe each other's effects).
// CountFileBlobRefs always reports "still referenced" so deleteOrphanedBlobs
// never reaches its physical-backend-delete path -- this test's Service has
// no real blob backend configured, and the point here is the file_blobs row
// deletion, which DeleteFileBlobRow already covers directly. Embeds a nil
// store.MediaStore for the rest of that large interface's methods, which
// this test never calls.
type fakeEncryptedBlobStore struct {
store.MediaStore
blobs map[string]domain.FileBlob
deletedRows []string
countRefsErr error
getFileBlobErr error
}
func (f *fakeEncryptedBlobStore) GetFileBlob(_ context.Context, key string) (domain.FileBlob, bool, error) {
if f.getFileBlobErr != nil {
return domain.FileBlob{}, false, f.getFileBlobErr
}
b, ok := f.blobs[key]
return b, ok, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobRow(_ context.Context, key string) error {
f.deletedRows = append(f.deletedRows, key)
delete(f.blobs, key)
return nil
}
func (f *fakeEncryptedBlobStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
if f.countRefsErr != nil {
return 0, f.countRefsErr
}
// Reported as still-referenced (>0) so deleteOrphanedBlobs's physical
// backend-delete path (this test's Service has no real backend
// configured) is never reached -- this test only cares about the
// file_blobs row itself, which DeleteFileBlobRow already removed by the
// time deleteOrphanedBlobs runs.
return 1, nil
}
// The rest of mediaRetentionStore's methods aren't part of store.MediaStore
// (kept out of the hot RPC-facing interface on purpose -- see that
// interface's own doc comment), so embedding store.MediaStore alone doesn't
// satisfy the type assertion DeleteEncryptedFileBlob performs. Stub them
// out; this test never exercises them.
func (f *fakeEncryptedBlobStore) ListOrphanedDocumentIDsOlderThan(context.Context, domain.MediaCategory, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeEncryptedBlobStore) ListDocumentIDsForHardRetentionOlderThan(context.Context, domain.MediaCategory, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, *time.Time, int) ([]int64, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) CountDocumentsForHardRetention(context.Context, domain.MediaCategory, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) CountPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) CountAvatarPhotosForHardRetention(context.Context, *time.Time) (int, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobsForDocument(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) DeleteFileBlobsForPhoto(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) SumFileBlobBytes(context.Context) (int64, error) {
return 0, nil
}
func (f *fakeEncryptedBlobStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeEncryptedBlobStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
func newTestServiceForSecretChatFiles(media store.MediaStore, enabled bool) *Service {
return &Service{
media: media,
log: zap.NewNop(),
blobCache: newBlobMetaCache(8),
secretChatDeleteFileAfterDownload: enabled,
}
}
func TestDeleteEncryptedFileBlobNoOpWhenDisabled(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{
"enc:1": {LocationKey: "enc:1", Backend: domain.MediaBackendS3, ObjectKey: "obj-1", Size: 42},
}}
s := newTestServiceForSecretChatFiles(fake, false)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob: %v", err)
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none (feature disabled)", fake.deletedRows)
}
if _, ok := fake.blobs["enc:1"]; !ok {
t.Fatal("blob was removed from the store despite the feature being disabled")
}
}
func TestDeleteEncryptedFileBlobDeletesWhenEnabled(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{
"enc:1": {LocationKey: "enc:1", Backend: domain.MediaBackendS3, ObjectKey: "obj-1", Size: 42},
}}
s := newTestServiceForSecretChatFiles(fake, true)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob: %v", err)
}
if len(fake.deletedRows) != 1 || fake.deletedRows[0] != "enc:1" {
t.Fatalf("deleted rows = %v, want [enc:1]", fake.deletedRows)
}
if _, ok := fake.blobs["enc:1"]; ok {
t.Fatal("blob row still present after DeleteEncryptedFileBlob")
}
}
func TestDeleteEncryptedFileBlobNoOpWhenAlreadyGone(t *testing.T) {
fake := &fakeEncryptedBlobStore{blobs: map[string]domain.FileBlob{}}
s := newTestServiceForSecretChatFiles(fake, true)
// A racing duplicate final-chunk request for the same file: the blob is
// already deleted. Must not error.
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:404"); err != nil {
t.Fatalf("DeleteEncryptedFileBlob for a missing blob: %v", err)
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none", fake.deletedRows)
}
}
func TestDeleteEncryptedFileBlobPropagatesLookupError(t *testing.T) {
fake := &fakeEncryptedBlobStore{
blobs: map[string]domain.FileBlob{"enc:1": {LocationKey: "enc:1"}},
getFileBlobErr: errors.New("db unavailable"),
}
s := newTestServiceForSecretChatFiles(fake, true)
if err := s.DeleteEncryptedFileBlob(context.Background(), "enc:1"); err == nil {
t.Fatal("DeleteEncryptedFileBlob = nil error, want the underlying lookup failure surfaced")
}
if len(fake.deletedRows) != 0 {
t.Fatalf("deleted rows = %v, want none (lookup failed before any delete)", fake.deletedRows)
}
}

View file

@ -157,6 +157,13 @@ func (f *fakeMediaStore) GetFileBlob(_ context.Context, key string) (domain.File
return b, ok, nil
}
func (f *fakeMediaStore) DeleteFileBlobRow(_ context.Context, key string) error {
f.mu.Lock()
defer f.mu.Unlock()
delete(f.blobs, key)
return nil
}
func (f *fakeMediaStore) GetFileBlobs(_ context.Context, keys []string) (map[string]domain.FileBlob, error) {
f.mu.Lock()
defer f.mu.Unlock()

View file

@ -112,6 +112,10 @@ type Service struct {
// EvictOldestMediaOverBudget as the active-eviction trigger threshold
// (<=0 disables eviction regardless of TELESRV_STORAGE_EVICTION_ENABLE).
storageMaxTotalBytes int64
// secretChatDeleteFileAfterDownload is
// TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD -- see
// DeleteEncryptedFileBlob's doc comment.
secretChatDeleteFileAfterDownload bool
// retentionNotifyMu guards retentionMessages/retentionChannels: they are
// set post-construction (see SetRetentionPurgeNotifier) from
@ -242,6 +246,18 @@ func WithStorageMaxTotalBytes(maxBytes int64) Option {
}
}
// WithSecretChatDeleteFileAfterDownload enables DeleteEncryptedFileBlob's
// actual deletion (TELESRV_SECRET_CHAT_DELETE_FILE_AFTER_DOWNLOAD). Default
// true -- see that env var's doc comment in internal/config/config.go for
// why (secret-chat blobs have no other cleanup path at all otherwise).
// Passing false opts back out for a self-hoster who wants the ciphertext
// kept around regardless.
func WithSecretChatDeleteFileAfterDownload(enabled bool) Option {
return func(s *Service) {
s.secretChatDeleteFileAfterDownload = enabled
}
}
// WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so
// AdminDeleteUncategorizedGifs can remove a seed-imported entry's source
// file alongside its DB row -- see the field's doc comment for why that