adding more functions to media managament system

This commit is contained in:
onysd 2026-09-03 00:54:09 +03:00
parent 95e62c2d77
commit 70c0ba44f0
30 changed files with 1494 additions and 104 deletions

View file

@ -87,8 +87,11 @@ func (c *stickerSetNegativeCache) delete(refs ...domain.StickerSetRef) {
// 每个 chunk 一次 GetFileBlob 的 PG 往返(一个文件按 ≤512KB/1MB 分多次 getFile,热门贴纸/
// reaction/头像更被大量用户重复拉)。
//
// FileBlob 元数据小(约百字节)且内容不可变:location_key 一旦写入即固定指向同一 object_key,
// 新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,故只读填充、无需失效。
// FileBlob 元数据小(约百字节),新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,
// 故写入路径只读填充、无需失效。但一个 location_key 的 file_blobs 行可以被存储 retention
// 清理主动删除(尤其是 hard 模式:不等 orphaned,仍被引用的媒体到期也会被清理字节),这种情况
// 下必须调用 delete 使该 key 失效,否则缓存会继续认为它存在,导致后续下载走到已被删除的字节
// 而不是优雅返回 LOCATION_INVALID。
type blobMetaCache struct {
mu sync.Mutex
cap int
@ -141,6 +144,17 @@ func (c *blobMetaCache) put(key string, blob domain.FileBlob) {
}
}
// delete 使一个 location_key 的缓存元数据立即失效(如果存在)。见上方类型注释:
// 存储 retention 清理主动删掉该 location_key 的 file_blobs 行后必须调用。
func (c *blobMetaCache) delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[key]; ok {
c.ll.Remove(el)
delete(c.m, key)
}
}
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
// 缩略图通常只有几 KB 到几十 KB,缓存全量内容可以避开点击历史时的本地磁盘冷读抖动;
// 大媒体仍由 BlobBackend.GetRange 分段读取,避免把大文件放进内存。
@ -220,6 +234,21 @@ func (c *blobBytesCache) put(key string, bytes []byte) {
}
}
// delete evicts a cached object_key's full bytes (if present) and reclaims
// its budget. Called after a retention sweep physically deletes the object
// from its backend, so a later GetFile can't keep serving bytes that no
// longer exist on disk/S3.
func (c *blobBytesCache) delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[key]; ok {
entry := el.Value.(*blobBytesEntry)
c.ll.Remove(el)
delete(c.m, key)
c.used -= entry.size
}
}
type stickerSetFullCache struct {
mu sync.RWMutex
byID map[int64]stickerSetFullEntry

View file

@ -24,6 +24,17 @@ type mediaRetentionStore interface {
// OrphanDocumentIfUnreferenced is the immediate (no grace period)
// counterpart to the age-based sweep above -- see its doc comment.
OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)
// -- "hard" retention mode (TELESRV_STORAGE_RETENTION_MODE=hard) --
// Candidate selection ignores media_references entirely: a document/
// photo still referenced by a live message is exactly as eligible as an
// orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
}
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
@ -68,6 +79,63 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
return deleted, nil
}
// DeleteBlobBytesForMediaOlderThan implements
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
// documents/photos whose upload/created_at is older than cutoff, physically
// deletes their blob bytes (main body + thumbnail/rendition variants) from
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
// live message/profile-photo/sticker-set still references them. It
// deliberately never touches the documents/photos metadata row itself: a
// message must still be able to render "here was a photo/document"
// (dimensions, mime type, filename) after its bytes are gone, rather than
// the message breaking outright. A subsequent upload.getFile for the same
// location key finds no file_blobs row and returns LOCATION_INVALID, which
// stock clients already render as a "media unavailable" placeholder.
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, nil
}
purged := 0
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
if err != nil {
return purged, fmt.Errorf("list documents for hard retention: %w", err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
remaining := limit - len(docIDs)
if remaining <= 0 {
return purged, nil
}
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
return purged, nil
}
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
// id right now (skipping the grace period) and, only if that succeeds --
@ -106,6 +174,17 @@ func (s *Service) deleteDocumentNowIfUnreferenced(ctx context.Context, id int64)
// delete it from.
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
for _, b := range blobs {
// The file_blobs row for this exact location_key is already gone
// (the caller deleted it in the same transaction that produced this
// blob list) -- so any cached "found" metadata for it is now wrong
// regardless of whether the underlying bytes turn out to still be
// shared by another row below. Without this, a hot GetFile path that
// had this location_key's metadata cached would keep trying to read
// bytes that may no longer exist (hard retention mode purges blobs
// for actively-referenced, potentially still-hot media), producing
// an internal error instead of the graceful LOCATION_INVALID a stock
// client knows how to render.
s.blobCache.delete(b.LocationKey)
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
if err != nil {
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
@ -123,5 +202,6 @@ func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionS
if err := backend.Delete(ctx, b.ObjectKey); err != nil {
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
}
s.byteCache.delete(b.ObjectKey)
}
}

View file

@ -72,6 +72,10 @@ type Service struct {
stickerSetNegCache *stickerSetNegativeCache
uploadQuota domain.UploadPartQuota
spaceGuard SpaceGuard
// maxUploadFileBytes caps a single upload's total assembled size (sum of
// all parts). <=0 means unlimited (still bounded by the protocol's own
// MaxUploadPartBytes*MaxUploadParts ceiling). See WithMaxUploadFileBytes.
maxUploadFileBytes int64
mapTiles *mapTileProxy
externalMedia *externalMediaFetcher
webpage *webpageFetcher
@ -143,6 +147,15 @@ func WithSpaceGuard(guard SpaceGuard) Option {
}
}
// WithMaxUploadFileBytes caps a single uploaded file's total assembled size
// (sum of all parts, not any one part) -- TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES.
// <=0 leaves it unlimited (still bounded by MaxUploadPartBytes*MaxUploadParts).
func WithMaxUploadFileBytes(maxBytes int64) Option {
return func(s *Service) {
s.maxUploadFileBytes = maxBytes
}
}
// WithAdditionalBlobBackend registers a non-active blob backend purely for
// reading/deleting rows written while it used to be active. Register the
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
@ -753,6 +766,11 @@ func (s *Service) loadAndValidateUploadParts(ctx context.Context, ownerUserID, f
return nil, 0, domain.ErrFilePartsInvalid
}
}
// Checked against the total assembled size, not any single part --
// MaxUploadPartBytes above already bounds each part individually.
if s.maxUploadFileBytes > 0 && total > s.maxUploadFileBytes {
return nil, 0, domain.ErrFileTooLarge
}
return parts, total, nil
}

View file

@ -171,6 +171,89 @@ func TestCreatePhotoFromUploadReceiptReplaysAfterPartCleanup(t *testing.T) {
}
}
// TestMaxUploadFileBytesRejectsOversizedAssembledDocument covers
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES enforcement (WithMaxUploadFileBytes):
// the check must fire against the TOTAL assembled size (sum of every part),
// not any single part -- each individual part here is well under the limit,
// only their sum exceeds it.
func TestMaxUploadFileBytesRejectsOversizedAssembledDocument(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
local, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const partSize = 1024
const maxUploadFileBytes = 2 * partSize // exactly 2 parts' worth; 3 parts must be rejected
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
parts := []string{
strings.Repeat("a", partSize),
strings.Repeat("b", partSize),
strings.Repeat("c", partSize),
}
for i, part := range parts {
if _, err := svc.SaveBigFilePart(ctx, 10, 300, i, len(parts), []byte(part)); err != nil {
t.Fatalf("SaveBigFilePart %d: %v", i, err)
}
}
_, err = svc.CreateDocumentFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 300, Parts: len(parts), Name: "big.bin", Big: true},
domain.DocumentSpec{MimeType: "application/octet-stream"},
)
if !errors.Is(err, domain.ErrFileTooLarge) {
t.Fatalf("CreateDocumentFromUpload over max size err = %v, want ErrFileTooLarge", err)
}
}
// TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit ensures the check
// is an upper bound, not an off-by-one trap: a total exactly at the
// configured ceiling must still succeed.
func TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
local, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const partSize = 1024
const maxUploadFileBytes = 2 * partSize
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
parts := []string{strings.Repeat("a", partSize), strings.Repeat("b", partSize)}
for i, part := range parts {
if _, err := svc.SaveBigFilePart(ctx, 10, 301, i, len(parts), []byte(part)); err != nil {
t.Fatalf("SaveBigFilePart %d: %v", i, err)
}
}
doc, err := svc.CreateDocumentFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 301, Parts: len(parts), Name: "exact.bin", Big: true},
domain.DocumentSpec{MimeType: "application/octet-stream"},
)
if err != nil {
t.Fatalf("CreateDocumentFromUpload at exact max size: %v", err)
}
if doc.Size != int64(maxUploadFileBytes) {
t.Fatalf("doc size = %d, want %d", doc.Size, maxUploadFileBytes)
}
}
// TestMaxUploadFileBytesUnlimitedByDefault ensures leaving
// WithMaxUploadFileBytes unset (or 0) never rejects on size -- only the
// protocol's own MaxUploadPartBytes/MaxUploadParts ceiling still applies.
func TestMaxUploadFileBytesUnlimitedByDefault(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
svc, _ := newUploadPartTestService(t, media, domain.UploadPartQuota{})
file := domain.UploadedFileRef{OwnerUserID: 10, FileID: 302, Parts: 1, Name: "photo.jpg"}
if _, err := svc.SaveFilePart(ctx, file.OwnerUserID, file.FileID, 0, []byte(strings.Repeat("z", 4096))); err != nil {
t.Fatalf("SaveFilePart: %v", err)
}
if _, err := svc.CreatePhotoFromUpload(ctx, file); err != nil {
t.Fatalf("CreatePhotoFromUpload with no configured max size: %v", err)
}
}
type countingUploadPartBackend struct {
*LocalFS
getUploadPartCalls int

View file

@ -19,9 +19,38 @@ type OrphanedMediaRetentionStore interface {
// how long a document/photo must have been orphaned before it's actually
// deleted (not how old the media itself is) -- <=0 leaves the sweep
// disabled even if a store is provided, matching
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
// TELESRV_STORAGE_RETENTION_MODE=off/orphan-with-no-age being the safe
// default. Mutually exclusive with WithHardMediaRetention -- the resolved
// TELESRV_STORAGE_RETENTION_MODE is one of "off"/"orphan"/"hard", never
// both orphan and hard sweeps at once.
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.orphanedMedia = store
w.orphanedMediaMaxAge = maxAge
return w
}
// HardMediaRetentionStore physically deletes a document/photo's blob bytes
// once the media itself (its upload/created time, not how long it has been
// orphaned) is older than the configured age, REGARDLESS of whether a live
// message/profile-photo/sticker-set still references it. Unlike
// OrphanedMediaRetentionStore, it must never delete the document/photo
// metadata row -- only the underlying file_blobs row(s) and, once confirming
// no other row still needs the same content-addressed object, the physical
// bytes. This keeps a message rendering its media placeholder (dimensions,
// mime type, filename) after the bytes are gone; a subsequent download
// resolves to LOCATION_INVALID instead of the message breaking outright.
type HardMediaRetentionStore interface {
DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
}
// WithHardMediaRetention enables the aggressive storage retention sweep
// (TELESRV_STORAGE_RETENTION_MODE=hard). maxAge is how old the media itself
// must be (its created_at, not an orphaned_at grace period) before its blob
// bytes are purged -- <=0 leaves the sweep disabled even if a store is
// provided. Mutually exclusive with WithOrphanedMediaRetention -- call at
// most one of the two, matching the config's single 3-way retention mode.
func (w *RetentionWorker) WithHardMediaRetention(store HardMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.hardMedia = store
w.hardMediaMaxAge = maxAge
return w
}

View file

@ -95,6 +95,10 @@ const (
// 继续保留,在线漏推由正常 difference 路径补偿。
defaultOutboxPoisonRetention = time.Minute
defaultOutboxPoisonInterval = 15 * time.Second
// minMediaRetentionInterval floors the derived media-sweep cadence below
// so a very short (e.g. test-only) TELESRV_STORAGE_RETENTION_MAX_AGE
// can't spin the sweep query in a tight loop.
minMediaRetentionInterval = 15 * time.Second
)
// RetentionWorker 周期性回收存储中的死数据。
@ -119,6 +123,7 @@ type RetentionWorker struct {
activeAuthKeys ActiveRawAuthKeyProvider
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
orphanedMedia OrphanedMediaRetentionStore
hardMedia HardMediaRetentionStore
logger *zap.Logger
retention time.Duration
botAPIRetention time.Duration
@ -128,6 +133,7 @@ type RetentionWorker struct {
outboxPoisonRetention time.Duration
outboxPoisonInterval time.Duration
orphanedMediaMaxAge time.Duration
hardMediaMaxAge time.Duration
interval time.Duration
batch int
}
@ -259,6 +265,15 @@ func (w *RetentionWorker) Run(ctx context.Context) {
heartbeatC = heartbeatTicker.C
defer heartbeatTicker.Stop()
}
var (
mediaTicker *time.Ticker
mediaC <-chan time.Time
)
if interval := w.mediaRetentionInterval(); interval > 0 {
mediaTicker = time.NewTicker(interval)
mediaC = mediaTicker.C
defer mediaTicker.Stop()
}
for {
select {
case <-ctx.Done():
@ -269,6 +284,8 @@ func (w *RetentionWorker) Run(ctx context.Context) {
w.runOutboxPoisonOnce(ctx)
case <-heartbeatC:
w.heartbeatActiveAuthKeys(ctx)
case <-mediaC:
w.runMediaRetentionOnce(ctx)
}
}
}
@ -276,6 +293,63 @@ func (w *RetentionWorker) Run(ctx context.Context) {
func (w *RetentionWorker) runOnce(ctx context.Context) {
w.runOutboxPoisonOnce(ctx)
w.runRetentionOnce(ctx)
w.runMediaRetentionOnce(ctx)
}
// mediaRetentionInterval derives how often the storage media sweep
// (orphan/hard) runs, independent of the shared housekeeping w.interval.
// A short TELESRV_STORAGE_RETENTION_MAX_AGE (e.g. 30m) configured under a
// much longer TELESRV_RETENTION_INTERVAL (default 1h) would otherwise let
// media sit for up to age+interval past its cutoff before actually being
// swept -- capping the sweep cadence at the retention age itself bounds
// that worst case to at most 2x the configured age instead.
func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
var maxAge time.Duration
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
maxAge = w.orphanedMediaMaxAge
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
maxAge = w.hardMediaMaxAge
}
if maxAge <= 0 {
return 0
}
interval := w.interval
if interval <= 0 || maxAge < interval {
interval = maxAge
}
if interval < minMediaRetentionInterval {
interval = minMediaRetentionInterval
}
return interval
}
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
// "Hard" mode: purges blob bytes for documents/photos older than
// hardMediaMaxAge regardless of whether a live reference remains --
// the store is required to keep the document/photo metadata row
// intact, only removing file_blobs rows/bytes, so a message still
// renders its media placeholder (LOCATION_INVALID on download)
// instead of breaking outright.
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
} else if hardDeleted > 0 {
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
}
}
}
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
@ -414,17 +488,6 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
}
}
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
}
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {

View file

@ -393,3 +393,67 @@ func TestRetentionWorkerSkipsOrphanDeleteWhenHeartbeatFails(t *testing.T) {
t.Fatalf("heartbeat failure signals = %+v", entries)
}
}
type fakeHardMediaRetention struct {
calls int
cutoff time.Time
limit int
deleted int
}
func (f *fakeHardMediaRetention) DeleteBlobBytesForMediaOlderThan(_ context.Context, cutoff time.Time, limit int) (int, error) {
f.calls++
f.cutoff = cutoff
f.limit = limit
return f.deleted, nil
}
// TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge guards the fix for
// a real gotcha reported live: with a short storage retention age (e.g. 30m)
// under the default 1h TELESRV_RETENTION_INTERVAL housekeeping cadence,
// media eligible for hard/orphan deletion used to sit for up to age+interval
// before the shared ticker actually got around to sweeping it (a 30m age
// could take up to 1h30m to actually disappear). The sweep now runs on its
// own ticker capped at the configured age, so it never lags by more than
// roughly one age-window.
func TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge(t *testing.T) {
hard := &fakeHardMediaRetention{}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, 30*time.Minute)
if got := w.mediaRetentionInterval(); got != 30*time.Minute {
t.Fatalf("media retention interval = %v, want min(interval=1h, maxAge=30m) = 30m", got)
}
}
func TestRetentionWorkerMediaSweepIntervalFloorsAtMinimum(t *testing.T) {
hard := &fakeHardMediaRetention{}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, time.Second)
if got := w.mediaRetentionInterval(); got != minMediaRetentionInterval {
t.Fatalf("media retention interval = %v, want floor %v", got, minMediaRetentionInterval)
}
}
func TestRetentionWorkerMediaSweepIntervalZeroWhenNoModeConfigured(t *testing.T) {
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50)
if got := w.mediaRetentionInterval(); got != 0 {
t.Fatalf("media retention interval = %v, want 0 (no separate ticker) when retention is off", got)
}
}
// TestRetentionWorkerRunsHardMediaSweepOnStartup confirms runOnce (called
// once immediately when Run starts, matching every other retention check)
// still fires the media sweep too, not just the new dedicated ticker.
func TestRetentionWorkerRunsHardMediaSweepOnStartup(t *testing.T) {
hard := &fakeHardMediaRetention{deleted: 4}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, 30*time.Minute)
w.runOnce(context.Background())
if hard.calls != 1 {
t.Fatalf("hard media sweep calls = %d, want 1", hard.calls)
}
}

View file

@ -13,11 +13,20 @@ import (
"golang.org/x/text/language"
"telesrv/internal/app/files"
"telesrv/internal/branding"
"telesrv/internal/domain"
"telesrv/internal/links"
)
// Storage retention modes for TELESRV_STORAGE_RETENTION_MODE. See
// Config.StorageRetentionMode for what each one does.
const (
StorageRetentionModeOff = "off"
StorageRetentionModeOrphan = "orphan"
StorageRetentionModeHard = "hard"
)
const (
defaultConfigFile = ".env"
defaultCountryCode = "CN"
@ -311,19 +320,41 @@ type Config struct {
// the s3 backend (no OS-level free space concept), and usable as an
// optional soft budget cap on localfs too. <=0 disables this check.
StorageMaxTotalBytes int64
// StorageMaxUploadFileBytes caps the total assembled size of a single
// uploaded file (sum of all its parts, not any one part) -- rejected at
// upload-finalize time with a FILE_TOO_BIG rpc error. <=0 disables this
// check (the only remaining ceiling is then the protocol's own part-count
// limit, files.MaxUploadPartBytes*files.MaxUploadParts, ~4GB). Validated
// at config-load time to never exceed that protocol ceiling -- a larger
// value could never actually be reached, so it's refused as a
// startup-time misconfiguration rather than silently accepted as a no-op.
StorageMaxUploadFileBytes int64
// StorageUsageRefreshInterval is how often the cached free-space/budget
// usage gauge refreshes; <=0 uses a 1 minute default.
StorageUsageRefreshInterval time.Duration
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
// default: orphaned media (no live message/profile-photo/sticker-set
// reference) is tracked and visible in the admin panel, but nothing is
// auto-deleted until the operator explicitly opts in.
StorageRetentionEnable bool
// StorageRetentionMaxAge is how long a document/photo must have been
// orphaned (not how old the media itself is) before the sweep deletes
// it. Never deletes media that still has a live reference, regardless
// of age. <=0 uses a 30 day default. The sweep itself runs on the
// shared RetentionInterval/RetentionBatch cadence alongside every other
// StorageRetentionMode selects the storage retention sweep behavior:
// - "off" (default): no sweep at all.
// - "orphan": safe mode -- deletes a document/photo's blob only once it
// has had no live message/profile-photo/sticker-set reference for at
// least StorageRetentionMaxAge. Never touches media still referenced,
// regardless of age.
// - "hard": aggressive mode -- deletes a document/photo's blob bytes
// once the media itself (its upload/created time, not how long it's
// been orphaned) is older than StorageRetentionMaxAge, REGARDLESS of
// whether it's still referenced by a live message. Only the blob
// bytes are removed; the document/photo metadata row is kept so the
// message still renders (dimensions, mime type, filename) --
// subsequent downloads get LOCATION_INVALID ("media no longer
// available") instead of the message breaking outright. This is
// irreversible and can surprise users if misunderstood: old media in
// active conversations WILL disappear.
// Any other value fails config validation.
StorageRetentionMode string
// StorageRetentionMaxAge is the shared age threshold for both "orphan"
// and "hard" modes -- see StorageRetentionMode for how its meaning
// differs between them. Ignored when StorageRetentionMode is "off". <=0
// uses a 30 day default. The sweep itself runs on the shared
// RetentionInterval/RetentionBatch cadence alongside every other
// retention check (see maintenance.RetentionWorker).
StorageRetentionMaxAge time.Duration
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob)。
@ -945,8 +976,9 @@ func Load() (Config, error) {
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
StorageMaxUploadFileBytes: envInt64Or("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", 0),
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
@ -1285,6 +1317,9 @@ func validateBlobStorageConfig(cfg Config) error {
if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
}
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
return err
}
if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled")
}
@ -1474,14 +1509,39 @@ func validateStorageConfig(cfg Config) error {
if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
}
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
return err
}
if cfg.StorageUsageRefreshInterval < 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
}
if cfg.StorageRetentionMaxAge < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
}
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
switch cfg.StorageRetentionMode {
case StorageRetentionModeOff:
// no sweep; StorageRetentionMaxAge is ignored.
case StorageRetentionModeOrphan, StorageRetentionModeHard:
if cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
}
default:
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
}
return nil
}
// validateMaxUploadFileBytes checks TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: it
// must be non-negative, and if set (>0) must not exceed the protocol's own
// part-count upload ceiling -- a larger configured value could never actually
// be hit, so it's refused as a startup-time misconfiguration.
func validateMaxUploadFileBytes(v int64) error {
if v < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES must be non-negative")
}
const protocolCeiling = int64(files.MaxUploadPartBytes) * int64(files.MaxUploadParts)
if v > protocolCeiling {
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES (%d) must not exceed the protocol upload ceiling of %d bytes (%d parts x %d bytes)", v, protocolCeiling, files.MaxUploadParts, files.MaxUploadPartBytes)
}
return nil
}

View file

@ -1,6 +1,7 @@
package config
import (
"fmt"
"os"
"path/filepath"
"testing"
@ -467,6 +468,101 @@ func TestLoadRejectsInvalidStorageCapacityConfig(t *testing.T) {
}
}
func TestLoadMaxUploadFileBytesDefaultsToUnlimited(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageMaxUploadFileBytes != 0 {
t.Fatalf("expected default StorageMaxUploadFileBytes=0 (unlimited), got %d", cfg.StorageMaxUploadFileBytes)
}
}
func TestLoadAcceptsMaxUploadFileBytesAtProtocolCeiling(t *testing.T) {
disableDefaultConfigFile(t)
const ceiling = int64(524288) * 8000 // files.MaxUploadPartBytes * files.MaxUploadParts
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", fmt.Sprint(ceiling))
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageMaxUploadFileBytes != ceiling {
t.Fatalf("expected StorageMaxUploadFileBytes=%d, got %d", ceiling, cfg.StorageMaxUploadFileBytes)
}
}
func TestLoadRejectsInvalidMaxUploadFileBytes(t *testing.T) {
const ceiling = int64(524288) * 8000
for _, item := range []struct {
name string
value string
}{
{"negative", "-1"},
{"exceeds protocol ceiling", fmt.Sprint(ceiling + 1)},
} {
t.Run(item.name, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", item.value)
if _, err := Load(); err == nil {
t.Fatalf("Load accepted invalid TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=%s", item.value)
}
})
}
}
func TestLoadStorageRetentionModeDefaultsToOff(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageRetentionMode != StorageRetentionModeOff {
t.Fatalf("expected default StorageRetentionMode=%q, got %q", StorageRetentionModeOff, cfg.StorageRetentionMode)
}
}
func TestLoadStorageRetentionModeOrphanAndHard(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "48h")
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageRetentionMode != mode {
t.Fatalf("expected StorageRetentionMode=%q, got %q", mode, cfg.StorageRetentionMode)
}
if cfg.StorageRetentionMaxAge != 48*time.Hour {
t.Fatalf("expected StorageRetentionMaxAge=48h, got %v", cfg.StorageRetentionMaxAge)
}
})
}
}
func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", "sometimes")
if _, err := Load(); err == nil {
t.Fatal("Load accepted an unknown TELESRV_STORAGE_RETENTION_MODE")
}
}
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
if _, err := Load(); err == nil {
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
}
})
}
}
func TestLoadUpdateServiceConfig(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/")

View file

@ -9,6 +9,11 @@ var (
ErrFilePartsInvalid = errors.New("file parts invalid")
ErrFilePartTooBig = errors.New("file part too big")
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
// ErrFileTooLarge is returned when a file's total assembled size (sum of
// all its parts, not any single part) exceeds the configured
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES ceiling. Distinct from
// ErrFilePartTooBig, which is a single oversized part.
ErrFileTooLarge = errors.New("file too large")
ErrPhotoInvalid = errors.New("photo invalid")
ErrDocumentInvalid = errors.New("document invalid")
// ErrStorageFull is returned when the configured low-space guard rejects a

View file

@ -133,3 +133,56 @@ func TestWriteEnvValuesRoundTrip(t *testing.T) {
t.Error(".env lost .env.example's comment lines on write")
}
}
// TestWriteEnvValuesPreservesUnrelatedCustomValues guards against a real
// regression: a save that only touches one section's keys used to reset
// every OTHER already-customized key (e.g. TELESRV_ADMIN_UI_PASSWORD) back
// to .env.example's bare template default, because the old implementation
// fell back to the template line instead of the current .env value for any
// key missing from that save's payload.
func TestWriteEnvValuesPreservesUnrelatedCustomValues(t *testing.T) {
root := findRepoRoot(t)
tmpl, err := os.ReadFile(filepath.Join(root, ".env.example"))
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, ".env.example"), tmpl, 0o644); err != nil {
t.Fatal(err)
}
m := NewManager(dir)
// First save sets the admin password, as a one-time setup step would.
if err := m.WriteEnvValues(map[string]string{
"TELESRV_ADMIN_UI_PASSWORD": "s3cret",
}); err != nil {
t.Fatal(err)
}
// A later, unrelated save (e.g. the Storage page) that never mentions
// the password must not disturb it.
if err := m.WriteEnvValues(map[string]string{
"TELESRV_STORAGE_MAX_TOTAL_BYTES": "209715200",
}); err != nil {
t.Fatal(err)
}
groups, err := m.ReadEnvGroups()
if err != nil {
t.Fatal(err)
}
values := map[string]string{}
for _, g := range groups {
for _, f := range g.Fields {
values[f.Key] = f.Value
}
}
if values["TELESRV_ADMIN_UI_PASSWORD"] != "s3cret" {
t.Errorf("TELESRV_ADMIN_UI_PASSWORD = %q, want it preserved as \"s3cret\" after an unrelated save", values["TELESRV_ADMIN_UI_PASSWORD"])
}
if values["TELESRV_STORAGE_MAX_TOTAL_BYTES"] != "209715200" {
t.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES = %q, want 209715200", values["TELESRV_STORAGE_MAX_TOTAL_BYTES"])
}
}

View file

@ -637,15 +637,25 @@ func (m *Manager) readEnvFile() (map[string]string, error) {
// WriteEnvValues rewrites .env from .env.example's exact text, substituting
// each known key's value in place -- see save_env()'s docstring in
// server-panel.py for why this (not a fresh key=value dump) is what
// preserves comments/layout. Only keys present in values are touched; a
// template-commented optional field is uncommented when given a non-empty
// value and left as-is when given an empty one.
// preserves comments/layout. Only keys present in values are set to a new
// value; every other key keeps whatever is already in the current .env
// (falling back to the template's own default only for a key .env never
// set) -- previously this fell straight back to the template default for
// any key not in this particular save's payload, silently wiping out every
// other customized setting (e.g. TELESRV_ADMIN_UI_PASSWORD) on every save
// that only touches one section's keys. A template-commented optional field
// is uncommented when given a non-empty value and left as-is when given an
// empty one.
func (m *Manager) WriteEnvValues(values map[string]string) error {
tmplPath := filepath.Join(m.Root, ".env.example")
tmplData, err := os.ReadFile(tmplPath)
if err != nil {
return fmt.Errorf("read .env.example: %w", err)
}
existing, err := m.readEnvFile()
if err != nil {
return err
}
lines := strings.Split(string(tmplData), "\n")
// Split() on a trailing "\n" leaves one empty trailing element; drop it
// so the join below doesn't add a spurious blank line before the final
@ -657,15 +667,20 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
seen := map[string]bool{}
for _, raw := range lines {
line := strings.TrimSpace(raw)
if a := activeFieldRe.FindStringSubmatch(line); a != nil {
if v, ok := values[a[1]]; ok && !seen[a[1]] {
if a := activeFieldRe.FindStringSubmatch(line); a != nil && !seen[a[1]] {
if v, ok := values[a[1]]; ok {
seen[a[1]] = true
out = append(out, a[1]+"="+v)
continue
}
if v, ok := existing[a[1]]; ok {
seen[a[1]] = true
out = append(out, a[1]+"="+v)
continue
}
}
if c := commentedFieldRe.FindStringSubmatch(line); c != nil {
if v, ok := values[c[1]]; ok && !seen[c[1]] {
if c := commentedFieldRe.FindStringSubmatch(line); c != nil && !seen[c[1]] {
if v, ok := values[c[1]]; ok {
seen[c[1]] = true
if v != "" {
out = append(out, c[1]+"="+v)
@ -674,6 +689,14 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
}
continue
}
// A previously-enabled optional field shows up in the current
// .env as an active line even though the template still has it
// commented out -- keep it enabled with its existing value.
if v, ok := existing[c[1]]; ok {
seen[c[1]] = true
out = append(out, c[1]+"="+v)
continue
}
}
out = append(out, raw)
}

View file

@ -107,6 +107,15 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
// fileTooBigErr surfaces TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: the upload's
// total assembled size (not any single part) exceeds the configured
// per-file ceiling. FILE_TOO_BIG is not an official Telegram desktop-RPC
// error string, but it's already this codebase's own convention for the
// same concept on the Bot API surface (see internal/botapi/server.go) --
// reused here for consistency rather than inventing a second name for the
// same condition.
func fileTooBigErr() error { return tgerr.New(400, "FILE_TOO_BIG") }
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
// the client to retry shortly would be misleading.

View file

@ -1152,6 +1152,8 @@ func mediaUploadErr(err error) error {
switch {
case errors.Is(err, domain.ErrFilePartsInvalid):
return filePartsInvalidErr()
case errors.Is(err, domain.ErrFileTooLarge):
return fileTooBigErr()
case errors.Is(err, domain.ErrPhotoInvalid):
return photoInvalidErr()
case errors.Is(err, domain.ErrDocumentInvalid):

View file

@ -348,6 +348,8 @@ func fileSaveErr(err error) error {
return filePartsInvalidErr()
case errors.Is(err, domain.ErrFilePartTooBig):
return filePartTooBigErr()
case errors.Is(err, domain.ErrFileTooLarge):
return fileTooBigErr()
case errors.Is(err, domain.ErrUploadQuotaExceeded):
return floodWaitErr(60)
case errors.Is(err, domain.ErrStorageFull):

View file

@ -0,0 +1,105 @@
package postgres
import (
"context"
"strconv"
"testing"
"time"
"telesrv/internal/domain"
)
// TestHardRetentionPurgesBlobBytesButKeepsMetadataRow exercises the "hard"
// storage retention mode's store methods end-to-end against a real
// Postgres: a document old enough (by created_at, not orphaned_at) is a
// candidate for ListDocumentIDsForHardRetentionOlderThan REGARDLESS of
// still having a live media_references row, DeleteFileBlobsForDocument
// removes only its file_blobs row (never the documents row itself), and a
// second sweep pass no longer finds it a candidate since it no longer owns
// any file_blobs row. This is the correctness property the whole "hard"
// mode design hinges on: a message must still be able to render its media
// placeholder after the bytes are gone.
func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
docID := time.Now().UnixNano()
locationKey := "doc:" + strconv.FormatInt(docID, 10)
if err := media.PutDocument(ctx, domain.Document{
ID: docID,
MimeType: "application/octet-stream",
Size: 1024,
}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
})
// Backdate created_at well past any plausible test cutoff -- PutDocument
// always stamps now() and has no created_at parameter.
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days' WHERE id = $1", docID); err != nil {
t.Fatalf("backdate document: %v", err)
}
blob := postgresTestBlob(locationKey, "hard-retention-doc", 1024, "application/octet-stream")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
// A LIVE reference (as if a message still embeds this document) must not
// exempt it from "hard" mode -- that's the entire point of the mode,
// unlike the orphan-only sweep.
if err := addMediaReferencesTx(ctx, pool, &domain.MessageMedia{
Kind: domain.MessageMediaKindDocument,
Document: &domain.Document{ID: docID},
}, domain.MediaRefKindMessageBox, "hard-retention-test:1"); err != nil {
t.Fatalf("add media reference: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE ref_key = 'hard-retention-test:1'")
})
cutoff := time.Now().Add(-24 * time.Hour)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
}
if !containsInt64(ids, docID) {
t.Fatalf("hard retention candidates = %v, want to include still-referenced but old document %d", ids, docID)
}
blobs, err := media.DeleteFileBlobsForDocument(ctx, docID)
if err != nil {
t.Fatalf("DeleteFileBlobsForDocument: %v", err)
}
if len(blobs) != 1 || blobs[0].LocationKey != locationKey {
t.Fatalf("deleted blobs = %+v, want exactly one for %q", blobs, locationKey)
}
// The documents row itself must survive -- a message referencing it
// still needs to render its placeholder (mime type, size, filename).
if _, found, err := media.GetDocument(ctx, docID); err != nil || !found {
t.Fatalf("document metadata row missing after hard blob purge: found=%v err=%v", found, err)
}
// The file_blobs row is gone: a subsequent download attempt resolves via
// GetFileBlob (files.Service.GetFile's path) to not-found, which the rpc
// layer already maps to LOCATION_INVALID.
if _, found, err := media.GetFileBlob(ctx, locationKey); err != nil || found {
t.Fatalf("file_blobs row still present after hard purge: found=%v err=%v", found, err)
}
// Idempotent / self-terminating: a second sweep pass no longer selects
// this document, since it no longer owns any file_blobs row.
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
}
if containsInt64(ids, docID) {
t.Fatalf("hard retention re-selected already-purged document %d", docID)
}
}

View file

@ -197,6 +197,101 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
return blobs, nil
}
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
// cutoff (by upload/created_at) that still own at least one file_blobs row,
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
// entirely: a document still referenced by a live message is exactly as
// eligible as an orphaned one.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
// ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
// (main body + thumbnail variants), returning what was deleted so the caller
// can physically remove each object from its backend once confirming (via
// CountFileBlobRefs, after this call) no other row still needs it. Unlike
// DeleteDocumentAndBlobs, this deliberately does NOT delete the documents
// row itself -- "hard" retention mode keeps the metadata (dimensions, mime
// type, filename) so a message can still render "this document is no longer
// available" instead of disappearing outright. A subsequent
// upload.getFile/GetFileBlob lookup for a location key this call removed
// correctly finds nothing and reports not-found.
func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete document blob bytes (hard retention)", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("doc:%d", id),
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list document blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}
// DeleteFileBlobsForPhoto is the photo counterpart of
// DeleteFileBlobsForDocument -- see its doc comment. Deliberately does not
// delete the photos row.
func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete photo blob bytes (hard retention)", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("photo:%d", id),
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list photo blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
// (one per rendition size), returning what was deleted so the caller can
// physically remove each object from its backend once confirming (via

View file

@ -237,6 +237,37 @@ WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
ORDER BY orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListDocumentIDsForHardRetentionOlderThan :many
-- "Hard" retention mode: candidates are documents older than cutoff (by
-- upload/created_at, NOT orphaned_at -- a live reference does not exempt
-- them) that still own at least one file_blobs row. The EXISTS check is what
-- keeps this sweep from re-selecting the same document forever: once its
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
-- rows but deliberately leaves this documents row in place), it naturally
-- drops out of this query on the next pass.
SELECT d.id FROM documents d
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListPhotoIDsForHardRetentionOlderThan :many
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
-- candidate selection, for photos.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;

View file

@ -825,6 +825,50 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
return items, nil
}
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
SELECT d.id FROM documents d
WHERE d.created_at < $1::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT $2::int
`
type ListDocumentIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// "Hard" retention mode: candidates are documents older than cutoff (by
// upload/created_at, NOT orphaned_at -- a live reference does not exempt
// them) that still own at least one file_blobs row. The EXISTS check is what
// keeps this sweep from re-selecting the same document forever: once its
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
// rows but deliberately leaves this documents row in place), it naturally
// drops out of this query on the next pass.
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
SELECT location_key, backend, object_key, size
FROM file_blobs
@ -937,6 +981,45 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
return items, nil
}
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT $2::int
`
type ListPhotoIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
// candidate selection, for photos.
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listProfilePhotos = `-- name: ListProfilePhotos :many
SELECT photo_id
FROM profile_photos