adding more functions to media managament system
This commit is contained in:
parent
95e62c2d77
commit
70c0ba44f0
30 changed files with 1494 additions and 104 deletions
|
|
@ -87,8 +87,11 @@ func (c *stickerSetNegativeCache) delete(refs ...domain.StickerSetRef) {
|
|||
// 每个 chunk 一次 GetFileBlob 的 PG 往返(一个文件按 ≤512KB/1MB 分多次 getFile,热门贴纸/
|
||||
// reaction/头像更被大量用户重复拉)。
|
||||
//
|
||||
// FileBlob 元数据小(约百字节)且内容不可变:location_key 一旦写入即固定指向同一 object_key,
|
||||
// 新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,故只读填充、无需失效。
|
||||
// FileBlob 元数据小(约百字节),新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,
|
||||
// 故写入路径只读填充、无需失效。但一个 location_key 的 file_blobs 行可以被存储 retention
|
||||
// 清理主动删除(尤其是 hard 模式:不等 orphaned,仍被引用的媒体到期也会被清理字节),这种情况
|
||||
// 下必须调用 delete 使该 key 失效,否则缓存会继续认为它存在,导致后续下载走到已被删除的字节
|
||||
// 而不是优雅返回 LOCATION_INVALID。
|
||||
type blobMetaCache struct {
|
||||
mu sync.Mutex
|
||||
cap int
|
||||
|
|
@ -141,6 +144,17 @@ func (c *blobMetaCache) put(key string, blob domain.FileBlob) {
|
|||
}
|
||||
}
|
||||
|
||||
// delete 使一个 location_key 的缓存元数据立即失效(如果存在)。见上方类型注释:
|
||||
// 存储 retention 清理主动删掉该 location_key 的 file_blobs 行后必须调用。
|
||||
func (c *blobMetaCache) delete(key string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
c.ll.Remove(el)
|
||||
delete(c.m, key)
|
||||
}
|
||||
}
|
||||
|
||||
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
|
||||
// 缩略图通常只有几 KB 到几十 KB,缓存全量内容可以避开点击历史时的本地磁盘冷读抖动;
|
||||
// 大媒体仍由 BlobBackend.GetRange 分段读取,避免把大文件放进内存。
|
||||
|
|
@ -220,6 +234,21 @@ func (c *blobBytesCache) put(key string, bytes []byte) {
|
|||
}
|
||||
}
|
||||
|
||||
// delete evicts a cached object_key's full bytes (if present) and reclaims
|
||||
// its budget. Called after a retention sweep physically deletes the object
|
||||
// from its backend, so a later GetFile can't keep serving bytes that no
|
||||
// longer exist on disk/S3.
|
||||
func (c *blobBytesCache) delete(key string) {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
if el, ok := c.m[key]; ok {
|
||||
entry := el.Value.(*blobBytesEntry)
|
||||
c.ll.Remove(el)
|
||||
delete(c.m, key)
|
||||
c.used -= entry.size
|
||||
}
|
||||
}
|
||||
|
||||
type stickerSetFullCache struct {
|
||||
mu sync.RWMutex
|
||||
byID map[int64]stickerSetFullEntry
|
||||
|
|
|
|||
|
|
@ -24,6 +24,17 @@ type mediaRetentionStore interface {
|
|||
// OrphanDocumentIfUnreferenced is the immediate (no grace period)
|
||||
// counterpart to the age-based sweep above -- see its doc comment.
|
||||
OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)
|
||||
|
||||
// -- "hard" retention mode (TELESRV_STORAGE_RETENTION_MODE=hard) --
|
||||
// Candidate selection ignores media_references entirely: a document/
|
||||
// photo still referenced by a live message is exactly as eligible as an
|
||||
// orphaned one once it's old enough. The delete methods physically
|
||||
// remove only the file_blobs row(s)/bytes, never the document/photo
|
||||
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
|
||||
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||
}
|
||||
|
||||
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
||||
|
|
@ -68,6 +79,63 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
|
|||
return deleted, nil
|
||||
}
|
||||
|
||||
// DeleteBlobBytesForMediaOlderThan implements
|
||||
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
|
||||
// documents/photos whose upload/created_at is older than cutoff, physically
|
||||
// deletes their blob bytes (main body + thumbnail/rendition variants) from
|
||||
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
|
||||
// live message/profile-photo/sticker-set still references them. It
|
||||
// deliberately never touches the documents/photos metadata row itself: a
|
||||
// message must still be able to render "here was a photo/document"
|
||||
// (dimensions, mime type, filename) after its bytes are gone, rather than
|
||||
// the message breaking outright. A subsequent upload.getFile for the same
|
||||
// location key finds no file_blobs row and returns LOCATION_INVALID, which
|
||||
// stock clients already render as a "media unavailable" placeholder.
|
||||
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
||||
store, ok := s.media.(mediaRetentionStore)
|
||||
if !ok || limit <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
purged := 0
|
||||
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
|
||||
if err != nil {
|
||||
return purged, fmt.Errorf("list documents for hard retention: %w", err)
|
||||
}
|
||||
for _, id := range docIDs {
|
||||
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
purged++
|
||||
}
|
||||
remaining := limit - len(docIDs)
|
||||
if remaining <= 0 {
|
||||
return purged, nil
|
||||
}
|
||||
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
|
||||
if err != nil {
|
||||
return purged, fmt.Errorf("list photos for hard retention: %w", err)
|
||||
}
|
||||
for _, id := range photoIDs {
|
||||
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
|
||||
if err != nil {
|
||||
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||
continue
|
||||
}
|
||||
if len(blobs) == 0 {
|
||||
continue
|
||||
}
|
||||
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||
purged++
|
||||
}
|
||||
return purged, nil
|
||||
}
|
||||
|
||||
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
|
||||
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
|
||||
// id right now (skipping the grace period) and, only if that succeeds --
|
||||
|
|
@ -106,6 +174,17 @@ func (s *Service) deleteDocumentNowIfUnreferenced(ctx context.Context, id int64)
|
|||
// delete it from.
|
||||
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
|
||||
for _, b := range blobs {
|
||||
// The file_blobs row for this exact location_key is already gone
|
||||
// (the caller deleted it in the same transaction that produced this
|
||||
// blob list) -- so any cached "found" metadata for it is now wrong
|
||||
// regardless of whether the underlying bytes turn out to still be
|
||||
// shared by another row below. Without this, a hot GetFile path that
|
||||
// had this location_key's metadata cached would keep trying to read
|
||||
// bytes that may no longer exist (hard retention mode purges blobs
|
||||
// for actively-referenced, potentially still-hot media), producing
|
||||
// an internal error instead of the graceful LOCATION_INVALID a stock
|
||||
// client knows how to render.
|
||||
s.blobCache.delete(b.LocationKey)
|
||||
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
|
||||
if err != nil {
|
||||
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
|
|
@ -123,5 +202,6 @@ func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionS
|
|||
if err := backend.Delete(ctx, b.ObjectKey); err != nil {
|
||||
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||
}
|
||||
s.byteCache.delete(b.ObjectKey)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -72,6 +72,10 @@ type Service struct {
|
|||
stickerSetNegCache *stickerSetNegativeCache
|
||||
uploadQuota domain.UploadPartQuota
|
||||
spaceGuard SpaceGuard
|
||||
// maxUploadFileBytes caps a single upload's total assembled size (sum of
|
||||
// all parts). <=0 means unlimited (still bounded by the protocol's own
|
||||
// MaxUploadPartBytes*MaxUploadParts ceiling). See WithMaxUploadFileBytes.
|
||||
maxUploadFileBytes int64
|
||||
mapTiles *mapTileProxy
|
||||
externalMedia *externalMediaFetcher
|
||||
webpage *webpageFetcher
|
||||
|
|
@ -143,6 +147,15 @@ func WithSpaceGuard(guard SpaceGuard) Option {
|
|||
}
|
||||
}
|
||||
|
||||
// WithMaxUploadFileBytes caps a single uploaded file's total assembled size
|
||||
// (sum of all parts, not any one part) -- TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES.
|
||||
// <=0 leaves it unlimited (still bounded by MaxUploadPartBytes*MaxUploadParts).
|
||||
func WithMaxUploadFileBytes(maxBytes int64) Option {
|
||||
return func(s *Service) {
|
||||
s.maxUploadFileBytes = maxBytes
|
||||
}
|
||||
}
|
||||
|
||||
// WithAdditionalBlobBackend registers a non-active blob backend purely for
|
||||
// reading/deleting rows written while it used to be active. Register the
|
||||
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
|
||||
|
|
@ -753,6 +766,11 @@ func (s *Service) loadAndValidateUploadParts(ctx context.Context, ownerUserID, f
|
|||
return nil, 0, domain.ErrFilePartsInvalid
|
||||
}
|
||||
}
|
||||
// Checked against the total assembled size, not any single part --
|
||||
// MaxUploadPartBytes above already bounds each part individually.
|
||||
if s.maxUploadFileBytes > 0 && total > s.maxUploadFileBytes {
|
||||
return nil, 0, domain.ErrFileTooLarge
|
||||
}
|
||||
return parts, total, nil
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -171,6 +171,89 @@ func TestCreatePhotoFromUploadReceiptReplaysAfterPartCleanup(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
// TestMaxUploadFileBytesRejectsOversizedAssembledDocument covers
|
||||
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES enforcement (WithMaxUploadFileBytes):
|
||||
// the check must fire against the TOTAL assembled size (sum of every part),
|
||||
// not any single part -- each individual part here is well under the limit,
|
||||
// only their sum exceeds it.
|
||||
func TestMaxUploadFileBytesRejectsOversizedAssembledDocument(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
media := newFakeMediaStore()
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalFS: %v", err)
|
||||
}
|
||||
const partSize = 1024
|
||||
const maxUploadFileBytes = 2 * partSize // exactly 2 parts' worth; 3 parts must be rejected
|
||||
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
|
||||
|
||||
parts := []string{
|
||||
strings.Repeat("a", partSize),
|
||||
strings.Repeat("b", partSize),
|
||||
strings.Repeat("c", partSize),
|
||||
}
|
||||
for i, part := range parts {
|
||||
if _, err := svc.SaveBigFilePart(ctx, 10, 300, i, len(parts), []byte(part)); err != nil {
|
||||
t.Fatalf("SaveBigFilePart %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
_, err = svc.CreateDocumentFromUpload(ctx,
|
||||
domain.UploadedFileRef{OwnerUserID: 10, FileID: 300, Parts: len(parts), Name: "big.bin", Big: true},
|
||||
domain.DocumentSpec{MimeType: "application/octet-stream"},
|
||||
)
|
||||
if !errors.Is(err, domain.ErrFileTooLarge) {
|
||||
t.Fatalf("CreateDocumentFromUpload over max size err = %v, want ErrFileTooLarge", err)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit ensures the check
|
||||
// is an upper bound, not an off-by-one trap: a total exactly at the
|
||||
// configured ceiling must still succeed.
|
||||
func TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
media := newFakeMediaStore()
|
||||
local, err := NewLocalFS(t.TempDir())
|
||||
if err != nil {
|
||||
t.Fatalf("NewLocalFS: %v", err)
|
||||
}
|
||||
const partSize = 1024
|
||||
const maxUploadFileBytes = 2 * partSize
|
||||
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
|
||||
|
||||
parts := []string{strings.Repeat("a", partSize), strings.Repeat("b", partSize)}
|
||||
for i, part := range parts {
|
||||
if _, err := svc.SaveBigFilePart(ctx, 10, 301, i, len(parts), []byte(part)); err != nil {
|
||||
t.Fatalf("SaveBigFilePart %d: %v", i, err)
|
||||
}
|
||||
}
|
||||
doc, err := svc.CreateDocumentFromUpload(ctx,
|
||||
domain.UploadedFileRef{OwnerUserID: 10, FileID: 301, Parts: len(parts), Name: "exact.bin", Big: true},
|
||||
domain.DocumentSpec{MimeType: "application/octet-stream"},
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("CreateDocumentFromUpload at exact max size: %v", err)
|
||||
}
|
||||
if doc.Size != int64(maxUploadFileBytes) {
|
||||
t.Fatalf("doc size = %d, want %d", doc.Size, maxUploadFileBytes)
|
||||
}
|
||||
}
|
||||
|
||||
// TestMaxUploadFileBytesUnlimitedByDefault ensures leaving
|
||||
// WithMaxUploadFileBytes unset (or 0) never rejects on size -- only the
|
||||
// protocol's own MaxUploadPartBytes/MaxUploadParts ceiling still applies.
|
||||
func TestMaxUploadFileBytesUnlimitedByDefault(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
media := newFakeMediaStore()
|
||||
svc, _ := newUploadPartTestService(t, media, domain.UploadPartQuota{})
|
||||
file := domain.UploadedFileRef{OwnerUserID: 10, FileID: 302, Parts: 1, Name: "photo.jpg"}
|
||||
if _, err := svc.SaveFilePart(ctx, file.OwnerUserID, file.FileID, 0, []byte(strings.Repeat("z", 4096))); err != nil {
|
||||
t.Fatalf("SaveFilePart: %v", err)
|
||||
}
|
||||
if _, err := svc.CreatePhotoFromUpload(ctx, file); err != nil {
|
||||
t.Fatalf("CreatePhotoFromUpload with no configured max size: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
type countingUploadPartBackend struct {
|
||||
*LocalFS
|
||||
getUploadPartCalls int
|
||||
|
|
|
|||
|
|
@ -19,9 +19,38 @@ type OrphanedMediaRetentionStore interface {
|
|||
// how long a document/photo must have been orphaned before it's actually
|
||||
// deleted (not how old the media itself is) -- <=0 leaves the sweep
|
||||
// disabled even if a store is provided, matching
|
||||
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
|
||||
// TELESRV_STORAGE_RETENTION_MODE=off/orphan-with-no-age being the safe
|
||||
// default. Mutually exclusive with WithHardMediaRetention -- the resolved
|
||||
// TELESRV_STORAGE_RETENTION_MODE is one of "off"/"orphan"/"hard", never
|
||||
// both orphan and hard sweeps at once.
|
||||
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
||||
w.orphanedMedia = store
|
||||
w.orphanedMediaMaxAge = maxAge
|
||||
return w
|
||||
}
|
||||
|
||||
// HardMediaRetentionStore physically deletes a document/photo's blob bytes
|
||||
// once the media itself (its upload/created time, not how long it has been
|
||||
// orphaned) is older than the configured age, REGARDLESS of whether a live
|
||||
// message/profile-photo/sticker-set still references it. Unlike
|
||||
// OrphanedMediaRetentionStore, it must never delete the document/photo
|
||||
// metadata row -- only the underlying file_blobs row(s) and, once confirming
|
||||
// no other row still needs the same content-addressed object, the physical
|
||||
// bytes. This keeps a message rendering its media placeholder (dimensions,
|
||||
// mime type, filename) after the bytes are gone; a subsequent download
|
||||
// resolves to LOCATION_INVALID instead of the message breaking outright.
|
||||
type HardMediaRetentionStore interface {
|
||||
DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
|
||||
}
|
||||
|
||||
// WithHardMediaRetention enables the aggressive storage retention sweep
|
||||
// (TELESRV_STORAGE_RETENTION_MODE=hard). maxAge is how old the media itself
|
||||
// must be (its created_at, not an orphaned_at grace period) before its blob
|
||||
// bytes are purged -- <=0 leaves the sweep disabled even if a store is
|
||||
// provided. Mutually exclusive with WithOrphanedMediaRetention -- call at
|
||||
// most one of the two, matching the config's single 3-way retention mode.
|
||||
func (w *RetentionWorker) WithHardMediaRetention(store HardMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
||||
w.hardMedia = store
|
||||
w.hardMediaMaxAge = maxAge
|
||||
return w
|
||||
}
|
||||
|
|
|
|||
|
|
@ -95,6 +95,10 @@ const (
|
|||
// 继续保留,在线漏推由正常 difference 路径补偿。
|
||||
defaultOutboxPoisonRetention = time.Minute
|
||||
defaultOutboxPoisonInterval = 15 * time.Second
|
||||
// minMediaRetentionInterval floors the derived media-sweep cadence below
|
||||
// so a very short (e.g. test-only) TELESRV_STORAGE_RETENTION_MAX_AGE
|
||||
// can't spin the sweep query in a tight loop.
|
||||
minMediaRetentionInterval = 15 * time.Second
|
||||
)
|
||||
|
||||
// RetentionWorker 周期性回收存储中的死数据。
|
||||
|
|
@ -119,6 +123,7 @@ type RetentionWorker struct {
|
|||
activeAuthKeys ActiveRawAuthKeyProvider
|
||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||
orphanedMedia OrphanedMediaRetentionStore
|
||||
hardMedia HardMediaRetentionStore
|
||||
logger *zap.Logger
|
||||
retention time.Duration
|
||||
botAPIRetention time.Duration
|
||||
|
|
@ -128,6 +133,7 @@ type RetentionWorker struct {
|
|||
outboxPoisonRetention time.Duration
|
||||
outboxPoisonInterval time.Duration
|
||||
orphanedMediaMaxAge time.Duration
|
||||
hardMediaMaxAge time.Duration
|
||||
interval time.Duration
|
||||
batch int
|
||||
}
|
||||
|
|
@ -259,6 +265,15 @@ func (w *RetentionWorker) Run(ctx context.Context) {
|
|||
heartbeatC = heartbeatTicker.C
|
||||
defer heartbeatTicker.Stop()
|
||||
}
|
||||
var (
|
||||
mediaTicker *time.Ticker
|
||||
mediaC <-chan time.Time
|
||||
)
|
||||
if interval := w.mediaRetentionInterval(); interval > 0 {
|
||||
mediaTicker = time.NewTicker(interval)
|
||||
mediaC = mediaTicker.C
|
||||
defer mediaTicker.Stop()
|
||||
}
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
|
|
@ -269,6 +284,8 @@ func (w *RetentionWorker) Run(ctx context.Context) {
|
|||
w.runOutboxPoisonOnce(ctx)
|
||||
case <-heartbeatC:
|
||||
w.heartbeatActiveAuthKeys(ctx)
|
||||
case <-mediaC:
|
||||
w.runMediaRetentionOnce(ctx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -276,6 +293,63 @@ func (w *RetentionWorker) Run(ctx context.Context) {
|
|||
func (w *RetentionWorker) runOnce(ctx context.Context) {
|
||||
w.runOutboxPoisonOnce(ctx)
|
||||
w.runRetentionOnce(ctx)
|
||||
w.runMediaRetentionOnce(ctx)
|
||||
}
|
||||
|
||||
// mediaRetentionInterval derives how often the storage media sweep
|
||||
// (orphan/hard) runs, independent of the shared housekeeping w.interval.
|
||||
// A short TELESRV_STORAGE_RETENTION_MAX_AGE (e.g. 30m) configured under a
|
||||
// much longer TELESRV_RETENTION_INTERVAL (default 1h) would otherwise let
|
||||
// media sit for up to age+interval past its cutoff before actually being
|
||||
// swept -- capping the sweep cadence at the retention age itself bounds
|
||||
// that worst case to at most 2x the configured age instead.
|
||||
func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
|
||||
var maxAge time.Duration
|
||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||
maxAge = w.orphanedMediaMaxAge
|
||||
}
|
||||
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
|
||||
maxAge = w.hardMediaMaxAge
|
||||
}
|
||||
if maxAge <= 0 {
|
||||
return 0
|
||||
}
|
||||
interval := w.interval
|
||||
if interval <= 0 || maxAge < interval {
|
||||
interval = maxAge
|
||||
}
|
||||
if interval < minMediaRetentionInterval {
|
||||
interval = minMediaRetentionInterval
|
||||
}
|
||||
return interval
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
|
||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||
// Only ever touches documents/photos already marked orphaned (no live
|
||||
// message/profile-photo/sticker-set reference remains) -- media still
|
||||
// visible in a conversation is never a candidate, regardless of age.
|
||||
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
||||
} else if mediaDeleted > 0 {
|
||||
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
|
||||
}
|
||||
}
|
||||
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
|
||||
// "Hard" mode: purges blob bytes for documents/photos older than
|
||||
// hardMediaMaxAge regardless of whether a live reference remains --
|
||||
// the store is required to keep the document/photo metadata row
|
||||
// intact, only removing file_blobs rows/bytes, so a message still
|
||||
// renders its media placeholder (LOCATION_INVALID on download)
|
||||
// instead of breaking outright.
|
||||
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
|
||||
} else if hardDeleted > 0 {
|
||||
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
|
||||
|
|
@ -414,17 +488,6 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
|
|||
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
|
||||
}
|
||||
}
|
||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||
// Only ever touches documents/photos already marked orphaned (no live
|
||||
// message/profile-photo/sticker-set reference remains) -- media still
|
||||
// visible in a conversation is never a candidate, regardless of age.
|
||||
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
||||
if err != nil {
|
||||
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
||||
} else if mediaDeleted > 0 {
|
||||
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {
|
||||
|
|
|
|||
|
|
@ -393,3 +393,67 @@ func TestRetentionWorkerSkipsOrphanDeleteWhenHeartbeatFails(t *testing.T) {
|
|||
t.Fatalf("heartbeat failure signals = %+v", entries)
|
||||
}
|
||||
}
|
||||
|
||||
type fakeHardMediaRetention struct {
|
||||
calls int
|
||||
cutoff time.Time
|
||||
limit int
|
||||
deleted int
|
||||
}
|
||||
|
||||
func (f *fakeHardMediaRetention) DeleteBlobBytesForMediaOlderThan(_ context.Context, cutoff time.Time, limit int) (int, error) {
|
||||
f.calls++
|
||||
f.cutoff = cutoff
|
||||
f.limit = limit
|
||||
return f.deleted, nil
|
||||
}
|
||||
|
||||
// TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge guards the fix for
|
||||
// a real gotcha reported live: with a short storage retention age (e.g. 30m)
|
||||
// under the default 1h TELESRV_RETENTION_INTERVAL housekeeping cadence,
|
||||
// media eligible for hard/orphan deletion used to sit for up to age+interval
|
||||
// before the shared ticker actually got around to sweeping it (a 30m age
|
||||
// could take up to 1h30m to actually disappear). The sweep now runs on its
|
||||
// own ticker capped at the configured age, so it never lags by more than
|
||||
// roughly one age-window.
|
||||
func TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge(t *testing.T) {
|
||||
hard := &fakeHardMediaRetention{}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
|
||||
WithHardMediaRetention(hard, 30*time.Minute)
|
||||
|
||||
if got := w.mediaRetentionInterval(); got != 30*time.Minute {
|
||||
t.Fatalf("media retention interval = %v, want min(interval=1h, maxAge=30m) = 30m", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionWorkerMediaSweepIntervalFloorsAtMinimum(t *testing.T) {
|
||||
hard := &fakeHardMediaRetention{}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
|
||||
WithHardMediaRetention(hard, time.Second)
|
||||
|
||||
if got := w.mediaRetentionInterval(); got != minMediaRetentionInterval {
|
||||
t.Fatalf("media retention interval = %v, want floor %v", got, minMediaRetentionInterval)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetentionWorkerMediaSweepIntervalZeroWhenNoModeConfigured(t *testing.T) {
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50)
|
||||
if got := w.mediaRetentionInterval(); got != 0 {
|
||||
t.Fatalf("media retention interval = %v, want 0 (no separate ticker) when retention is off", got)
|
||||
}
|
||||
}
|
||||
|
||||
// TestRetentionWorkerRunsHardMediaSweepOnStartup confirms runOnce (called
|
||||
// once immediately when Run starts, matching every other retention check)
|
||||
// still fires the media sweep too, not just the new dedicated ticker.
|
||||
func TestRetentionWorkerRunsHardMediaSweepOnStartup(t *testing.T) {
|
||||
hard := &fakeHardMediaRetention{deleted: 4}
|
||||
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
|
||||
WithHardMediaRetention(hard, 30*time.Minute)
|
||||
|
||||
w.runOnce(context.Background())
|
||||
|
||||
if hard.calls != 1 {
|
||||
t.Fatalf("hard media sweep calls = %d, want 1", hard.calls)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -13,11 +13,20 @@ import (
|
|||
|
||||
"golang.org/x/text/language"
|
||||
|
||||
"telesrv/internal/app/files"
|
||||
"telesrv/internal/branding"
|
||||
"telesrv/internal/domain"
|
||||
"telesrv/internal/links"
|
||||
)
|
||||
|
||||
// Storage retention modes for TELESRV_STORAGE_RETENTION_MODE. See
|
||||
// Config.StorageRetentionMode for what each one does.
|
||||
const (
|
||||
StorageRetentionModeOff = "off"
|
||||
StorageRetentionModeOrphan = "orphan"
|
||||
StorageRetentionModeHard = "hard"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultConfigFile = ".env"
|
||||
defaultCountryCode = "CN"
|
||||
|
|
@ -311,19 +320,41 @@ type Config struct {
|
|||
// the s3 backend (no OS-level free space concept), and usable as an
|
||||
// optional soft budget cap on localfs too. <=0 disables this check.
|
||||
StorageMaxTotalBytes int64
|
||||
// StorageMaxUploadFileBytes caps the total assembled size of a single
|
||||
// uploaded file (sum of all its parts, not any one part) -- rejected at
|
||||
// upload-finalize time with a FILE_TOO_BIG rpc error. <=0 disables this
|
||||
// check (the only remaining ceiling is then the protocol's own part-count
|
||||
// limit, files.MaxUploadPartBytes*files.MaxUploadParts, ~4GB). Validated
|
||||
// at config-load time to never exceed that protocol ceiling -- a larger
|
||||
// value could never actually be reached, so it's refused as a
|
||||
// startup-time misconfiguration rather than silently accepted as a no-op.
|
||||
StorageMaxUploadFileBytes int64
|
||||
// StorageUsageRefreshInterval is how often the cached free-space/budget
|
||||
// usage gauge refreshes; <=0 uses a 1 minute default.
|
||||
StorageUsageRefreshInterval time.Duration
|
||||
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
|
||||
// default: orphaned media (no live message/profile-photo/sticker-set
|
||||
// reference) is tracked and visible in the admin panel, but nothing is
|
||||
// auto-deleted until the operator explicitly opts in.
|
||||
StorageRetentionEnable bool
|
||||
// StorageRetentionMaxAge is how long a document/photo must have been
|
||||
// orphaned (not how old the media itself is) before the sweep deletes
|
||||
// it. Never deletes media that still has a live reference, regardless
|
||||
// of age. <=0 uses a 30 day default. The sweep itself runs on the
|
||||
// shared RetentionInterval/RetentionBatch cadence alongside every other
|
||||
// StorageRetentionMode selects the storage retention sweep behavior:
|
||||
// - "off" (default): no sweep at all.
|
||||
// - "orphan": safe mode -- deletes a document/photo's blob only once it
|
||||
// has had no live message/profile-photo/sticker-set reference for at
|
||||
// least StorageRetentionMaxAge. Never touches media still referenced,
|
||||
// regardless of age.
|
||||
// - "hard": aggressive mode -- deletes a document/photo's blob bytes
|
||||
// once the media itself (its upload/created time, not how long it's
|
||||
// been orphaned) is older than StorageRetentionMaxAge, REGARDLESS of
|
||||
// whether it's still referenced by a live message. Only the blob
|
||||
// bytes are removed; the document/photo metadata row is kept so the
|
||||
// message still renders (dimensions, mime type, filename) --
|
||||
// subsequent downloads get LOCATION_INVALID ("media no longer
|
||||
// available") instead of the message breaking outright. This is
|
||||
// irreversible and can surprise users if misunderstood: old media in
|
||||
// active conversations WILL disappear.
|
||||
// Any other value fails config validation.
|
||||
StorageRetentionMode string
|
||||
// StorageRetentionMaxAge is the shared age threshold for both "orphan"
|
||||
// and "hard" modes -- see StorageRetentionMode for how its meaning
|
||||
// differs between them. Ignored when StorageRetentionMode is "off". <=0
|
||||
// uses a 30 day default. The sweep itself runs on the shared
|
||||
// RetentionInterval/RetentionBatch cadence alongside every other
|
||||
// retention check (see maintenance.RetentionWorker).
|
||||
StorageRetentionMaxAge time.Duration
|
||||
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob)。
|
||||
|
|
@ -945,8 +976,9 @@ func Load() (Config, error) {
|
|||
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
|
||||
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
|
||||
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
|
||||
StorageMaxUploadFileBytes: envInt64Or("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", 0),
|
||||
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
||||
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
|
||||
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
|
||||
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
||||
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
||||
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
||||
|
|
@ -1285,6 +1317,9 @@ func validateBlobStorageConfig(cfg Config) error {
|
|||
if cfg.StorageMaxTotalBytes < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
||||
}
|
||||
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled")
|
||||
}
|
||||
|
|
@ -1474,14 +1509,39 @@ func validateStorageConfig(cfg Config) error {
|
|||
if cfg.StorageMaxTotalBytes < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
||||
}
|
||||
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
|
||||
return err
|
||||
}
|
||||
if cfg.StorageUsageRefreshInterval < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
|
||||
}
|
||||
if cfg.StorageRetentionMaxAge < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
||||
}
|
||||
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
|
||||
switch cfg.StorageRetentionMode {
|
||||
case StorageRetentionModeOff:
|
||||
// no sweep; StorageRetentionMaxAge is ignored.
|
||||
case StorageRetentionModeOrphan, StorageRetentionModeHard:
|
||||
if cfg.StorageRetentionMaxAge <= 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// validateMaxUploadFileBytes checks TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: it
|
||||
// must be non-negative, and if set (>0) must not exceed the protocol's own
|
||||
// part-count upload ceiling -- a larger configured value could never actually
|
||||
// be hit, so it's refused as a startup-time misconfiguration.
|
||||
func validateMaxUploadFileBytes(v int64) error {
|
||||
if v < 0 {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES must be non-negative")
|
||||
}
|
||||
const protocolCeiling = int64(files.MaxUploadPartBytes) * int64(files.MaxUploadParts)
|
||||
if v > protocolCeiling {
|
||||
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES (%d) must not exceed the protocol upload ceiling of %d bytes (%d parts x %d bytes)", v, protocolCeiling, files.MaxUploadParts, files.MaxUploadPartBytes)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
|
@ -467,6 +468,101 @@ func TestLoadRejectsInvalidStorageCapacityConfig(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestLoadMaxUploadFileBytesDefaultsToUnlimited(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.StorageMaxUploadFileBytes != 0 {
|
||||
t.Fatalf("expected default StorageMaxUploadFileBytes=0 (unlimited), got %d", cfg.StorageMaxUploadFileBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadAcceptsMaxUploadFileBytesAtProtocolCeiling(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
const ceiling = int64(524288) * 8000 // files.MaxUploadPartBytes * files.MaxUploadParts
|
||||
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", fmt.Sprint(ceiling))
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.StorageMaxUploadFileBytes != ceiling {
|
||||
t.Fatalf("expected StorageMaxUploadFileBytes=%d, got %d", ceiling, cfg.StorageMaxUploadFileBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidMaxUploadFileBytes(t *testing.T) {
|
||||
const ceiling = int64(524288) * 8000
|
||||
for _, item := range []struct {
|
||||
name string
|
||||
value string
|
||||
}{
|
||||
{"negative", "-1"},
|
||||
{"exceeds protocol ceiling", fmt.Sprint(ceiling + 1)},
|
||||
} {
|
||||
t.Run(item.name, func(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", item.value)
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatalf("Load accepted invalid TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=%s", item.value)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStorageRetentionModeDefaultsToOff(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.StorageRetentionMode != StorageRetentionModeOff {
|
||||
t.Fatalf("expected default StorageRetentionMode=%q, got %q", StorageRetentionModeOff, cfg.StorageRetentionMode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadStorageRetentionModeOrphanAndHard(t *testing.T) {
|
||||
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
|
||||
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "48h")
|
||||
cfg, err := Load()
|
||||
if err != nil {
|
||||
t.Fatalf("Load: %v", err)
|
||||
}
|
||||
if cfg.StorageRetentionMode != mode {
|
||||
t.Fatalf("expected StorageRetentionMode=%q, got %q", mode, cfg.StorageRetentionMode)
|
||||
}
|
||||
if cfg.StorageRetentionMaxAge != 48*time.Hour {
|
||||
t.Fatalf("expected StorageRetentionMaxAge=48h, got %v", cfg.StorageRetentionMaxAge)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", "sometimes")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatal("Load accepted an unknown TELESRV_STORAGE_RETENTION_MODE")
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
|
||||
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
|
||||
t.Run(mode, func(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
|
||||
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
|
||||
if _, err := Load(); err == nil {
|
||||
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadUpdateServiceConfig(t *testing.T) {
|
||||
disableDefaultConfigFile(t)
|
||||
t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/")
|
||||
|
|
|
|||
|
|
@ -9,6 +9,11 @@ var (
|
|||
ErrFilePartsInvalid = errors.New("file parts invalid")
|
||||
ErrFilePartTooBig = errors.New("file part too big")
|
||||
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
|
||||
// ErrFileTooLarge is returned when a file's total assembled size (sum of
|
||||
// all its parts, not any single part) exceeds the configured
|
||||
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES ceiling. Distinct from
|
||||
// ErrFilePartTooBig, which is a single oversized part.
|
||||
ErrFileTooLarge = errors.New("file too large")
|
||||
ErrPhotoInvalid = errors.New("photo invalid")
|
||||
ErrDocumentInvalid = errors.New("document invalid")
|
||||
// ErrStorageFull is returned when the configured low-space guard rejects a
|
||||
|
|
|
|||
|
|
@ -133,3 +133,56 @@ func TestWriteEnvValuesRoundTrip(t *testing.T) {
|
|||
t.Error(".env lost .env.example's comment lines on write")
|
||||
}
|
||||
}
|
||||
|
||||
// TestWriteEnvValuesPreservesUnrelatedCustomValues guards against a real
|
||||
// regression: a save that only touches one section's keys used to reset
|
||||
// every OTHER already-customized key (e.g. TELESRV_ADMIN_UI_PASSWORD) back
|
||||
// to .env.example's bare template default, because the old implementation
|
||||
// fell back to the template line instead of the current .env value for any
|
||||
// key missing from that save's payload.
|
||||
func TestWriteEnvValuesPreservesUnrelatedCustomValues(t *testing.T) {
|
||||
root := findRepoRoot(t)
|
||||
tmpl, err := os.ReadFile(filepath.Join(root, ".env.example"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, ".env.example"), tmpl, 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
m := NewManager(dir)
|
||||
|
||||
// First save sets the admin password, as a one-time setup step would.
|
||||
if err := m.WriteEnvValues(map[string]string{
|
||||
"TELESRV_ADMIN_UI_PASSWORD": "s3cret",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
// A later, unrelated save (e.g. the Storage page) that never mentions
|
||||
// the password must not disturb it.
|
||||
if err := m.WriteEnvValues(map[string]string{
|
||||
"TELESRV_STORAGE_MAX_TOTAL_BYTES": "209715200",
|
||||
}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
groups, err := m.ReadEnvGroups()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
values := map[string]string{}
|
||||
for _, g := range groups {
|
||||
for _, f := range g.Fields {
|
||||
values[f.Key] = f.Value
|
||||
}
|
||||
}
|
||||
if values["TELESRV_ADMIN_UI_PASSWORD"] != "s3cret" {
|
||||
t.Errorf("TELESRV_ADMIN_UI_PASSWORD = %q, want it preserved as \"s3cret\" after an unrelated save", values["TELESRV_ADMIN_UI_PASSWORD"])
|
||||
}
|
||||
if values["TELESRV_STORAGE_MAX_TOTAL_BYTES"] != "209715200" {
|
||||
t.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES = %q, want 209715200", values["TELESRV_STORAGE_MAX_TOTAL_BYTES"])
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -637,15 +637,25 @@ func (m *Manager) readEnvFile() (map[string]string, error) {
|
|||
// WriteEnvValues rewrites .env from .env.example's exact text, substituting
|
||||
// each known key's value in place -- see save_env()'s docstring in
|
||||
// server-panel.py for why this (not a fresh key=value dump) is what
|
||||
// preserves comments/layout. Only keys present in values are touched; a
|
||||
// template-commented optional field is uncommented when given a non-empty
|
||||
// value and left as-is when given an empty one.
|
||||
// preserves comments/layout. Only keys present in values are set to a new
|
||||
// value; every other key keeps whatever is already in the current .env
|
||||
// (falling back to the template's own default only for a key .env never
|
||||
// set) -- previously this fell straight back to the template default for
|
||||
// any key not in this particular save's payload, silently wiping out every
|
||||
// other customized setting (e.g. TELESRV_ADMIN_UI_PASSWORD) on every save
|
||||
// that only touches one section's keys. A template-commented optional field
|
||||
// is uncommented when given a non-empty value and left as-is when given an
|
||||
// empty one.
|
||||
func (m *Manager) WriteEnvValues(values map[string]string) error {
|
||||
tmplPath := filepath.Join(m.Root, ".env.example")
|
||||
tmplData, err := os.ReadFile(tmplPath)
|
||||
if err != nil {
|
||||
return fmt.Errorf("read .env.example: %w", err)
|
||||
}
|
||||
existing, err := m.readEnvFile()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lines := strings.Split(string(tmplData), "\n")
|
||||
// Split() on a trailing "\n" leaves one empty trailing element; drop it
|
||||
// so the join below doesn't add a spurious blank line before the final
|
||||
|
|
@ -657,15 +667,20 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
|
|||
seen := map[string]bool{}
|
||||
for _, raw := range lines {
|
||||
line := strings.TrimSpace(raw)
|
||||
if a := activeFieldRe.FindStringSubmatch(line); a != nil {
|
||||
if v, ok := values[a[1]]; ok && !seen[a[1]] {
|
||||
if a := activeFieldRe.FindStringSubmatch(line); a != nil && !seen[a[1]] {
|
||||
if v, ok := values[a[1]]; ok {
|
||||
seen[a[1]] = true
|
||||
out = append(out, a[1]+"="+v)
|
||||
continue
|
||||
}
|
||||
if v, ok := existing[a[1]]; ok {
|
||||
seen[a[1]] = true
|
||||
out = append(out, a[1]+"="+v)
|
||||
continue
|
||||
}
|
||||
}
|
||||
if c := commentedFieldRe.FindStringSubmatch(line); c != nil {
|
||||
if v, ok := values[c[1]]; ok && !seen[c[1]] {
|
||||
if c := commentedFieldRe.FindStringSubmatch(line); c != nil && !seen[c[1]] {
|
||||
if v, ok := values[c[1]]; ok {
|
||||
seen[c[1]] = true
|
||||
if v != "" {
|
||||
out = append(out, c[1]+"="+v)
|
||||
|
|
@ -674,6 +689,14 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
|
|||
}
|
||||
continue
|
||||
}
|
||||
// A previously-enabled optional field shows up in the current
|
||||
// .env as an active line even though the template still has it
|
||||
// commented out -- keep it enabled with its existing value.
|
||||
if v, ok := existing[c[1]]; ok {
|
||||
seen[c[1]] = true
|
||||
out = append(out, c[1]+"="+v)
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, raw)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -107,6 +107,15 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
|
|||
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
|
||||
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
|
||||
|
||||
// fileTooBigErr surfaces TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: the upload's
|
||||
// total assembled size (not any single part) exceeds the configured
|
||||
// per-file ceiling. FILE_TOO_BIG is not an official Telegram desktop-RPC
|
||||
// error string, but it's already this codebase's own convention for the
|
||||
// same concept on the Bot API surface (see internal/botapi/server.go) --
|
||||
// reused here for consistency rather than inventing a second name for the
|
||||
// same condition.
|
||||
func fileTooBigErr() error { return tgerr.New(400, "FILE_TOO_BIG") }
|
||||
|
||||
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
|
||||
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
|
||||
// the client to retry shortly would be misleading.
|
||||
|
|
|
|||
|
|
@ -1152,6 +1152,8 @@ func mediaUploadErr(err error) error {
|
|||
switch {
|
||||
case errors.Is(err, domain.ErrFilePartsInvalid):
|
||||
return filePartsInvalidErr()
|
||||
case errors.Is(err, domain.ErrFileTooLarge):
|
||||
return fileTooBigErr()
|
||||
case errors.Is(err, domain.ErrPhotoInvalid):
|
||||
return photoInvalidErr()
|
||||
case errors.Is(err, domain.ErrDocumentInvalid):
|
||||
|
|
|
|||
|
|
@ -348,6 +348,8 @@ func fileSaveErr(err error) error {
|
|||
return filePartsInvalidErr()
|
||||
case errors.Is(err, domain.ErrFilePartTooBig):
|
||||
return filePartTooBigErr()
|
||||
case errors.Is(err, domain.ErrFileTooLarge):
|
||||
return fileTooBigErr()
|
||||
case errors.Is(err, domain.ErrUploadQuotaExceeded):
|
||||
return floodWaitErr(60)
|
||||
case errors.Is(err, domain.ErrStorageFull):
|
||||
|
|
|
|||
105
internal/store/postgres/media_hard_retention_integration_test.go
Normal file
105
internal/store/postgres/media_hard_retention_integration_test.go
Normal file
|
|
@ -0,0 +1,105 @@
|
|||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"telesrv/internal/domain"
|
||||
)
|
||||
|
||||
// TestHardRetentionPurgesBlobBytesButKeepsMetadataRow exercises the "hard"
|
||||
// storage retention mode's store methods end-to-end against a real
|
||||
// Postgres: a document old enough (by created_at, not orphaned_at) is a
|
||||
// candidate for ListDocumentIDsForHardRetentionOlderThan REGARDLESS of
|
||||
// still having a live media_references row, DeleteFileBlobsForDocument
|
||||
// removes only its file_blobs row (never the documents row itself), and a
|
||||
// second sweep pass no longer finds it a candidate since it no longer owns
|
||||
// any file_blobs row. This is the correctness property the whole "hard"
|
||||
// mode design hinges on: a message must still be able to render its media
|
||||
// placeholder after the bytes are gone.
|
||||
func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
||||
pool := testPool(t)
|
||||
ctx := context.Background()
|
||||
media := NewMediaStore(pool)
|
||||
|
||||
docID := time.Now().UnixNano()
|
||||
locationKey := "doc:" + strconv.FormatInt(docID, 10)
|
||||
|
||||
if err := media.PutDocument(ctx, domain.Document{
|
||||
ID: docID,
|
||||
MimeType: "application/octet-stream",
|
||||
Size: 1024,
|
||||
}); err != nil {
|
||||
t.Fatalf("PutDocument: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
|
||||
})
|
||||
|
||||
// Backdate created_at well past any plausible test cutoff -- PutDocument
|
||||
// always stamps now() and has no created_at parameter.
|
||||
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days' WHERE id = $1", docID); err != nil {
|
||||
t.Fatalf("backdate document: %v", err)
|
||||
}
|
||||
|
||||
blob := postgresTestBlob(locationKey, "hard-retention-doc", 1024, "application/octet-stream")
|
||||
if err := media.PutFileBlob(ctx, blob); err != nil {
|
||||
t.Fatalf("PutFileBlob: %v", err)
|
||||
}
|
||||
|
||||
// A LIVE reference (as if a message still embeds this document) must not
|
||||
// exempt it from "hard" mode -- that's the entire point of the mode,
|
||||
// unlike the orphan-only sweep.
|
||||
if err := addMediaReferencesTx(ctx, pool, &domain.MessageMedia{
|
||||
Kind: domain.MessageMediaKindDocument,
|
||||
Document: &domain.Document{ID: docID},
|
||||
}, domain.MediaRefKindMessageBox, "hard-retention-test:1"); err != nil {
|
||||
t.Fatalf("add media reference: %v", err)
|
||||
}
|
||||
t.Cleanup(func() {
|
||||
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE ref_key = 'hard-retention-test:1'")
|
||||
})
|
||||
|
||||
cutoff := time.Now().Add(-24 * time.Hour)
|
||||
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
|
||||
}
|
||||
if !containsInt64(ids, docID) {
|
||||
t.Fatalf("hard retention candidates = %v, want to include still-referenced but old document %d", ids, docID)
|
||||
}
|
||||
|
||||
blobs, err := media.DeleteFileBlobsForDocument(ctx, docID)
|
||||
if err != nil {
|
||||
t.Fatalf("DeleteFileBlobsForDocument: %v", err)
|
||||
}
|
||||
if len(blobs) != 1 || blobs[0].LocationKey != locationKey {
|
||||
t.Fatalf("deleted blobs = %+v, want exactly one for %q", blobs, locationKey)
|
||||
}
|
||||
|
||||
// The documents row itself must survive -- a message referencing it
|
||||
// still needs to render its placeholder (mime type, size, filename).
|
||||
if _, found, err := media.GetDocument(ctx, docID); err != nil || !found {
|
||||
t.Fatalf("document metadata row missing after hard blob purge: found=%v err=%v", found, err)
|
||||
}
|
||||
|
||||
// The file_blobs row is gone: a subsequent download attempt resolves via
|
||||
// GetFileBlob (files.Service.GetFile's path) to not-found, which the rpc
|
||||
// layer already maps to LOCATION_INVALID.
|
||||
if _, found, err := media.GetFileBlob(ctx, locationKey); err != nil || found {
|
||||
t.Fatalf("file_blobs row still present after hard purge: found=%v err=%v", found, err)
|
||||
}
|
||||
|
||||
// Idempotent / self-terminating: a second sweep pass no longer selects
|
||||
// this document, since it no longer owns any file_blobs row.
|
||||
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
||||
if err != nil {
|
||||
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
|
||||
}
|
||||
if containsInt64(ids, docID) {
|
||||
t.Fatalf("hard retention re-selected already-purged document %d", docID)
|
||||
}
|
||||
}
|
||||
|
|
@ -197,6 +197,101 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
|
|||
return blobs, nil
|
||||
}
|
||||
|
||||
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
|
||||
// cutoff (by upload/created_at) that still own at least one file_blobs row,
|
||||
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
|
||||
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
|
||||
// entirely: a document still referenced by a live message is exactly as
|
||||
// eligible as an orphaned one.
|
||||
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
||||
// ListDocumentIDsForHardRetentionOlderThan.
|
||||
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||
if limit <= 0 {
|
||||
return nil, nil
|
||||
}
|
||||
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
|
||||
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||
BatchLimit: int32(limit),
|
||||
})
|
||||
}
|
||||
|
||||
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
||||
// (main body + thumbnail variants), returning what was deleted so the caller
|
||||
// can physically remove each object from its backend once confirming (via
|
||||
// CountFileBlobRefs, after this call) no other row still needs it. Unlike
|
||||
// DeleteDocumentAndBlobs, this deliberately does NOT delete the documents
|
||||
// row itself -- "hard" retention mode keeps the metadata (dimensions, mime
|
||||
// type, filename) so a message can still render "this document is no longer
|
||||
// available" instead of disappearing outright. A subsequent
|
||||
// upload.getFile/GetFileBlob lookup for a location key this call removed
|
||||
// correctly finds nothing and reports not-found.
|
||||
func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||
var blobs []domain.FileBlob
|
||||
err := withTx(ctx, s.db, "delete document blob bytes (hard retention)", func(tx pgx.Tx) error {
|
||||
qtx := s.q.WithTx(tx)
|
||||
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||
ExactKey: fmt.Sprintf("doc:%d", id),
|
||||
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list document blobs: %w", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
blobs = append(blobs, domain.FileBlob{
|
||||
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||
})
|
||||
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||
return fmt.Errorf("delete file blob row: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
// DeleteFileBlobsForPhoto is the photo counterpart of
|
||||
// DeleteFileBlobsForDocument -- see its doc comment. Deliberately does not
|
||||
// delete the photos row.
|
||||
func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||
var blobs []domain.FileBlob
|
||||
err := withTx(ctx, s.db, "delete photo blob bytes (hard retention)", func(tx pgx.Tx) error {
|
||||
qtx := s.q.WithTx(tx)
|
||||
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||
ExactKey: fmt.Sprintf("photo:%d", id),
|
||||
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("list photo blobs: %w", err)
|
||||
}
|
||||
for _, r := range rows {
|
||||
blobs = append(blobs, domain.FileBlob{
|
||||
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||
})
|
||||
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||
return fmt.Errorf("delete file blob row: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return blobs, nil
|
||||
}
|
||||
|
||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||
// (one per rendition size), returning what was deleted so the caller can
|
||||
// physically remove each object from its backend once confirming (via
|
||||
|
|
|
|||
|
|
@ -237,6 +237,37 @@ WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
|||
ORDER BY orphaned_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||
-- "Hard" retention mode: candidates are documents older than cutoff (by
|
||||
-- upload/created_at, NOT orphaned_at -- a live reference does not exempt
|
||||
-- them) that still own at least one file_blobs row. The EXISTS check is what
|
||||
-- keeps this sweep from re-selecting the same document forever: once its
|
||||
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||
-- rows but deliberately leaves this documents row in place), it naturally
|
||||
-- drops out of this query on the next pass.
|
||||
SELECT d.id FROM documents d
|
||||
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
)
|
||||
ORDER BY d.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||
-- candidate selection, for photos.
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
ORDER BY p.created_at ASC
|
||||
LIMIT sqlc.arg(batch_limit)::int;
|
||||
|
||||
-- name: CountFileBlobRefs :one
|
||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
||||
|
||||
|
|
|
|||
|
|
@ -825,6 +825,50 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||
SELECT d.id FROM documents d
|
||||
WHERE d.created_at < $1::timestamptz
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'doc:' || d.id::text
|
||||
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||
)
|
||||
ORDER BY d.created_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
// "Hard" retention mode: candidates are documents older than cutoff (by
|
||||
// upload/created_at, NOT orphaned_at -- a live reference does not exempt
|
||||
// them) that still own at least one file_blobs row. The EXISTS check is what
|
||||
// keeps this sweep from re-selecting the same document forever: once its
|
||||
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||
// rows but deliberately leaves this documents row in place), it naturally
|
||||
// drops out of this query on the next pass.
|
||||
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
|
||||
SELECT location_key, backend, object_key, size
|
||||
FROM file_blobs
|
||||
|
|
@ -937,6 +981,45 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
|
|||
return items, nil
|
||||
}
|
||||
|
||||
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||
SELECT p.id FROM photos p
|
||||
WHERE p.created_at < $1::timestamptz
|
||||
AND EXISTS (
|
||||
SELECT 1 FROM file_blobs fb
|
||||
WHERE fb.location_key = 'photo:' || p.id::text
|
||||
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||
)
|
||||
ORDER BY p.created_at ASC
|
||||
LIMIT $2::int
|
||||
`
|
||||
|
||||
type ListPhotoIDsForHardRetentionOlderThanParams struct {
|
||||
Cutoff pgtype.Timestamptz
|
||||
BatchLimit int32
|
||||
}
|
||||
|
||||
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||
// candidate selection, for photos.
|
||||
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
var items []int64
|
||||
for rows.Next() {
|
||||
var id int64
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, id)
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return items, nil
|
||||
}
|
||||
|
||||
const listProfilePhotos = `-- name: ListProfilePhotos :many
|
||||
SELECT photo_id
|
||||
FROM profile_photos
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue