fix for retention logic

This commit is contained in:
onysd 2026-09-03 05:00:42 +03:00
parent 70c0ba44f0
commit e6bfe2d444
35 changed files with 2108 additions and 132 deletions

View file

@ -357,6 +357,32 @@ type Config struct {
// RetentionInterval/RetentionBatch cadence alongside every other
// retention check (see maintenance.RetentionWorker).
StorageRetentionMaxAge time.Duration
// StorageRetentionMaxAgeByCategory overrides StorageRetentionMaxAge for
// one document media category (Photo/Video/RoundVideo/Gif/Music/Voice/
// File); a category absent from the map inherits the shared global age.
// Populated from TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> env vars,
// one per category -- unset or non-positive means "inherit global" for
// that category (a document that hasn't been classified, e.g. a sticker,
// always uses the global age -- there is no override for that bucket).
// Avatar is not a domain.MediaCategory (a photo becomes/stops being an
// avatar via profile_photos, not a document attribute), so it gets its
// own StorageRetentionMaxAgeAvatar field instead of a map entry.
StorageRetentionMaxAgeByCategory map[domain.MediaCategory]time.Duration
// StorageRetentionMaxAgeAvatar overrides StorageRetentionMaxAge for
// photos currently active as someone's avatar (profile_photos.active).
// <=0 means "inherit global", same convention as
// StorageRetentionMaxAgeByCategory.
StorageRetentionMaxAgeAvatar time.Duration
// StorageEvictionEnable turns on active eviction: once the total physical
// blob bytes exceed StorageMaxTotalBytes, the oldest documents/photos
// (interleaved by created_at across both tables, regardless of category
// or age) are purged the same way "hard" retention mode purges blob
// bytes -- repeated until back under budget. Independent of
// StorageRetentionMode: eviction can run even when retention mode is
// "off". Default false (opt-in), since this changes what
// StorageMaxTotalBytes has meant so far (block new uploads only ->
// block-and-reclaim from existing ones too).
StorageEvictionEnable bool
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob
StickerSeedDir string
// StickerSeedMaxSets 限制导入的常规贴纸集数量(避免启动时导入过多包),<=0 表示不限。
@ -980,6 +1006,9 @@ func Load() (Config, error) {
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
StorageRetentionMaxAgeByCategory: storageRetentionMaxAgeByCategoryFromEnv(envDurationOr),
StorageRetentionMaxAgeAvatar: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", 0),
StorageEvictionEnable: envBoolOr("TELESRV_STORAGE_EVICTION_ENABLE", false),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
@ -1518,13 +1547,21 @@ func validateStorageConfig(cfg Config) error {
if cfg.StorageRetentionMaxAge < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
}
if cfg.StorageRetentionMaxAgeAvatar < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR must be non-negative")
}
switch cfg.StorageRetentionMode {
case StorageRetentionModeOff:
// no sweep; StorageRetentionMaxAge is ignored.
case StorageRetentionModeOrphan, StorageRetentionModeHard:
if cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
}
// StorageRetentionMaxAge (the shared default) is no longer required to
// be positive here: it can legitimately be 0 -- "no sweep by default"
// -- while individual TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY>/
// _AVATAR overrides opt specific categories in. The sweep itself
// (internal/app/files/retention.go) skips any category whose
// *effective* age (its own override, or this shared default) is <=0,
// so a 0 default with no overrides simply means the mode runs and
// purges nothing -- equivalent to "off" in practice, not dangerous.
default:
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
}
@ -2075,6 +2112,33 @@ func (e envSource) envFloatOr(key string, def float64) float64 {
return def
}
// storageRetentionMaxAgeByCategoryFromEnv builds Config.StorageRetentionMaxAgeByCategory:
// one TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> env var per document
// media category, using envDurationOr(key, 0) so "unset" and "explicitly 0"
// both cleanly mean "inherit the shared global age" -- only a positive value
// gets an entry in the map.
func storageRetentionMaxAgeByCategoryFromEnv(envDurationOr func(key string, def time.Duration) time.Duration) map[domain.MediaCategory]time.Duration {
entries := []struct {
category domain.MediaCategory
envKey string
}{
{domain.MediaCategoryPhoto, "TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO"},
{domain.MediaCategoryVideo, "TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO"},
{domain.MediaCategoryRoundVideo, "TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO"},
{domain.MediaCategoryGif, "TELESRV_STORAGE_RETENTION_MAX_AGE_GIF"},
{domain.MediaCategoryMusic, "TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC"},
{domain.MediaCategoryVoice, "TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE"},
{domain.MediaCategoryFile, "TELESRV_STORAGE_RETENTION_MAX_AGE_FILE"},
}
out := make(map[domain.MediaCategory]time.Duration, len(entries))
for _, e := range entries {
if age := envDurationOr(e.envKey, 0); age > 0 {
out[e.category] = age
}
}
return out
}
// envDurationOr 读取 time.ParseDuration 格式(如 "200ms"、"30s")的时长配置;解析失败回退默认值。
func (e envSource) envDurationOr(key string, def time.Duration) time.Duration {
if v := e.envOr(key, ""); v != "" {

View file

@ -550,14 +550,31 @@ func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
}
}
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
// TestLoadAcceptsStorageRetentionModeWithZeroMaxAgeAndCategoryOverride guards
// a real reported bug: TELESRV_STORAGE_RETENTION_MAX_AGE=0 (the shared
// default) used to be rejected outright whenever the mode was orphan/hard,
// which made it impossible to run the sweep for only specific
// TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> overrides while leaving
// everything else alone. A zero shared default is now legitimate: the sweep
// itself (internal/app/files/retention.go) skips any category whose
// *effective* age is <=0, so 0 here just means "no default sweep" rather
// than "purge everything immediately".
func TestLoadAcceptsStorageRetentionModeWithZeroMaxAgeAndCategoryOverride(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
if _, err := Load(); err == nil {
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO", "24h")
cfg, err := Load()
if err != nil {
t.Fatalf("Load rejected TELESRV_STORAGE_RETENTION_MODE=%s with zero shared max age + a category override: %v", mode, err)
}
if cfg.StorageRetentionMaxAge != 0 {
t.Fatalf("StorageRetentionMaxAge = %v, want 0", cfg.StorageRetentionMaxAge)
}
if got := cfg.StorageRetentionMaxAgeByCategory[domain.MediaCategoryVideo]; got != 24*time.Hour {
t.Fatalf("StorageRetentionMaxAgeByCategory[Video] = %v, want 24h", got)
}
})
}