astra/owpengram-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

fix for retention logic

Browse source
This commit is contained in:
onysd 2026-09-03 05:00:42 +03:00
parent 70c0ba44f0
commit e6bfe2d444
35 changed files with 2108 additions and 132 deletions
Download patch file Download diff file Expand all files Collapse all files

21
.env.example
View file

@ -319,6 +319,27 @@ TELESRV_STORAGE_RETENTION_MODE=off
# check on the shared TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH
# cadence (Advanced section below).
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
# Optional per-category overrides of the shared age above (Photo/Video/Round
# Video/Gif/Music/Voice/File/Avatar) -- each empty/unset value falls back to
# TELESRV_STORAGE_RETENTION_MAX_AGE. The mode switch above still applies to
# all of them; these only let one category expire sooner or later than the
# rest (e.g. purge voice notes after a week but keep files for a year).
TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO=
TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO=
TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO=
TELESRV_STORAGE_RETENTION_MAX_AGE_GIF=
TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC=
TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE=
TELESRV_STORAGE_RETENTION_MAX_AGE_FILE=
TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR=
# Once enabled, actively reclaims space once total physical storage exceeds
# TELESRV_STORAGE_MAX_TOTAL_BYTES above: the oldest files (regardless of
# category/age) are purged the same way "hard" retention mode purges blob
# bytes, until back under budget. Independent of the retention mode switch
# above -- can run even when that's "off". Default false: TELESRV_STORAGE_MAX_
# TOTAL_BYTES otherwise only ever blocks new uploads, never reclaims from
# existing ones.
TELESRV_STORAGE_EVICTION_ENABLE=false
# ==============================================================================

2
cmd/telesrv-admin/web/dist/assets/index-IZM8WmmY.js → cmd/telesrv-admin/web/dist/assets/index-By5gK7SA.js vendored
View file

File diff suppressed because one or more lines are too long

2
cmd/telesrv-admin/web/dist/index.html vendored
View file

@ -23,7 +23,7 @@
})();
</script>
<script type="module" crossorigin src="/assets/index-IZM8WmmY.js"></script>
<script type="module" crossorigin src="/assets/index-By5gK7SA.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
</head>
<body>

168
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
View file

@ -1,5 +1,6 @@
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search } from "lucide-react";
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search, Settings2, X } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { createPortal } from "react-dom";
import { api, errorMessage } from "../api";
import { ActionButton } from "../components/ActionButton";
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
@ -200,13 +201,28 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
// GB inputs for byte budgets, a mode dropdown + day count for retention.
const STORAGE_ENV_KEYS = {
blobBackend: "TELESRV_BLOB_BACKEND",
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE"
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE",
evictionEnable: "TELESRV_STORAGE_EVICTION_ENABLE"
} as const;
// Per-category retention age overrides -- each optional, empty/0 means
// "inherit the shared Retention age" above. Order matches the admin UI list.
const CATEGORY_AGE_FIELDS: { key: string; envKey: string; label: string }[] = [
{ key: "photo", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO", label: "Photo" },
{ key: "video", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO", label: "Video" },
{ key: "round_video", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO", label: "Round video (video message)" },
{ key: "gif", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_GIF", label: "GIF" },
{ key: "music", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC", label: "Music" },
{ key: "voice", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE", label: "Voice message" },
{ key: "file", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_FILE", label: "File" },
{ key: "avatar", envKey: "TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", label: "Avatar" }
];
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
// can never legally exceed (internal/config validates this server-side too;
@ -272,11 +288,23 @@ function DurationField({
const totalMinutes = Number(minutes || "0");
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
const amount = totalMinutes > 0 ? totalMinutes / unit.minutes : NaN;
// 0 is a real, distinct value here (e.g. the shared Retention age
// deliberately set to 0 to disable the default sweep) -- unlike
// ByteSizeField's "0/empty = Unlimited" convention, this must not collapse
// to a blank input, or there'd be no way to tell "0" from "not typed yet".
const amount = Number.isFinite(totalMinutes) ? totalMinutes / unit.minutes : NaN;
function handleAmountChange(raw: string) {
if (!raw.trim()) {
onChange("0");
return;
}
const parsed = Number(raw);
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
if (Number.isNaN(parsed) || parsed < 0) {
onChange("0");
return;
}
if (parsed === 0) {
onChange("0");
return;
}
@ -292,6 +320,7 @@ function DurationField({
min="0"
step="any"
value={Number.isNaN(amount) ? "" : amount}
placeholder={"0"}
disabled={disabled}
onChange={(event) => handleAmountChange(event.target.value)}
/>
@ -357,15 +386,71 @@ function ByteSizeField({
);
}
// CategoryRetentionModal is a focused editor for the per-category age
// overrides -- pulled out of the main Limits & Retention flow (which was
// getting crowded with 8 extra duration fields) into its own dialog, following
// the same modal-backdrop/command-modal pattern as MintCollectibleUsernameModal.
// It edits the same categoryAgeMinutes state the parent already owns; there's
// no separate save here, just Close -- the one shared "Save limits &
// retention settings" button below still covers these values too.
function CategoryRetentionModal({
categoryAgeMinutes,
onChange,
disabled,
onClose
}: {
categoryAgeMinutes: Record<string, string>;
onChange: (key: string, minutes: string) => void;
disabled: boolean;
onClose: () => void;
}) {
return createPortal(
<div className="modal-backdrop" role="presentation">
<section className="modal command-modal" role="dialog" aria-modal="true" aria-label={"Per-category retention overrides"}>
<div className="modal-head">
<div>
<div className="eyebrow">{"Limits & Retention"}</div>
<h2>{"Per-category overrides"}</h2>
</div>
<button className="icon-btn" type="button" onClick={onClose} aria-label={"Close"}><X size={15} /></button>
</div>
<div className="command-body">
<p className="env-field-desc">
{"Leave a category at 0 to inherit the shared Retention age. The mode switch still applies to all of them -- these only change how old that one category's media must be."}
</p>
{CATEGORY_AGE_FIELDS.map((field) => (
<DurationField
key={field.key}
label={field.label}
help={"Inherits the shared Retention age when left at 0."}
minutes={categoryAgeMinutes[field.key] || "0"}
disabled={disabled}
onChange={(minutes) => onChange(field.key, minutes)}
/>
))}
</div>
<div className="modal-actions">
<button className="btn primary" type="button" onClick={onClose}>{"Close"}</button>
</div>
</section>
</div>,
document.body
);
}
function LimitsRetentionSection() {
const [loaded, setLoaded] = useState(false);
const [error, setError] = useState("");
const [initial, setInitial] = useState<Record<string, string>>({});
const [blobBackend, setBlobBackend] = useState("s3");
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
const [minFreeBytes, setMinFreeBytes] = useState("0");
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
const [retentionMode, setRetentionMode] = useState("off");
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
const [categoryAgeMinutes, setCategoryAgeMinutes] = useState<Record<string, string>>({});
const [categoryModalOpen, setCategoryModalOpen] = useState(false);
const [evictionEnable, setEvictionEnable] = useState(false);
async function load() {
setError("");
@ -378,6 +463,7 @@ function LimitsRetentionSection() {
}
}
setInitial(values);
setBlobBackend((values[STORAGE_ENV_KEYS.blobBackend] || "s3").trim().toLowerCase());
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
@ -385,6 +471,14 @@ function LimitsRetentionSection() {
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
const nextCategoryAges: Record<string, string> = {};
for (const field of CATEGORY_AGE_FIELDS) {
const raw = values[field.envKey] || "";
const categoryMins = parseDurationMinutes(raw);
nextCategoryAges[field.key] = categoryMins > 0 ? String(Math.round(categoryMins)) : "0";
}
setCategoryAgeMinutes(nextCategoryAges);
setEvictionEnable((values[STORAGE_ENV_KEYS.evictionEnable] || "false").trim().toLowerCase() === "true");
setLoaded(true);
} catch (err) {
setError(errorMessage(err));
@ -404,14 +498,26 @@ function LimitsRetentionSection() {
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
: `${Math.max(1, Math.round(Number(retentionAgeMinutes || "0")))}m`
// 0 is a legitimate value here: "no sweep by default", left for
// per-category overrides below to opt specific categories in. Don't
// clamp it up to a minimum of 1 -- that used to make it impossible to
// ever save a 0 global age.
: `${Math.max(0, Math.round(Number(retentionAgeMinutes || "0")))}m`,
[STORAGE_ENV_KEYS.evictionEnable]: evictionEnable ? "true" : "false"
};
// Per-category overrides: a field left at "inherit global" (0) saves as
// empty rather than a redundant explicit duration, matching the "unset
// means inherit" contract on the server side.
for (const field of CATEGORY_AGE_FIELDS) {
const mins = Math.round(Number(categoryAgeMinutes[field.key] || "0"));
next[field.envKey] = mins > 0 ? `${mins}m` : "";
}
const changed: Record<string, string> = {};
for (const [key, value] of Object.entries(next)) {
if ((initial[key] ?? "") !== value) changed[key] = value;
}
return changed;
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, initial]);
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, categoryAgeMinutes, evictionEnable, initial]);
const hasChanges = Object.keys(pendingValues).length > 0;
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
@ -434,12 +540,14 @@ function LimitsRetentionSection() {
bytes={maxTotalBytes}
onChange={setMaxTotalBytes}
/>
<ByteSizeField
label={"Min free space guard"}
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
bytes={minFreeBytes}
onChange={setMinFreeBytes}
/>
{blobBackend === "localfs" && (
<ByteSizeField
label={"Min free space guard"}
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
bytes={minFreeBytes}
onChange={setMinFreeBytes}
/>
)}
<ByteSizeField
label={"Max single file size"}
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
@ -471,9 +579,10 @@ function LimitsRetentionSection() {
<DurationField
label={"Retention age"}
help={
retentionMode === "hard"
(retentionMode === "hard"
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
: "How long a document or photo must have had zero references before its file is deleted."
: "How long a document or photo must have had zero references before its file is deleted.") +
" Set to 0 to disable the sweep by default and only clean up categories you explicitly override below."
}
minutes={retentionAgeMinutes}
disabled={retentionMode === "off"}
@ -481,6 +590,37 @@ function LimitsRetentionSection() {
/>
</div>
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<button className="btn icon-text" type="button" onClick={() => setCategoryModalOpen(true)}>
<Settings2 size={15} /> {"Per-category overrides..."}
</button>
<span className="env-field-desc">
{"Optional -- give Photo/Video/GIF/Music/Voice/File/Avatar their own retention age instead of the shared one above."}
</span>
</div>
{categoryModalOpen && (
<CategoryRetentionModal
categoryAgeMinutes={categoryAgeMinutes}
onChange={(key, minutes) => setCategoryAgeMinutes((prev) => ({ ...prev, [key]: minutes }))}
disabled={retentionMode === "off"}
onClose={() => setCategoryModalOpen(false)}
/>
)}
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<label className="checkline">
<input
type="checkbox"
checked={evictionEnable}
onChange={(event) => setEvictionEnable(event.target.checked)}
/>
{" "}{"Actively reclaim space once over budget"}
</label>
<p className="env-field-desc">
{"Once total physical storage exceeds the Max total storage budget above, actively delete the oldest files (regardless of category or age) until back under budget -- the same way \"hard\" retention mode purges files. Independent of the retention mode above: this can run even when that's Off. Off by default, since this changes the storage budget from block-new-uploads-only to also reclaiming from existing files."}
</p>
</div>
<div className="gift-table-actions env-save-row">
<ActionButton
tone="warn"

51
cmd/telesrv/main.go
View file

@ -550,6 +550,25 @@ func webPagePreviewOption(cfg config.Config) filesapp.Option {
return filesapp.WithWebPagePreview(cfg.WebPagePreviewMaxBytes, cfg.WebPagePreviewRatePerMin)
}
// fastestPositiveDuration returns the smaller of a and b, treating a
// non-positive value as "not configured" rather than as the smallest
// possible duration -- 0 only when both are non-positive (nothing
// configured). Used to derive the storage retention sweep's ticker cadence
// from whichever of the shared default age and its per-category overrides
// asks to run soonest.
func fastestPositiveDuration(a, b time.Duration) time.Duration {
if a <= 0 {
return b
}
if b <= 0 {
return a
}
if a < b {
return a
}
return b
}
func run(logger *zap.Logger) error {
cfg, err := config.Load()
if err != nil {
@ -933,6 +952,8 @@ func run(logger *zap.Logger) error {
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
filesapp.WithSpaceGuard(spaceGuard),
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
filesapp.WithStorageRetentionAges(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeByCategory, cfg.StorageRetentionMaxAgeAvatar),
filesapp.WithStorageMaxTotalBytes(cfg.StorageMaxTotalBytes),
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
externalMediaOption(cfg),
webPagePreviewOption(cfg),
@ -1081,12 +1102,31 @@ func run(logger *zap.Logger) error {
// TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
// the orphan-only (safe) and hard (age-based, ignores live references)
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
// The worker's own maxAge parameter only drives how often the sweep
// ticks (internal/app/maintenance.RetentionWorker.mediaRetentionInterval)
// -- the real per-category cutoff math lives entirely in
// files.Service (WithStorageRetentionAges above). Passing the raw shared
// default here would tick as slowly as a 30-day default even when a
// TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> override asks for a much
// shorter age (or, if the shared default is 0 -- "disabled by default,
// only specific categories opt in" -- would disable the sweep outright,
// since a 0 maxAge here used to gate the whole sweep off). Use the
// fastest positive age across the shared default and every override
// instead, so the ticker -- and the sweep-enabled gate -- reflect
// whatever is actually configured to run soonest.
fastestRetentionAge := fastestPositiveDuration(cfg.StorageRetentionMaxAge, cfg.StorageRetentionMaxAgeAvatar)
for _, age := range cfg.StorageRetentionMaxAgeByCategory {
fastestRetentionAge = fastestPositiveDuration(fastestRetentionAge, age)
}
switch cfg.StorageRetentionMode {
case config.StorageRetentionModeOrphan:
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, cfg.StorageRetentionMaxAge)
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, fastestRetentionAge)
case config.StorageRetentionModeHard:
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, cfg.StorageRetentionMaxAge)
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, fastestRetentionAge)
}
// Active eviction is independent of TELESRV_STORAGE_RETENTION_MODE (can
// run even when that's "off") and reuses the same media sweep ticker.
retentionWorker = retentionWorker.WithStorageEviction(filesService, cfg.StorageEvictionEnable)
go retentionWorker.Run(ctx)
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
cfg.UploadPartTTL,
@ -1383,6 +1423,13 @@ func run(logger *zap.Logger) error {
messageapp.WithSendPermissionChecker(adminService),
messageapp.WithBusinessAutomation(passwordStore, businessAutomationOptions...),
)
// Wires the storage retention sweep's purge-notice capability now that
// both edit-capable app services exist -- filesService (and the
// background retentionWorker goroutine reading it) was constructed
// earlier, before either was available. A sweep tick that races ahead of
// this call simply finds no notifier yet and skips the notice for that
// tick (best-effort, see files.SetRetentionPurgeNotifier).
filesService.SetRetentionPurgeNotifier(messagesService, channelsService)
moderationService := moderationapp.NewService(
moderationReportStore,
moderationapp.WithMessageReaders(messagesService, channelsService),

33
cmd/telesrv/main_test.go
View file

@ -2,10 +2,43 @@ package main
import (
"testing"
"time"
telegramloginapp "telesrv/internal/app/telegramlogin"
)
// TestFastestPositiveDurationIgnoresNonPositiveValues guards a real reported
// bug: the storage retention sweep's ticker cadence used to be driven purely
// by the shared TELESRV_STORAGE_RETENTION_MAX_AGE default (e.g. 30 days),
// even when a much shorter TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY>
// override was configured -- so a category set to "1 minute" would still
// only actually get swept on the shared default's own slow cadence (or, if
// the shared default was 0, would disable the sweep outright). The worker
// must be handed the fastest positive age across the default and every
// override instead.
func TestFastestPositiveDurationIgnoresNonPositiveValues(t *testing.T) {
cases := []struct {
name string
a, b time.Duration
want time.Duration
}{
{"both positive, a smaller", 30 * 24 * time.Hour, time.Minute, time.Minute},
{"both positive, b smaller", time.Minute, 30 * 24 * time.Hour, time.Minute},
{"a zero (disabled), b positive", 0, time.Minute, time.Minute},
{"a positive, b zero (disabled)", time.Minute, 0, time.Minute},
{"a negative, b positive", -time.Hour, time.Minute, time.Minute},
{"both zero (nothing configured)", 0, 0, 0},
{"both negative", -time.Hour, -time.Minute, -time.Minute},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := fastestPositiveDuration(c.a, c.b); got != c.want {
t.Fatalf("fastestPositiveDuration(%v, %v) = %v, want %v", c.a, c.b, got, c.want)
}
})
}
}
func TestTelegramLoginRPCDependencyPreservesDisabledNil(t *testing.T) {
var disabled *telegramloginapp.Service
if dependency := telegramLoginRPCDependency(disabled); dependency != nil {

3
deploy/migrations/20260902000001_documents_category.down.sql Normal file
View file

@ -0,0 +1,3 @@
DROP INDEX IF EXISTS public.documents_category_created_at_idx;
ALTER TABLE public.documents
DROP COLUMN IF EXISTS category;

18
deploy/migrations/20260902000001_documents_category.up.sql Normal file
View file

@ -0,0 +1,18 @@
-- Adds a per-document media category column so the storage retention sweep
-- can use a different retention age per category (Photo/Video/RoundVideo/
-- Gif/Music/Voice/File/Avatar -- see TELESRV_STORAGE_RETENTION_MAX_AGE_*)
-- instead of one shared age for every document regardless of kind.
--
-- The value mirrors domain.MediaCategory (int16): 0 = None/unclassified
-- (stickers and anything else classifyDocumentCategory doesn't tag -- these
-- always fall back to the shared global age, there is no per-category
-- override for this bucket), 2 = Video, 3 = Gif, 4 = File, 5 = Music,
-- 6 = Voice, 7 = RoundVideo. Deliberately NOT backfilled for existing rows:
-- they stay at the default 0 (global age) until they are next
-- created/edited, which is an acceptable one-time transitional behavior --
-- backfilling would require re-deriving the category from each document's
-- already-stored attributes for potentially every row in the table.
ALTER TABLE public.documents
ADD COLUMN category smallint NOT NULL DEFAULT 0;
CREATE INDEX documents_category_created_at_idx ON public.documents (category, created_at);

6
deploy/migrations/20260903000001_account_settings_disallowed_gifts.down.sql Normal file
View file

@ -0,0 +1,6 @@
ALTER TABLE public.account_settings
DROP COLUMN IF EXISTS disallow_unlimited_stargifts,
DROP COLUMN IF EXISTS disallow_limited_stargifts,
DROP COLUMN IF EXISTS disallow_unique_stargifts,
DROP COLUMN IF EXISTS disallow_premium_gifts,
DROP COLUMN IF EXISTS disallow_stargifts_from_channels;

15
deploy/migrations/20260903000001_account_settings_disallowed_gifts.up.sql Normal file
View file

@ -0,0 +1,15 @@
-- account_settings gained global gift-reception switches (Layer 228
-- disallowedGifts) in code (internal/domain/account.go's DisallowedGifts,
-- read/written by internal/store/postgres/account.go's GetAccountSettings/
-- GetAccountSettingsBatch/SaveAccountSettings) without a matching migration
-- ever being added -- every account_settings read/write has been failing
-- with "column does not exist" on any database that never got these columns,
-- which breaks not just gift settings but every RPC that loads cached
-- account settings (privacy, account TTL, contact sign-up notification,
-- sendMessage's privacy check, etc).
ALTER TABLE public.account_settings
ADD COLUMN IF NOT EXISTS disallow_unlimited_stargifts boolean NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS disallow_limited_stargifts boolean NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS disallow_unique_stargifts boolean NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS disallow_premium_gifts boolean NOT NULL DEFAULT false,
ADD COLUMN IF NOT EXISTS disallow_stargifts_from_channels boolean NOT NULL DEFAULT false;

16
internal/app/channels/service.go
View file

@ -1514,6 +1514,22 @@ func (s *Service) EditMessage(ctx context.Context, userID int64, req domain.Edit
return s.channels.EditChannelMessage(ctx, req)
}
// EditChannelMessageInternal performs a server-internal channel message edit
// with no acting user -- currently only the storage retention sweep
// (internal/app/files.notifyRetentionPurge), which needs to turn a purged
// message into a visible notice but has no RPC caller/channel member behind
// it. Bypasses the userID==0 gate EditMessage enforces for ordinary
// RPC-driven edits; the store's own RetentionPurge bypass (see
// EditChannelMessageRequest.RetentionPurge) enforces the actual permission
// bypass semantics, so this refuses anything that isn't actually a retention
// purge request rather than becoming a second general-purpose edit path.
func (s *Service) EditChannelMessageInternal(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
if s == nil || s.channels == nil || !req.RetentionPurge || req.RetentionPurgeAction == nil || req.ChannelID == 0 || req.ID <= 0 {
return domain.EditChannelMessageResult{}, domain.ErrChannelInvalid
}
return s.channels.EditChannelMessage(ctx, req)
}
// GetInlineBotMessage returns one live channel message addressed by a signed inline id.
func (s *Service) GetInlineBotMessage(ctx context.Context, botID, channelID int64, id int) (domain.Channel, domain.ChannelMessage, bool, error) {
if s == nil || s.channels == nil || botID == 0 || channelID == 0 || id <= 0 || id > domain.MaxMessageBoxID {

312
internal/app/files/retention.go
View file

@ -3,6 +3,7 @@ package files
import (
"context"
"fmt"
"sort"
"time"
"go.uber.org/zap"
@ -10,14 +11,52 @@ import (
"telesrv/internal/domain"
)
// hardRetentionDocumentCategories enumerates every documents.category bucket
// the per-category retention sweep loops over. MediaCategoryNone covers
// unclassified documents (stickers and anything else classifyDocumentCategory
// doesn't tag) and always uses the shared global age -- there is no
// per-category override for that bucket. Photo/Avatar are not in this list:
// photos have no category column and are instead split by a profile_photos
// join (see categoryRetentionAge/avatarRetentionAge below and the dedicated
// photo/avatar query variants).
var hardRetentionDocumentCategories = []domain.MediaCategory{
domain.MediaCategoryNone,
domain.MediaCategoryVideo,
domain.MediaCategoryGif,
domain.MediaCategoryFile,
domain.MediaCategoryMusic,
domain.MediaCategoryVoice,
domain.MediaCategoryRoundVideo,
}
// categoryRetentionAge returns the effective retention age for a document
// media category: its configured override if positive, otherwise the shared
// global age.
func (s *Service) categoryRetentionAge(category domain.MediaCategory) time.Duration {
if age, ok := s.storageRetentionCategoryAges[category]; ok && age > 0 {
return age
}
return s.storageRetentionGlobalMaxAge
}
// avatarRetentionAge is the Photo category's counterpart for photos currently
// active as someone's avatar -- see categoryRetentionAge.
func (s *Service) avatarRetentionAge() time.Duration {
if s.storageRetentionAvatarMaxAge > 0 {
return s.storageRetentionAvatarMaxAge
}
return s.storageRetentionGlobalMaxAge
}
// mediaRetentionStore is implemented by store.MediaStore backends that
// support the storage retention sweep (currently only the Postgres store).
// A type assertion, not a MediaStore interface method, keeps these
// admin/maintenance-only queries out of the hot RPC-facing interface --
// same convention as photoBatchStore above.
type mediaRetentionStore interface {
ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
CountFileBlobRefs(ctx context.Context, backend, objectKey string) (int, error)
DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeletePhotoAndBlobs(ctx context.Context, id int64) ([]domain.FileBlob, error)
@ -31,111 +70,250 @@ type mediaRetentionStore interface {
// orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
// -- active eviction (TELESRV_STORAGE_EVICTION_ENABLE) --
SumFileBlobBytes(ctx context.Context) (int64, error)
ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error)
// -- retention purge notice (see retention_purge.go) --
ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error)
}
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
// permanently deletes documents/photos that have had no live reference
// (message/profile-photo/sticker-set, see media_references) since at least
// cutoff, along with their blob(s) -- but only physically removes bytes
// from the backend once confirming no other file_blobs row still needs the
// object, since content-addressed storage means the same bytes can be
// shared across documents/photos.
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
// the per-category cutoff derived from now, along with their blob(s) -- but
// only physically removes bytes from the backend once confirming no other
// file_blobs row still needs the object, since content-addressed storage
// means the same bytes can be shared across documents/photos. Loops one
// query per document category (each with its own effective age, see
// categoryRetentionAge) plus a regular/avatar split for photos; limit applies
// per category per tick, not as one shared budget across all of them --
// simplest correct behavior, revisit only if one category starves another in
// practice.
func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, nil
}
deleted := 0
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cutoff, limit)
if err != nil {
return deleted, fmt.Errorf("list orphaned documents: %w", err)
}
for _, id := range docIDs {
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
if err != nil {
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
for _, cat := range hardRetentionDocumentCategories {
age := s.categoryRetentionAge(cat)
if age <= 0 {
// Global age is 0 (retention only enabled for other, explicitly
// overridden categories) and this category has no override of its
// own -- skip it entirely rather than treating age<=0 as an
// immediate "everything is older than now" cutoff.
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, cutoff, limit)
if err != nil {
return deleted, fmt.Errorf("list orphaned photos: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
cutoff := now.Add(-age)
docIDs, err := store.ListOrphanedDocumentIDsOlderThan(ctx, cat, cutoff, limit)
if err != nil {
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
continue
return deleted, fmt.Errorf("list orphaned documents (category %d): %w", cat, err)
}
for _, id := range docIDs {
blobs, err := store.DeleteDocumentAndBlobs(ctx, id)
if err != nil {
s.log.Warn("delete orphaned document failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
}
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
photoCutoff := now.Add(-age)
photoIDs, err := store.ListOrphanedPhotoIDsOlderThan(ctx, photoCutoff, limit)
if err != nil {
return deleted, fmt.Errorf("list orphaned photos: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
if err != nil {
s.log.Warn("delete orphaned photo failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
}
if age := s.avatarRetentionAge(); age > 0 {
avatarCutoff := now.Add(-age)
avatarIDs, err := store.ListAvatarOrphanedPhotoIDsOlderThan(ctx, avatarCutoff, limit)
if err != nil {
return deleted, fmt.Errorf("list orphaned avatar photos: %w", err)
}
for _, id := range avatarIDs {
blobs, err := store.DeletePhotoAndBlobs(ctx, id)
if err != nil {
s.log.Warn("delete orphaned avatar photo failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
s.deleteOrphanedBlobs(ctx, store, blobs)
deleted++
}
return deleted, nil
}
// DeleteBlobBytesForMediaOlderThan implements
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
// documents/photos whose upload/created_at is older than cutoff, physically
// deletes their blob bytes (main body + thumbnail/rendition variants) from
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
// live message/profile-photo/sticker-set still references them. It
// deliberately never touches the documents/photos metadata row itself: a
// message must still be able to render "here was a photo/document"
// (dimensions, mime type, filename) after its bytes are gone, rather than
// the message breaking outright. A subsequent upload.getFile for the same
// location key finds no file_blobs row and returns LOCATION_INVALID, which
// stock clients already render as a "media unavailable" placeholder.
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
// documents/photos whose upload/created_at is older than the per-category
// cutoff derived from now (see categoryRetentionAge/avatarRetentionAge),
// physically deletes their blob bytes (main body + thumbnail/rendition
// variants) from the backend and removes their file_blobs rows --
// REGARDLESS of whether a live message/profile-photo/sticker-set still
// references them. It deliberately never touches the documents/photos
// metadata row itself: a message must still be able to render "here was a
// photo/document" (dimensions, mime type, filename) after its bytes are
// gone, rather than the message breaking outright. A subsequent
// upload.getFile for the same location key finds no file_blobs row and
// returns LOCATION_INVALID, which stock clients already render as a "media
// unavailable" placeholder -- and every message that still embeds the purged
// media additionally gets turned into a visible retention-purge notice (see
// notifyRetentionPurge in retention_purge.go).
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, now time.Time, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, nil
}
purged := 0
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
if err != nil {
return purged, fmt.Errorf("list documents for hard retention: %w", err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
for _, cat := range hardRetentionDocumentCategories {
age := s.categoryRetentionAge(cat)
if age <= 0 {
// See DeleteOrphanedOlderThan's identical guard: age<=0 means this
// category has no override and the global age is 0 (retention
// enabled only for other categories) -- skip, don't treat it as
// "everything is older than now".
continue
}
cutoff := now.Add(-age)
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cat, cutoff, limit)
if err != nil {
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
continue
return purged, fmt.Errorf("list documents for hard retention (category %d): %w", cat, err)
}
if len(blobs) == 0 {
continue
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
s.notifyRetentionPurge(ctx, domain.MediaKindDocument, id)
purged++
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
remaining := limit - len(docIDs)
if remaining <= 0 {
return purged, nil
}
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if age := s.categoryRetentionAge(domain.MediaCategoryPhoto); age > 0 {
photoCutoff := now.Add(-age)
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, photoCutoff, limit)
if err != nil {
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
continue
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
if len(blobs) == 0 {
continue
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purged++
}
}
if age := s.avatarRetentionAge(); age > 0 {
avatarCutoff := now.Add(-age)
avatarIDs, err := store.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, avatarCutoff, limit)
if err != nil {
return purged, fmt.Errorf("list avatar photos for hard retention: %w", err)
}
for _, id := range avatarIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("hard-delete avatar photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
s.notifyRetentionPurge(ctx, domain.MediaKindPhoto, id)
purged++
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
return purged, nil
}
// EvictOldestMediaOverBudget implements maintenance.StorageEvictionStore
// (TELESRV_STORAGE_EVICTION_ENABLE): once total physical blob bytes
// (SumFileBlobBytes) exceed TELESRV_STORAGE_MAX_TOTAL_BYTES, purges the
// oldest documents/photos overall -- interleaved by created_at across both
// tables, regardless of category or age -- reusing the exact same blob-purge
// (DeleteFileBlobsForDocument/DeleteFileBlobsForPhoto) and retention-purge
// notice primitive as "hard" mode. Stops once the running total (tracked
// locally from each purge's returned blob sizes, avoiding a re-query per
// item) is back under budget, or limit purges have happened this tick,
// whichever comes first -- bounding how much one tick can reclaim at once.
func (s *Service) EvictOldestMediaOverBudget(ctx context.Context, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 || s.storageMaxTotalBytes <= 0 {
return 0, nil
}
total, err := store.SumFileBlobBytes(ctx)
if err != nil {
return 0, fmt.Errorf("sum file blob bytes: %w", err)
}
if total <= s.storageMaxTotalBytes {
return 0, nil
}
candidates, err := store.ListOldestMediaForEviction(ctx, limit)
if err != nil {
return 0, fmt.Errorf("list oldest media for eviction: %w", err)
}
sort.Slice(candidates, func(i, j int) bool { return candidates[i].CreatedAt.Before(candidates[j].CreatedAt) })
evicted := 0
for _, c := range candidates {
if evicted >= limit || total <= s.storageMaxTotalBytes {
break
}
var blobs []domain.FileBlob
var delErr error
switch c.Kind {
case domain.MediaKindDocument:
blobs, delErr = store.DeleteFileBlobsForDocument(ctx, c.MediaID)
case domain.MediaKindPhoto:
blobs, delErr = store.DeleteFileBlobsForPhoto(ctx, c.MediaID)
default:
continue
}
if delErr != nil {
s.log.Warn("active storage eviction blob purge failed",
zap.String("media_kind", string(c.Kind)), zap.Int64("media_id", c.MediaID), zap.Error(delErr))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
for _, b := range blobs {
total -= b.Size
}
s.notifyRetentionPurge(ctx, c.Kind, c.MediaID)
evicted++
}
return evicted, nil
}
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
// id right now (skipping the grace period) and, only if that succeeds --

153
internal/app/files/retention_category_zero_test.go Normal file
View file

@ -0,0 +1,153 @@
package files
import (
"context"
"testing"
"time"
"go.uber.org/zap"
"telesrv/internal/domain"
"telesrv/internal/store"
)
// fakeCategorySweepStore records which category/photo/avatar queries
// DeleteBlobBytesForMediaOlderThan/DeleteOrphanedOlderThan actually issue, so
// the zero-age skip guard can be checked without a live database. Embeds a
// nil store.MediaStore so it satisfies that (large) interface for free via
// promoted methods that must never actually be called here -- only the
// mediaRetentionStore subset overridden below is exercised.
type fakeCategorySweepStore struct {
store.MediaStore
queriedDocCategories []domain.MediaCategory
queriedPhotos bool
queriedAvatars bool
}
func (f *fakeCategorySweepStore) ListDocumentIDsForHardRetentionOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil
}
func (f *fakeCategorySweepStore) ListPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
f.queriedPhotos = true
return nil, nil
}
func (f *fakeCategorySweepStore) ListAvatarPhotoIDsForHardRetentionOlderThan(context.Context, time.Time, int) ([]int64, error) {
f.queriedAvatars = true
return nil, nil
}
func (f *fakeCategorySweepStore) ListOrphanedDocumentIDsOlderThan(_ context.Context, category domain.MediaCategory, _ time.Time, _ int) ([]int64, error) {
f.queriedDocCategories = append(f.queriedDocCategories, category)
return nil, nil
}
func (f *fakeCategorySweepStore) ListOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
f.queriedPhotos = true
return nil, nil
}
func (f *fakeCategorySweepStore) ListAvatarOrphanedPhotoIDsOlderThan(context.Context, time.Time, int) ([]int64, error) {
f.queriedAvatars = true
return nil, nil
}
// The rest of mediaRetentionStore's methods aren't part of store.MediaStore
// (see that interface's own doc comment -- they're kept out of the hot
// RPC-facing interface on purpose), so embedding store.MediaStore alone
// doesn't provide them. Stub them out; this test never exercises them since
// every List* above returns no candidates.
func (f *fakeCategorySweepStore) CountFileBlobRefs(context.Context, string, string) (int, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) DeleteDocumentAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeCategorySweepStore) DeletePhotoAndBlobs(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeCategorySweepStore) OrphanDocumentIfUnreferenced(context.Context, int64) (bool, error) {
return false, nil
}
func (f *fakeCategorySweepStore) DeleteFileBlobsForDocument(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeCategorySweepStore) DeleteFileBlobsForPhoto(context.Context, int64) ([]domain.FileBlob, error) {
return nil, nil
}
func (f *fakeCategorySweepStore) SumFileBlobBytes(context.Context) (int64, error) {
return 0, nil
}
func (f *fakeCategorySweepStore) ListOldestMediaForEviction(context.Context, int) ([]domain.EvictionCandidate, error) {
return nil, nil
}
func (f *fakeCategorySweepStore) ListMediaReferences(context.Context, domain.MediaKind, int64) ([]domain.MediaReference, error) {
return nil, nil
}
// TestZeroGlobalRetentionAgeSkipsCategoriesWithoutOverride guards the bug
// reported live: setting the shared Retention age to 0 (meaning "only clean
// up categories I explicitly override") used to be indistinguishable from
// "everything is older than right now" at the per-category cutoff math,
// which would have hard-purged every uncategorized/photo/avatar item
// immediately. The sweep must instead skip any category whose *effective*
// age (its own override, or the 0 global) is <=0 entirely -- no query issued
// for it at all -- while still sweeping a category that has a real,
// positive override.
func TestZeroGlobalRetentionAgeSkipsCategoriesWithoutOverride(t *testing.T) {
fake := &fakeCategorySweepStore{}
s := &Service{
media: fake,
log: zap.NewNop(),
storageRetentionGlobalMaxAge: 0,
storageRetentionCategoryAges: map[domain.MediaCategory]time.Duration{
domain.MediaCategoryVideo: 24 * time.Hour,
},
storageRetentionAvatarMaxAge: 0,
}
if _, err := s.DeleteBlobBytesForMediaOlderThan(context.Background(), time.Now(), 50); err != nil {
t.Fatalf("DeleteBlobBytesForMediaOlderThan: %v", err)
}
if len(fake.queriedDocCategories) != 1 || fake.queriedDocCategories[0] != domain.MediaCategoryVideo {
t.Fatalf("queried document categories = %v, want only [Video]", fake.queriedDocCategories)
}
if fake.queriedPhotos {
t.Fatal("queried regular photos with no photo override and global age 0, want skipped")
}
if fake.queriedAvatars {
t.Fatal("queried avatar photos with no avatar override and global age 0, want skipped")
}
}
// TestZeroGlobalRetentionAgeSkipsOrphanSweepCategoriesWithoutOverride is the
// orphan-mode counterpart of the hard-mode test above.
func TestZeroGlobalRetentionAgeSkipsOrphanSweepCategoriesWithoutOverride(t *testing.T) {
fake := &fakeCategorySweepStore{}
s := &Service{
media: fake,
log: zap.NewNop(),
storageRetentionGlobalMaxAge: 0,
storageRetentionCategoryAges: map[domain.MediaCategory]time.Duration{
domain.MediaCategoryVoice: 48 * time.Hour,
},
storageRetentionAvatarMaxAge: 0,
}
if _, err := s.DeleteOrphanedOlderThan(context.Background(), time.Now(), 50); err != nil {
t.Fatalf("DeleteOrphanedOlderThan: %v", err)
}
if len(fake.queriedDocCategories) != 1 || fake.queriedDocCategories[0] != domain.MediaCategoryVoice {
t.Fatalf("queried document categories = %v, want only [Voice]", fake.queriedDocCategories)
}
if fake.queriedPhotos {
t.Fatal("queried regular photos with no photo override and global age 0, want skipped")
}
if fake.queriedAvatars {
t.Fatal("queried avatar photos with no avatar override and global age 0, want skipped")
}
}

204
internal/app/files/retention_purge.go Normal file
View file

@ -0,0 +1,204 @@
package files
import (
"context"
"errors"
"fmt"
"go.uber.org/zap"
"telesrv/internal/domain"
)
// RetentionPurgeMessageEditor is the private-message capability the storage
// retention sweep needs to turn a hard-retention/eviction blob purge into a
// visible messageActionCustomAction notice. Satisfied by
// internal/app/messages.Service (GetMessages resolves the box's peer --
// EditMessageRequest requires it, and message_box ref_key only encodes
// owner_user_id+box_id -- then EditMessage performs the actual in-place
// edit with RetentionPurge set).
type RetentionPurgeMessageEditor interface {
GetMessages(ctx context.Context, userID int64, ids []int) (domain.MessageList, error)
EditMessage(ctx context.Context, userID int64, req domain.EditMessageRequest) (domain.EditMessageResult, error)
}
// RetentionPurgeChannelEditor is the channel-message counterpart of
// RetentionPurgeMessageEditor. Satisfied by internal/app/channels.Service.
// Unlike private messages, EditChannelMessageRequest needs no peer lookup --
// channelMessageRefKey already encodes the channel id directly. Uses
// EditChannelMessageInternal (not the ordinary EditMessage) since this is a
// server-internal edit with no acting user/channel member behind it.
type RetentionPurgeChannelEditor interface {
EditChannelMessageInternal(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error)
}
// SetRetentionPurgeNotifier wires the private-message/channel-message edit
// capability the storage retention sweep needs to turn a hard-retention/
// eviction blob purge into a visible messageActionCustomAction notice on
// every message still embedding the purged document/photo. Both app-layer
// services are constructed after this Service in cmd/telesrv/main.go (and
// the background retention sweep goroutine is started before either exists),
// so this is a post-construction setter rather than a NewService option: a
// sweep tick that races ahead of this call simply finds both fields nil and
// skips the notice for that tick, same as any other per-reference failure
// below (best-effort -- the underlying blob purge has already committed and
// must never be undone or retried because of a notice failure).
func (s *Service) SetRetentionPurgeNotifier(messages RetentionPurgeMessageEditor, channels RetentionPurgeChannelEditor) {
s.retentionNotifyMu.Lock()
defer s.retentionNotifyMu.Unlock()
s.retentionMessages = messages
s.retentionChannels = channels
}
func (s *Service) retentionNotifier() (RetentionPurgeMessageEditor, RetentionPurgeChannelEditor) {
s.retentionNotifyMu.RLock()
defer s.retentionNotifyMu.RUnlock()
return s.retentionMessages, s.retentionChannels
}
// notifyRetentionPurge turns a just-completed hard-retention/eviction blob
// purge of a document/photo into a visible service-message notice on every
// message that still embeds it. profile_photo/sticker_set/gift references
// have no message to edit and are skipped. Best-effort throughout: a lookup
// or edit failure is logged and never propagated -- the underlying blob purge
// has already committed and must not be undone or retried because of this.
func (s *Service) notifyRetentionPurge(ctx context.Context, kind domain.MediaKind, mediaID int64) {
messages, channels := s.retentionNotifier()
if messages == nil && channels == nil {
return
}
store, ok := s.media.(mediaRetentionStore)
if !ok {
return
}
refs, err := store.ListMediaReferences(ctx, kind, mediaID)
if err != nil {
s.log.Warn("list media references for retention purge notice failed",
zap.String("media_kind", string(kind)), zap.Int64("media_id", mediaID), zap.Error(err))
return
}
for _, ref := range refs {
switch ref.RefKind {
case domain.MediaRefKindMessageBox:
s.notifyRetentionPurgeMessageBox(ctx, messages, ref.RefKey)
case domain.MediaRefKindChannelMessage:
s.notifyRetentionPurgeChannelMessage(ctx, channels, ref.RefKey)
case domain.MediaRefKindProfilePhoto, domain.MediaRefKindStickerSet, domain.MediaRefKindGift:
// No message to edit for these ref kinds.
}
}
}
// notifyRetentionPurgeMessageBox parses a message_box ref_key (format
// "user:<owner_user_id>:box:<box_id>", see postgres.messageBoxRefKey -- kept
// in sync by convention, not a shared symbol, since the ref_key encoding is
// an internal storage detail of that package) and edits the box in place.
func (s *Service) notifyRetentionPurgeMessageBox(ctx context.Context, messages RetentionPurgeMessageEditor, refKey string) {
if messages == nil {
return
}
ownerUserID, boxID, ok := parseMessageBoxRefKey(refKey)
if !ok {
s.log.Warn("unparseable message_box retention purge ref_key", zap.String("ref_key", refKey))
return
}
// EditMessageRequest requires the peer explicitly (an ordinary edit's RPC
// caller always supplies it) -- resolve it from the box row itself since
// this is a server-internal edit with no client request behind it.
list, err := messages.GetMessages(ctx, ownerUserID, []int{boxID})
if err != nil {
s.log.Warn("resolve message_box peer for retention purge notice failed",
zap.Int64("owner_user_id", ownerUserID), zap.Int("box_id", boxID), zap.Error(err))
return
}
if len(list.Messages) == 0 {
// Box no longer visible to its owner (deleted) -- nothing to notice.
return
}
peer := list.Messages[0].Peer
_, err = messages.EditMessage(ctx, ownerUserID, domain.EditMessageRequest{
OwnerUserID: ownerUserID,
Peer: peer,
ID: boxID,
SetRichMessage: true,
RichMessage: nil,
Media: retentionPurgeNoticeMedia(),
RetentionPurge: true,
})
if err != nil && !errors.Is(err, domain.ErrMessageNotModified) && !errors.Is(err, domain.ErrMessageIDInvalid) {
s.log.Warn("retention purge notice edit failed (message_box)",
zap.Int64("owner_user_id", ownerUserID), zap.Int("box_id", boxID), zap.Error(err))
}
}
// notifyRetentionPurgeChannelMessage parses a channel_message ref_key
// (format "channel:<channel_id>:msg:<message_id>", see
// postgres.channelMessageRefKey) and edits the message in place. Unlike
// private messages, no peer lookup is needed: EditChannelMessageRequest only
// needs the channel id, already encoded in the ref_key.
func (s *Service) notifyRetentionPurgeChannelMessage(ctx context.Context, channels RetentionPurgeChannelEditor, refKey string) {
if channels == nil {
return
}
channelID, messageID, ok := parseChannelMessageRefKey(refKey)
if !ok {
s.log.Warn("unparseable channel_message retention purge ref_key", zap.String("ref_key", refKey))
return
}
_, err := channels.EditChannelMessageInternal(ctx, domain.EditChannelMessageRequest{
ChannelID: channelID,
ID: messageID,
RetentionPurge: true,
RetentionPurgeAction: &domain.ChannelMessageAction{Type: domain.ChannelActionCustomText, Text: domain.RetentionPurgeNoticeText},
})
if err != nil && !errors.Is(err, domain.ErrMessageNotModified) && !errors.Is(err, domain.ErrMessageIDInvalid) {
s.log.Warn("retention purge notice edit failed (channel_message)",
zap.Int64("channel_id", channelID), zap.Int("message_id", messageID), zap.Error(err))
}
}
// retentionPurgeNoticeMedia builds the messageActionCustomAction service
// media payload private-message edits replace a purged message's media with.
func retentionPurgeNoticeMedia() *domain.MessageMedia {
return &domain.MessageMedia{
Kind: domain.MessageMediaKindService,
ServiceAction: &domain.MessageServiceAction{
Kind: domain.MessageServiceActionCustomText,
Text: domain.RetentionPurgeNoticeText,
},
}
}
// parseMessageBoxRefKey parses "user:<owner_user_id>:box:<box_id>".
// fmt.Sscanf reports success and silently ignores unconsumed trailing input
// (e.g. "user:1:box:2garbage" would still scan fine), so a bare cnt/err check
// isn't enough -- round-trip the parsed ids back through the same format
// (messageBoxRefKey's own layout) and require an exact match to reject any
// malformed/truncated/trailing-garbage key. Defensive only: ref_key is
// server-written and never externally supplied, but a bug or future
// encoding change should fail closed here, not silently misdirect an edit at
// the wrong box.
func parseMessageBoxRefKey(refKey string) (ownerUserID int64, boxID int, ok bool) {
cnt, err := fmt.Sscanf(refKey, "user:%d:box:%d", &ownerUserID, &boxID)
if err != nil || cnt != 2 || ownerUserID == 0 || boxID == 0 {
return 0, 0, false
}
if fmt.Sprintf("user:%d:box:%d", ownerUserID, boxID) != refKey {
return 0, 0, false
}
return ownerUserID, boxID, true
}
// parseChannelMessageRefKey parses "channel:<channel_id>:msg:<message_id>".
// See parseMessageBoxRefKey's doc comment -- same round-trip defense against
// trailing-garbage input Sscanf alone would silently accept.
func parseChannelMessageRefKey(refKey string) (channelID int64, messageID int, ok bool) {
cnt, err := fmt.Sscanf(refKey, "channel:%d:msg:%d", &channelID, &messageID)
if err != nil || cnt != 2 || channelID == 0 || messageID == 0 {
return 0, 0, false
}
if fmt.Sprintf("channel:%d:msg:%d", channelID, messageID) != refKey {
return 0, 0, false
}
return channelID, messageID, true
}

29
internal/app/files/retention_purge_test.go Normal file
View file

@ -0,0 +1,29 @@
package files
import "testing"
func TestParseMessageBoxRefKeyRoundTrip(t *testing.T) {
owner, box, ok := parseMessageBoxRefKey("user:42:box:7")
if !ok || owner != 42 || box != 7 {
t.Fatalf("parse = (%d, %d, %v), want (42, 7, true)", owner, box, ok)
}
malformed := []string{"", "garbage", "user:1:box:", "user:1:box:2extra", "user:0:box:1", "user:1:box:0", "channel:1:msg:2"}
for _, key := range malformed {
if _, _, ok := parseMessageBoxRefKey(key); ok {
t.Fatalf("parseMessageBoxRefKey(%q) ok=true, want false", key)
}
}
}
func TestParseChannelMessageRefKeyRoundTrip(t *testing.T) {
channel, msg, ok := parseChannelMessageRefKey("channel:99:msg:3")
if !ok || channel != 99 || msg != 3 {
t.Fatalf("parse = (%d, %d, %v), want (99, 3, true)", channel, msg, ok)
}
malformed := []string{"", "garbage", "channel:1:msg:", "channel:1:msg:2extra", "channel:0:msg:1", "channel:1:msg:0", "user:1:box:2"}
for _, key := range malformed {
if _, _, ok := parseChannelMessageRefKey(key); ok {
t.Fatalf("parseChannelMessageRefKey(%q) ok=true, want false", key)
}
}
}

42
internal/app/files/service.go
View file

@ -100,6 +100,27 @@ type Service struct {
// restart -- deleting only the DB row would make a "deleted" GIF come
// back on its own.
gifSeedDir string
// storageRetentionGlobalMaxAge/storageRetentionCategoryAges/
// storageRetentionAvatarMaxAge back categoryRetentionAge/avatarRetentionAge
// (retention.go) -- the per-category storage retention age overrides plus
// the shared global fallback age. Set via WithStorageRetentionAges.
storageRetentionGlobalMaxAge time.Duration
storageRetentionCategoryAges map[domain.MediaCategory]time.Duration
storageRetentionAvatarMaxAge time.Duration
// storageMaxTotalBytes is TELESRV_STORAGE_MAX_TOTAL_BYTES, reused by
// EvictOldestMediaOverBudget as the active-eviction trigger threshold
// (<=0 disables eviction regardless of TELESRV_STORAGE_EVICTION_ENABLE).
storageMaxTotalBytes int64
// retentionNotifyMu guards retentionMessages/retentionChannels: they are
// set post-construction (see SetRetentionPurgeNotifier) from
// cmd/telesrv/main.go once messageapp/channelapp services exist, which
// happens after this Service and the background retention sweep that
// reads them are already running.
retentionNotifyMu sync.RWMutex
retentionMessages RetentionPurgeMessageEditor
retentionChannels RetentionPurgeChannelEditor
}
// Option 配置 files 服务的可选能力。
@ -200,6 +221,27 @@ func WithGifCatalog(c store.GifCatalogStore) Option {
}
}
// WithStorageRetentionAges configures the per-category storage retention age
// overrides (Config.StorageRetentionMaxAgeByCategory/-MaxAgeAvatar) plus the
// shared global fallback age (Config.StorageRetentionMaxAge), consumed by
// categoryRetentionAge/avatarRetentionAge in retention.go.
func WithStorageRetentionAges(global time.Duration, byCategory map[domain.MediaCategory]time.Duration, avatarMaxAge time.Duration) Option {
return func(s *Service) {
s.storageRetentionGlobalMaxAge = global
s.storageRetentionCategoryAges = byCategory
s.storageRetentionAvatarMaxAge = avatarMaxAge
}
}
// WithStorageMaxTotalBytes records TELESRV_STORAGE_MAX_TOTAL_BYTES for
// EvictOldestMediaOverBudget's active-eviction trigger threshold, reusing the
// same cap SpaceGuard already enforces against new uploads.
func WithStorageMaxTotalBytes(maxBytes int64) Option {
return func(s *Service) {
s.storageMaxTotalBytes = maxBytes
}
}
// WithGifSeedDir records the gif seed directory (cfg.GifSeedDir) so
// AdminDeleteUncategorizedGifs can remove a seed-imported entry's source
// file alongside its DB row -- see the field's doc comment for why that

64
internal/app/maintenance/retention.go
View file

@ -124,6 +124,8 @@ type RetentionWorker struct {
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
orphanedMedia OrphanedMediaRetentionStore
hardMedia HardMediaRetentionStore
eviction StorageEvictionStore
evictionEnabled bool
logger *zap.Logger
retention time.Duration
botAPIRetention time.Duration
@ -311,8 +313,19 @@ func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
maxAge = w.hardMediaMaxAge
}
evictionActive := w.eviction != nil && w.evictionEnabled
if maxAge <= 0 {
return 0
if !evictionActive {
return 0
}
// Eviction is reactive to total bytes, not a fixed age, and is
// independent of TELESRV_STORAGE_RETENTION_MODE -- it can be the only
// thing enabled. Fall back to the shared housekeeping interval so it
// still gets its own ticker instead of never running.
maxAge = w.interval
if maxAge <= 0 {
maxAge = time.Hour
}
}
interval := w.interval
if interval <= 0 || maxAge < interval {
@ -324,12 +337,36 @@ func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
return interval
}
// StorageEvictionStore actively reclaims space once total physical blob bytes
// exceed TELESRV_STORAGE_MAX_TOTAL_BYTES: the oldest documents/photos
// (interleaved by created_at across both tables, regardless of category or
// age) are purged -- reusing the exact same blob-purge + retention-purge
// notice primitive as HardMediaRetentionStore -- until back under budget.
// Independent of TELESRV_STORAGE_RETENTION_MODE.
type StorageEvictionStore interface {
EvictOldestMediaOverBudget(ctx context.Context, limit int) (int, error)
}
// WithStorageEviction enables the active eviction sweep
// (TELESRV_STORAGE_EVICTION_ENABLE). It shares the same ticker as the
// orphan/hard media sweeps (see mediaRetentionInterval) and is independent of
// TELESRV_STORAGE_RETENTION_MODE -- it can run even when that is "off".
func (w *RetentionWorker) WithStorageEviction(store StorageEvictionStore, enabled bool) *RetentionWorker {
w.eviction = store
w.evictionEnabled = enabled
return w
}
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
// The store itself derives each category's effective cutoff from
// "now" (per-category retention age overrides, global age as
// fallback) -- this worker only owns when the sweep runs, not the
// per-category cutoff math.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now(), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
@ -337,19 +374,28 @@ func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
}
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
// "Hard" mode: purges blob bytes for documents/photos older than
// hardMediaMaxAge regardless of whether a live reference remains --
// the store is required to keep the document/photo metadata row
// intact, only removing file_blobs rows/bytes, so a message still
// renders its media placeholder (LOCATION_INVALID on download)
// instead of breaking outright.
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
// "Hard" mode: purges blob bytes for documents/photos older than the
// effective per-category age (falling back to hardMediaMaxAge)
// regardless of whether a live reference remains -- the store is
// required to keep the document/photo metadata row intact, only
// removing file_blobs rows/bytes, so a message still renders its
// media placeholder (LOCATION_INVALID on download) instead of
// breaking outright.
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now(), w.batch)
if err != nil {
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
} else if hardDeleted > 0 {
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
}
}
if w.eviction != nil && w.evictionEnabled {
evicted, err := w.eviction.EvictOldestMediaOverBudget(ctx, w.batch)
if err != nil {
w.logger.Warn("active storage eviction sweep failed", zap.Error(err))
} else if evicted > 0 {
w.logger.Info("active storage eviction sweep complete", zap.Int("evicted", evicted))
}
}
}
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {

70
internal/config/config.go
View file

@ -357,6 +357,32 @@ type Config struct {
// RetentionInterval/RetentionBatch cadence alongside every other
// retention check (see maintenance.RetentionWorker).
StorageRetentionMaxAge time.Duration
// StorageRetentionMaxAgeByCategory overrides StorageRetentionMaxAge for
// one document media category (Photo/Video/RoundVideo/Gif/Music/Voice/
// File); a category absent from the map inherits the shared global age.
// Populated from TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> env vars,
// one per category -- unset or non-positive means "inherit global" for
// that category (a document that hasn't been classified, e.g. a sticker,
// always uses the global age -- there is no override for that bucket).
// Avatar is not a domain.MediaCategory (a photo becomes/stops being an
// avatar via profile_photos, not a document attribute), so it gets its
// own StorageRetentionMaxAgeAvatar field instead of a map entry.
StorageRetentionMaxAgeByCategory map[domain.MediaCategory]time.Duration
// StorageRetentionMaxAgeAvatar overrides StorageRetentionMaxAge for
// photos currently active as someone's avatar (profile_photos.active).
// <=0 means "inherit global", same convention as
// StorageRetentionMaxAgeByCategory.
StorageRetentionMaxAgeAvatar time.Duration
// StorageEvictionEnable turns on active eviction: once the total physical
// blob bytes exceed StorageMaxTotalBytes, the oldest documents/photos
// (interleaved by created_at across both tables, regardless of category
// or age) are purged the same way "hard" retention mode purges blob
// bytes -- repeated until back under budget. Independent of
// StorageRetentionMode: eviction can run even when retention mode is
// "off". Default false (opt-in), since this changes what
// StorageMaxTotalBytes has meant so far (block new uploads only ->
// block-and-reclaim from existing ones too).
StorageEvictionEnable bool
// StickerSeedDir 是 reaction / sticker 资源种子目录（导入到 documents/sticker_sets + blob）。
StickerSeedDir string
// StickerSeedMaxSets 限制导入的常规贴纸集数量（避免启动时导入过多包），<=0 表示不限。
@ -980,6 +1006,9 @@ func Load() (Config, error) {
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
StorageRetentionMaxAgeByCategory: storageRetentionMaxAgeByCategoryFromEnv(envDurationOr),
StorageRetentionMaxAgeAvatar: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR", 0),
StorageEvictionEnable: envBoolOr("TELESRV_STORAGE_EVICTION_ENABLE", false),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
PremiumPromoSeedDir: envOr("TELESRV_PREMIUM_PROMO_SEED_DIR", "data/premium-promo"),
@ -1518,13 +1547,21 @@ func validateStorageConfig(cfg Config) error {
if cfg.StorageRetentionMaxAge < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
}
if cfg.StorageRetentionMaxAgeAvatar < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE_AVATAR must be non-negative")
}
switch cfg.StorageRetentionMode {
case StorageRetentionModeOff:
// no sweep; StorageRetentionMaxAge is ignored.
case StorageRetentionModeOrphan, StorageRetentionModeHard:
if cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
}
// StorageRetentionMaxAge (the shared default) is no longer required to
// be positive here: it can legitimately be 0 -- "no sweep by default"
// -- while individual TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY>/
// _AVATAR overrides opt specific categories in. The sweep itself
// (internal/app/files/retention.go) skips any category whose
// *effective* age (its own override, or this shared default) is <=0,
// so a 0 default with no overrides simply means the mode runs and
// purges nothing -- equivalent to "off" in practice, not dangerous.
default:
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
}
@ -2075,6 +2112,33 @@ func (e envSource) envFloatOr(key string, def float64) float64 {
return def
}
// storageRetentionMaxAgeByCategoryFromEnv builds Config.StorageRetentionMaxAgeByCategory:
// one TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> env var per document
// media category, using envDurationOr(key, 0) so "unset" and "explicitly 0"
// both cleanly mean "inherit the shared global age" -- only a positive value
// gets an entry in the map.
func storageRetentionMaxAgeByCategoryFromEnv(envDurationOr func(key string, def time.Duration) time.Duration) map[domain.MediaCategory]time.Duration {
entries := []struct {
category domain.MediaCategory
envKey string
}{
{domain.MediaCategoryPhoto, "TELESRV_STORAGE_RETENTION_MAX_AGE_PHOTO"},
{domain.MediaCategoryVideo, "TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO"},
{domain.MediaCategoryRoundVideo, "TELESRV_STORAGE_RETENTION_MAX_AGE_ROUND_VIDEO"},
{domain.MediaCategoryGif, "TELESRV_STORAGE_RETENTION_MAX_AGE_GIF"},
{domain.MediaCategoryMusic, "TELESRV_STORAGE_RETENTION_MAX_AGE_MUSIC"},
{domain.MediaCategoryVoice, "TELESRV_STORAGE_RETENTION_MAX_AGE_VOICE"},
{domain.MediaCategoryFile, "TELESRV_STORAGE_RETENTION_MAX_AGE_FILE"},
}
out := make(map[domain.MediaCategory]time.Duration, len(entries))
for _, e := range entries {
if age := envDurationOr(e.envKey, 0); age > 0 {
out[e.category] = age
}
}
return out
}
// envDurationOr 读取 time.ParseDuration 格式（如 "200ms"、"30s"）的时长配置；解析失败回退默认值。
func (e envSource) envDurationOr(key string, def time.Duration) time.Duration {
if v := e.envOr(key, ""); v != "" {

23
internal/config/config_test.go
View file

@ -550,14 +550,31 @@ func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
}
}
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
// TestLoadAcceptsStorageRetentionModeWithZeroMaxAgeAndCategoryOverride guards
// a real reported bug: TELESRV_STORAGE_RETENTION_MAX_AGE=0 (the shared
// default) used to be rejected outright whenever the mode was orphan/hard,
// which made it impossible to run the sweep for only specific
// TELESRV_STORAGE_RETENTION_MAX_AGE_<CATEGORY> overrides while leaving
// everything else alone. A zero shared default is now legitimate: the sweep
// itself (internal/app/files/retention.go) skips any category whose
// *effective* age is <=0, so 0 here just means "no default sweep" rather
// than "purge everything immediately".
func TestLoadAcceptsStorageRetentionModeWithZeroMaxAgeAndCategoryOverride(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
if _, err := Load(); err == nil {
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE_VIDEO", "24h")
cfg, err := Load()
if err != nil {
t.Fatalf("Load rejected TELESRV_STORAGE_RETENTION_MODE=%s with zero shared max age + a category override: %v", mode, err)
}
if cfg.StorageRetentionMaxAge != 0 {
t.Fatalf("StorageRetentionMaxAge = %v, want 0", cfg.StorageRetentionMaxAge)
}
if got := cfg.StorageRetentionMaxAgeByCategory[domain.MediaCategoryVideo]; got != 24*time.Hour {
t.Fatalf("StorageRetentionMaxAgeByCategory[Video] = %v, want 24h", got)
}
})
}

15
internal/domain/channel.go
View file

@ -634,6 +634,10 @@ const (
ChannelActionSuggestedPostApproval ChannelMessageActionType = "suggested_post_approval"
ChannelActionSuggestedPostSuccess ChannelMessageActionType = "suggested_post_success"
ChannelActionSuggestedPostRefund ChannelMessageActionType = "suggested_post_refund"
// ChannelActionCustomText 映射 messageActionCustomAction（stock TL 类型）：
// 当前唯一用途是把 storage retention 硬回收/主动淘汰清理掉的频道消息就地
// 转成这条通知（见 EditChannelMessageRequest.RetentionPurge）。
ChannelActionCustomText ChannelMessageActionType = "custom_text"
)
// ChannelMessageAction describes a service action without depending on tg.*.
@ -678,6 +682,8 @@ type ChannelMessageAction struct {
SuggestedPostScheduleDate int
SuggestedPostPrice *SuggestedPostPrice
SuggestedPostPayerInitiated bool
// Text 仅 custom_text 服务消息使用（messageActionCustomAction.message）。
Text string
}
// ChannelMessage is a single stored message in a channel/supergroup.
@ -1967,6 +1973,15 @@ type EditChannelMessageRequest struct {
// 仅当目标当前 media 仍是 ID==ExpectedWebPageID 的 pending 链接预览才替换。
WebPageResolve bool
ExpectedWebPageID int64
// RetentionPurge 置位时为存储 retention 硬回收/主动淘汰的服务端内部编辑：
// 绕过普通的发送者/管理员编辑权限校验，仅由 internal/app/files 的
// retention purge 通知路径设置。见 EditMessageRequest.RetentionPurge。
RetentionPurge bool
// RetentionPurgeAction 与 RetentionPurge 搭配使用：频道消息的服务动作
// 存于独立的 Action 字段（不像私聊消息把服务动作折进 Media.Kind=service），
// 所以 retention 通知在频道侧必须整条消息转成一条新的服务消息（清空
// body/media，只设置这个 Action），而不是像私聊那样替换 Media。
RetentionPurgeAction *ChannelMessageAction
}
// EditChannelMessageResult describes one channel edit update.

7
internal/domain/media.go
View file

@ -591,6 +591,11 @@ const (
// 状态切换与关闭请求。会话级保护不能写入普通消息的 NoForwards 字段。
MessageServiceActionNoForwardsToggle MessageServiceActionKind = "no_forwards_toggle"
MessageServiceActionNoForwardsRequest MessageServiceActionKind = "no_forwards_request"
// MessageServiceActionCustomText 映射 messageActionCustomAction（stock TL
// 类型，两端客户端已原生支持，无需任何 client 补丁）：服务端把纯文本状态
// 通知（当前唯一用途：storage retention 硬回收把已清理媒体的消息就地转成
// 这条通知）渲染成居中灰色系统气泡，而不是一次看起来像普通编辑的文本替换。
MessageServiceActionCustomText MessageServiceActionKind = "custom_text"
)
// MessagePhoneCallAction 是 messageActionPhoneCall 的协议中立载荷。
@ -673,6 +678,8 @@ type MessageServiceAction struct {
RequestedPeer *MessageRequestedPeerAction `json:"requested_peer,omitempty"`
ChatThemeEmoticon string `json:"chat_theme_emoticon,omitempty"`
NoForwards *MessageNoForwardsAction `json:"no_forwards,omitempty"`
// Text 承载 MessageServiceActionCustomText 的固定通知文案（messageActionCustomAction.message）。
Text string `json:"text,omitempty"`
}
// MessageMedia 是一条消息媒体载荷的业务表示（落库为消息行上的 JSONB 快照）。

20
internal/domain/media_category.go
View file

@ -159,6 +159,26 @@ func classifyDocumentCategory(doc *Document) (MediaCategory, bool) {
}
}
// DocumentMediaCategory returns the single document-attribute category
// (Video/Gif/Music/Voice/RoundVideo/File) for a document media payload, used
// to populate the documents.category column alongside the shared media
// index. Unlike ClassifyMediaCategories (which can also add unrelated
// categories such as URL from message entities), this only ever answers "what
// kind of document is this", returning MediaCategoryNone for a non-document
// payload or for sticker/custom-emoji documents (they classify to nothing --
// see classifyDocumentCategory). Zero new classification logic: reuses
// classifyDocumentCategory exactly.
func DocumentMediaCategory(media *MessageMedia) MediaCategory {
if media == nil || media.Kind != MessageMediaKindDocument {
return MediaCategoryNone
}
c, ok := classifyDocumentCategory(media.Document)
if !ok {
return MediaCategoryNone
}
return c
}
func hasURLEntity(entities []MessageEntity) bool {
for _, e := range entities {
if e.Type == MessageEntityURL || e.Type == MessageEntityTextURL || e.Type == MessageEntityEmail {

12
internal/domain/media_refs.go
View file

@ -1,5 +1,7 @@
package domain
import "time"
// MediaKind identifies which table a media_references row points into.
type MediaKind string
@ -45,6 +47,16 @@ type OrphanCandidate struct {
OrphanedAt int64 // unix seconds
}
// EvictionCandidate is one document/photo still owning file_blobs bytes,
// used by the active-eviction sweep (TELESRV_STORAGE_EVICTION_ENABLE) to pick
// the oldest media overall across both tables once total physical storage
// exceeds TELESRV_STORAGE_MAX_TOTAL_BYTES.
type EvictionCandidate struct {
Kind MediaKind
MediaID int64
CreatedAt time.Time
}
// MediaRefTarget identifies one document/photo embedded in a message's media
// snapshot.
type MediaRefTarget struct {

11
internal/domain/message.go
View file

@ -555,6 +555,12 @@ type OutboxReadDateRequest struct {
ID int
}
// RetentionPurgeNoticeText is the fixed notice text used for the
// messageActionCustomAction (MessageServiceActionCustomText) service message
// a storage-retention blob purge (hard mode or active eviction) turns a
// message into once its underlying document/photo bytes are gone.
const RetentionPurgeNoticeText = "This file was deleted by the server's storage retention policy."
// EditMessageRequest 是账号视角下编辑一条已发送私聊消息的命令。
// Media 非 nil 时整体替换消息媒体快照（当前唯一调用方是 live location 续报/停止，
// rpc 层负责限定媒体种类）；nil 表示纯文本编辑。
@ -589,6 +595,11 @@ type EditMessageRequest struct {
// 否则返回 ErrMessageNotModified（消息已删/已改/已解析）。
WebPageResolve bool
ExpectedWebPageID int64
// RetentionPurge 置位时为存储 retention 硬回收/主动淘汰的服务端内部编辑：
// 绕过 authorEdit/viaBotEdit/WebPageResolve 之外的普通作者校验（回收发生
// 在任意账号的历史消息上，不是消息真正作者的操作），仅由
// internal/app/files 的 retention purge 通知路径设置。
RetentionPurge bool
}
// EditedMessageForUser 描述一次编辑对某个 owner 视角造成的影响。

2
internal/rpc/convert_channels_core.go
View file

@ -224,6 +224,8 @@ func tgChannelMessageAction(action domain.ChannelMessageAction) tg.MessageAction
return &tg.MessageActionChannelCreate{Title: action.Title}
case domain.ChannelActionHistoryClear:
return &tg.MessageActionHistoryClear{}
case domain.ChannelActionCustomText:
return &tg.MessageActionCustomAction{Message: action.Text}
case domain.ChannelActionChatAddUser, domain.ChannelActionChatJoined:
return &tg.MessageActionChatAddUser{Users: append([]int64(nil), action.UserIDs...)}
case domain.ChannelActionChatJoinedByLink:

2
internal/rpc/convert_messages.go
View file

@ -139,6 +139,8 @@ func tgMessageServiceAction(msg domain.Message) tg.MessageActionClass {
switch m.ServiceAction.Kind {
case domain.MessageServiceActionHistoryClear:
return &tg.MessageActionHistoryClear{}
case domain.MessageServiceActionCustomText:
return &tg.MessageActionCustomAction{Message: m.ServiceAction.Text}
case domain.MessageServiceActionSuggestProfilePhoto:
if m.ServiceAction.Photo == nil || m.ServiceAction.Photo.ID == 0 {
return &tg.MessageActionEmpty{}

98
internal/store/postgres/channel_message_edit.go
View file

@ -12,7 +12,12 @@ import (
)
func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
if req.UserID == 0 || req.ChannelID == 0 || req.ID <= 0 {
retentionPurge := req.RetentionPurge && req.RetentionPurgeAction != nil
// RetentionPurge is a server-internal edit with no acting user (the
// storage retention sweep, not an RPC caller) -- UserID==0 is normally
// invalid, and req.UserID would otherwise also need to name an existing
// channel member for getChannelForMember below.
if (req.UserID == 0 && !retentionPurge) || req.ChannelID == 0 || req.ID <= 0 {
return domain.EditChannelMessageResult{}, domain.ErrChannelInvalid
}
beginner, ok := s.db.(txBeginner)
@ -44,7 +49,17 @@ func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditCh
_ = tx.Rollback(ctx)
}
}()
channel, member, err := s.getChannelForMember(ctx, tx, req.UserID, req.ChannelID)
var (
channel domain.Channel
member domain.ChannelMember
)
if retentionPurge {
// No membership requirement: this is a server-internal edit, not an
// action taken by any particular channel member.
channel, err = s.channelByID(ctx, tx, req.ChannelID)
} else {
channel, member, err = s.getChannelForMember(ctx, tx, req.UserID, req.ChannelID)
}
if err != nil {
return domain.EditChannelMessageResult{}, err
}
@ -52,6 +67,17 @@ func (s *ChannelStore) EditChannelMessage(ctx context.Context, req domain.EditCh
if err != nil {
return domain.EditChannelMessageResult{}, err
}
if retentionPurge {
if msg.Deleted {
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
}
if msg.Action != nil {
// 幂等守卫：已经是服务消息(含已被本路径转换过的 retention 通知)
// 不再重复编辑 -- 见 EditMessageRequest.RetentionPurge 同款守卫。
return domain.EditChannelMessageResult{}, domain.ErrMessageNotModified
}
return s.applyRetentionPurgeChannelMessage(ctx, tx, channel, msg, req)
}
if msg.Deleted || msg.Action != nil {
return domain.EditChannelMessageResult{}, domain.ErrMessageIDInvalid
}
@ -283,6 +309,74 @@ WHERE channel_id = $1 AND id = $2`, req.ChannelID, req.ID, mediaJSON); err != ni
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, ServiceMessage: serviceMsg, ServiceEvent: serviceEvent, Recipients: recipients}, nil
}
// applyRetentionPurgeChannelMessage turns msg into a real service message
// (messageActionCustomAction via req.RetentionPurgeAction) in place. Unlike a
// normal channel edit -- which only ever replaces body/entities/media --
// channel service actions live in a structurally separate Action column
// (see tgChannelMessage: m.Action != nil renders tg.MessageService instead of
// tg.Message), so this clears body/media and sets Action instead of touching
// them. Reuses the same pts/durable-event/admin-log machinery as a normal
// edit so getChannelDifference and the fanout dispatcher see it identically.
func (s *ChannelStore) applyRetentionPurgeChannelMessage(ctx context.Context, tx pgx.Tx, channel domain.Channel, msg domain.ChannelMessage, req domain.EditChannelMessageRequest) (domain.EditChannelMessageResult, error) {
actionJSON, err := marshalJSON(req.RetentionPurgeAction, "{}")
if err != nil {
return domain.EditChannelMessageResult{}, fmt.Errorf("encode retention purge channel action: %w", err)
}
pts, err := s.reserveChannelPts(ctx, tx, req.ChannelID)
if err != nil {
return domain.EditChannelMessageResult{}, fmt.Errorf("allocate retention purge channel pts: %w", err)
}
if _, err := tx.Exec(ctx, `
UPDATE channel_messages
SET body = '', entities = '[]'::jsonb, media = '{}'::jsonb, action = $3, edit_date = $4, pts = $5, updated_at = now()
WHERE channel_id = $1 AND id = $2`, req.ChannelID, req.ID, actionJSON, req.EditDate, pts); err != nil {
return domain.EditChannelMessageResult{}, fmt.Errorf("update retention purge channel message: %w", err)
}
prevMsg := msg
msg.Body = ""
msg.Entities = nil
msg.Media = nil
msg.Action = req.RetentionPurgeAction
msg.EditDate = req.EditDate
msg.Pts = pts
// 媒体索引/media_references 靠这次替换后的（空）媒体重建：原文档/照片
// 不再被这条消息引用，storage retention 的孤儿判定据此推进。
if err := replaceChannelMediaIndexTx(ctx, tx, req.ChannelID, req.ID, msg.Date, msg.Media, nil); err != nil {
return domain.EditChannelMessageResult{}, err
}
event := domain.ChannelUpdateEvent{
ChannelID: req.ChannelID,
Type: domain.ChannelUpdateEditMessage,
Pts: pts,
PtsCount: 1,
Date: req.EditDate,
Message: msg,
SenderUserID: req.UserID,
}
if err := insertChannelEventTx(ctx, tx, event); err != nil {
return domain.EditChannelMessageResult{}, err
}
if _, err := tx.Exec(ctx, `UPDATE channels SET pts = $2, updated_at = now() WHERE id = $1`, req.ChannelID, pts); err != nil {
return domain.EditChannelMessageResult{}, fmt.Errorf("update retention purge channel pts: %w", err)
}
channel.Pts = pts
if err := s.insertChannelAdminLogTx(ctx, tx, domain.ChannelAdminLogEvent{
ChannelID: req.ChannelID,
UserID: req.UserID,
Date: req.EditDate,
Type: domain.ChannelAdminLogEditMessage,
PrevMessage: &prevMsg,
NewMessage: &msg,
}); err != nil {
return domain.EditChannelMessageResult{}, err
}
if err := tx.Commit(ctx); err != nil {
return domain.EditChannelMessageResult{}, fmt.Errorf("commit retention purge channel message: %w", err)
}
recipients, _ := s.ListActiveChannelMemberIDs(ctx, req.UserID, req.ChannelID, 0)
return domain.EditChannelMessageResult{Channel: channel, Message: msg, Event: event, Recipients: recipients}, nil
}
func isChannelTodoParticipantEdit(req domain.EditChannelMessageRequest, msg domain.ChannelMessage) bool {
if !req.AllowTodoParticipantMutation || req.SetReplyMarkup || req.Media == nil || req.Media.Kind != domain.MessageMediaKindTodo || req.Media.Todo == nil {
return false

4
internal/store/postgres/media_hard_retention_integration_test.go
View file

@ -64,7 +64,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
})
cutoff := time.Now().Add(-24 * time.Hour)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
}
@ -95,7 +95,7 @@ func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
// Idempotent / self-terminating: a second sweep pass no longer selects
// this document, since it no longer owns any file_blobs row.
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryNone, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
}

24
internal/store/postgres/media_index.go
View file

@ -24,9 +24,30 @@ ON CONFLICT (channel_id, id, category) DO NOTHING`, channelID, id, int16(c), dat
return fmt.Errorf("insert channel media index: %w", err)
}
}
if err := setDocumentCategoryTx(ctx, tx, media); err != nil {
return err
}
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindChannelMessage, channelMessageRefKey(channelID, id))
}
// setDocumentCategoryTx persists media.Document's retention-sweep category
// (see domain.DocumentMediaCategory) onto its documents row, alongside the
// message_box_media/channel_message_media index write these callers already
// do -- zero new classification logic, just also stamping the already
// computed value directly onto documents.category so the per-category
// storage retention sweep (internal/app/files/retention.go) can filter on it
// without a join. No-op for non-document media.
func setDocumentCategoryTx(ctx context.Context, tx pgx.Tx, media *domain.MessageMedia) error {
if media == nil || media.Kind != domain.MessageMediaKindDocument || media.Document == nil || media.Document.ID == 0 {
return nil
}
category := domain.DocumentMediaCategory(media)
if _, err := tx.Exec(ctx, `UPDATE documents SET category = $2 WHERE id = $1`, media.Document.ID, int16(category)); err != nil {
return fmt.Errorf("set document category: %w", err)
}
return nil
}
// deleteChannelMediaIndexTx 清掉一条频道消息的全部索引行(编辑改媒体前先清后插)。
// 只在 replaceChannelMediaIndexTx 内被调用;真正的消息删除不清 *_media 分类索引
// (读时靠 JOIN deleted 过滤,见文件头注释),但仍需在此清掉 media_references,
@ -57,6 +78,9 @@ ON CONFLICT (owner_user_id, box_id, category) DO NOTHING`, ownerUserID, boxID, p
return fmt.Errorf("insert message box media index: %w", err)
}
}
if err := setDocumentCategoryTx(ctx, tx, media); err != nil {
return err
}
return addMediaReferencesTx(ctx, tx, media, domain.MediaRefKindMessageBox, messageBoxRefKey(ownerUserID, boxID))
}

105
internal/store/postgres/media_refs.go
View file

@ -105,6 +105,30 @@ RETURNING media_kind, media_id`, string(refKind), refKey)
return nil
}
// ListMediaReferences returns every live reference row for (kind, mediaID) --
// used by the storage retention sweep to turn a hard-retention/eviction blob
// purge into a visible notice on every message that still embeds the purged
// media. See internal/app/files.notifyRetentionPurge.
func (s *MediaStore) ListMediaReferences(ctx context.Context, kind domain.MediaKind, mediaID int64) ([]domain.MediaReference, error) {
rows, err := s.q.ListMediaReferences(ctx, sqlcgen.ListMediaReferencesParams{
MediaKind: string(kind),
MediaID: mediaID,
})
if err != nil {
return nil, fmt.Errorf("list media references: %w", err)
}
out := make([]domain.MediaReference, 0, len(rows))
for _, r := range rows {
out = append(out, domain.MediaReference{
Kind: domain.MediaKind(r.MediaKind),
MediaID: r.MediaID,
RefKind: domain.MediaRefKind(r.RefKind),
RefKey: r.RefKey,
})
}
return out, nil
}
// ---- storage retention sweep ----
// OrphanDocumentIfUnreferenced marks a document orphaned right now if
@ -128,20 +152,22 @@ WHERE id = $1
return tag.RowsAffected() > 0, nil
}
// ListOrphanedDocumentIDsOlderThan returns document ids whose orphaned_at is
// set and older than cutoff, oldest first, up to limit.
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
// ListOrphanedDocumentIDsOlderThan returns document ids in the given category
// whose orphaned_at is set and older than cutoff, oldest first, up to limit.
func (s *MediaStore) ListOrphanedDocumentIDsOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListOrphanedDocumentIDsOlderThan(ctx, sqlcgen.ListOrphanedDocumentIDsOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
Category: int16(category),
BatchLimit: int32(limit),
})
}
// ListOrphanedPhotoIDsOlderThan returns photo ids whose orphaned_at is set
// and older than cutoff, oldest first, up to limit.
// ListOrphanedPhotoIDsOlderThan returns photo ids (excluding a live avatar --
// see ListAvatarOrphanedPhotoIDsOlderThan) whose orphaned_at is set and older
// than cutoff, oldest first, up to limit.
func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
@ -152,6 +178,19 @@ func (s *MediaStore) ListOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff t
})
}
// ListAvatarOrphanedPhotoIDsOlderThan is the "Avatar" category counterpart of
// ListOrphanedPhotoIDsOlderThan -- see the query's doc comment for why this
// is expected to stay empty in practice under "orphan" mode.
func (s *MediaStore) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListAvatarOrphanedPhotoIDsOlderThan(ctx, sqlcgen.ListAvatarOrphanedPhotoIDsOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// CountFileBlobRefs reports how many file_blobs rows still point at
// (backend, objectKey) -- the caller must not physically delete the object
// from that backend while this is > 0 (content-addressed storage: the same
@ -197,24 +236,26 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
return blobs, nil
}
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
// cutoff (by upload/created_at) that still own at least one file_blobs row,
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
// entirely: a document still referenced by a live message is exactly as
// eligible as an orphaned one.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
// ListDocumentIDsForHardRetentionOlderThan returns document ids in the given
// category older than cutoff (by upload/created_at) that still own at least
// one file_blobs row, oldest first, up to limit -- the "hard" retention
// sweep's candidate list. Unlike ListOrphanedDocumentIDsOlderThan, this
// ignores media_references entirely: a document still referenced by a live
// message is exactly as eligible as an orphaned one.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, category domain.MediaCategory, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
Category: int16(category),
BatchLimit: int32(limit),
})
}
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
// ListDocumentIDsForHardRetentionOlderThan.
// ListDocumentIDsForHardRetentionOlderThan (excluding a live avatar -- see
// ListAvatarPhotoIDsForHardRetentionOlderThan).
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
@ -225,6 +266,18 @@ func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context,
})
}
// ListAvatarPhotoIDsForHardRetentionOlderThan is the "Avatar" category
// counterpart of ListPhotoIDsForHardRetentionOlderThan.
func (s *MediaStore) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListAvatarPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListAvatarPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
// (main body + thumbnail variants), returning what was deleted so the caller
// can physically remove each object from its backend once confirming (via
@ -292,6 +345,32 @@ func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]d
return blobs, nil
}
// ListOldestMediaForEviction returns up to limit documents and limit photos
// still owning file_blobs bytes, oldest-uploaded first, for the active
// eviction sweep to interleave by actual created_at (not drain one table
// before the other) -- see domain.EvictionCandidate.
func (s *MediaStore) ListOldestMediaForEviction(ctx context.Context, limit int) ([]domain.EvictionCandidate, error) {
if limit <= 0 {
return nil, nil
}
docs, err := s.q.ListOldestDocumentsForEviction(ctx, int32(limit))
if err != nil {
return nil, fmt.Errorf("list oldest documents for eviction: %w", err)
}
photos, err := s.q.ListOldestPhotosForEviction(ctx, int32(limit))
if err != nil {
return nil, fmt.Errorf("list oldest photos for eviction: %w", err)
}
out := make([]domain.EvictionCandidate, 0, len(docs)+len(photos))
for _, d := range docs {
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindDocument, MediaID: d.ID, CreatedAt: d.CreatedAt.Time})
}
for _, p := range photos {
out = append(out, domain.EvictionCandidate{Kind: domain.MediaKindPhoto, MediaID: p.ID, CreatedAt: p.CreatedAt.Time})
}
return out, nil
}
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
// (one per rendition size), returning what was deleted so the caller can
// physically remove each object from its backend once confirming (via

11
internal/store/postgres/message_edit.go
View file

@ -83,9 +83,18 @@ func (s *MessageStore) EditMessage(ctx context.Context, req domain.EditMessageRe
}
authorEdit := target.Outgoing && target.MessageSenderID == req.OwnerUserID && target.FromUserID == req.OwnerUserID
viaBotEdit := req.ViaBotEditBotID != 0 && target.ViaBotID == req.ViaBotEditBotID
if !authorEdit && !viaBotEdit && !req.WebPageResolve && !validTodoParticipantEdit(req, target, oldEntities) {
if !authorEdit && !viaBotEdit && !req.WebPageResolve && !req.RetentionPurge && !validTodoParticipantEdit(req, target, oldEntities) {
return res, domain.ErrMessageAuthorRequired
}
if req.RetentionPurge {
// 幂等守卫：已经是这条 retention 通知的消息不再重复编辑（同一被回收
// 媒体可能被多个 box 的 media_references 行各引用一次，比如自己与对端
// 各自的 box——同一条共享 private_message 只需真正编辑一次）。
if targetMedia, err := decodeMessageMedia(target.MediaJson); err == nil &&
targetMedia != nil && targetMedia.Kind == domain.MessageMediaKindService {
return res, domain.ErrMessageNotModified
}
}
richChanged := req.SetRichMessage && !richMessagesEqual(targetRich, req.RichMessage)
if req.Media == nil && !req.SetReplyMarkup && !richChanged && target.Body == req.Message && target.HideEdited == req.HideEdited && sameMessageEntities(oldEntities, req.Entities) {
return res, domain.ErrMessageNotModified

92
internal/store/postgres/queries/media.sql
View file

@ -202,6 +202,16 @@ WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
UPDATE photos SET orphaned_at = NULL
WHERE id = sqlc.arg(media_id)::bigint AND orphaned_at IS NOT NULL;
-- name: ListMediaReferences :many
-- Every live reference to a document/photo -- used by the storage retention
-- sweep to turn a hard-retention/eviction blob purge into a visible notice on
-- every message that still embeds the purged media (see
-- files.notifyRetentionPurge). profile_photo/sticker_set/gift refs have no
-- message to edit and are filtered by the caller, not this query.
SELECT media_kind, media_id, ref_kind, ref_key
FROM media_references
WHERE media_kind = sqlc.arg(media_kind)::text AND media_id = sqlc.arg(media_id)::bigint;
-- name: RemoveMediaReference :exec
DELETE FROM media_references
WHERE media_kind = sqlc.arg(media_kind)::text
@ -226,15 +236,36 @@ WHERE id = sqlc.arg(media_id)::bigint
);
-- name: ListOrphanedDocumentIDsOlderThan :many
-- category selects one documents.category bucket per sweep tick (per-category
-- retention age, see internal/app/files/retention.go) -- 0 (MediaCategoryNone)
-- covers unclassified documents (e.g. stickers), which always use the shared
-- global age since there is no per-category override for that bucket.
SELECT id FROM documents
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
AND category = sqlc.arg(category)::smallint
ORDER BY orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListOrphanedPhotoIDsOlderThan :many
SELECT id FROM photos
WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
ORDER BY orphaned_at ASC
-- Excludes photos currently active as someone's avatar -- see
-- ListAvatarOrphanedPhotoIDsOlderThan for that split-off bucket.
SELECT p.id FROM photos p
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < sqlc.arg(cutoff)::timestamptz
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
ORDER BY p.orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListAvatarOrphanedPhotoIDsOlderThan :many
-- Same as ListOrphanedPhotoIDsOlderThan but only photos currently active as
-- someone's avatar (profile_photos.active) -- lets the Avatar category carry
-- its own retention age. In practice a live avatar is never orphaned (an
-- active profile_photos row is itself a media_references entry), so this
-- bucket is expected to stay empty under "orphan" mode; kept for symmetry
-- with the "hard" mode avatar split, which is the one that actually matters.
SELECT p.id FROM photos p
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
ORDER BY p.orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListDocumentIDsForHardRetentionOlderThan :many
@ -244,9 +275,12 @@ LIMIT sqlc.arg(batch_limit)::int;
-- keeps this sweep from re-selecting the same document forever: once its
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
-- rows but deliberately leaves this documents row in place), it naturally
-- drops out of this query on the next pass.
-- drops out of this query on the next pass. category scopes to one
-- documents.category bucket per sweep tick -- see
-- ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
SELECT d.id FROM documents d
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
AND d.category = sqlc.arg(category)::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
@ -257,9 +291,11 @@ LIMIT sqlc.arg(batch_limit)::int;
-- name: ListPhotoIDsForHardRetentionOlderThan :many
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
-- candidate selection, for photos.
-- candidate selection, for photos. Excludes photos currently active as
-- someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
@ -268,6 +304,52 @@ WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
-- Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
-- active as someone's avatar (profile_photos.active) -- lets the Avatar
-- category carry its own retention age, independent of ordinary shared-media
-- photos.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListOldestDocumentsForEviction :many
-- Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
-- document that still owns at least one file_blobs row, oldest-uploaded
-- first, REGARDLESS of category or age -- unlike the retention sweeps above,
-- eviction only cares about reclaiming bytes once the total physical budget
-- (TELESRV_STORAGE_MAX_TOTAL_BYTES) is exceeded. created_at is returned so
-- the caller can interleave these with ListOldestPhotosForEviction by actual
-- age instead of draining one table before touching the other.
SELECT d.id, d.created_at FROM documents d
WHERE EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListOldestPhotosForEviction :many
-- See ListOldestDocumentsForEviction -- same oldest-first eviction candidate
-- selection, for photos (avatars included: eviction is bytes-only and does
-- not honor the Avatar category's separate retention age).
SELECT p.id, p.created_at FROM photos p
WHERE EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;

360
internal/store/postgres/retention_v2_integration_test.go Normal file
View file

@ -0,0 +1,360 @@
package postgres
import (
"context"
"strconv"
"testing"
"time"
"telesrv/internal/domain"
)
// TestRetentionPurgeBypassProducesServiceMessage_PrivateMessage exercises
// Part 1 of the storage retention v2 plan end-to-end against a real
// Postgres: EditMessage with RetentionPurge=true bypasses the ordinary
// author check (the storage retention sweep is not the message's author) and
// replaces the message media with a messageActionCustomAction service
// payload, and a second retention-purge edit on the now-already-service
// message is a no-op (ErrMessageNotModified) -- guarding the idempotency
// guard that keeps a shared media_references duplicate (owner+peer both
// referencing the same purged document) from double-editing the same box.
func TestRetentionPurgeBypassProducesServiceMessage_PrivateMessage(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
suffix := randomSuffix(t)
users := NewUserStore(pool)
sender, err := users.Create(ctx, domain.User{AccessHash: 81, Phone: "+1997" + suffix + "01", FirstName: "RetentionSender"})
if err != nil {
t.Fatalf("create sender: %v", err)
}
recipient, err := users.Create(ctx, domain.User{AccessHash: 82, Phone: "+1997" + suffix + "02", FirstName: "RetentionRecipient"})
if err != nil {
t.Fatalf("create recipient: %v", err)
}
ids := []int64{sender.ID, recipient.ID}
t.Cleanup(func() {
_, _ = pool.Exec(ctx, "DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = ANY($1::bigint[])", ids)
})
messages := NewMessageStore(pool, WithMessageAllocators(&perUserCounterAllocator{}))
sent, err := messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
SenderUserID: sender.ID,
RecipientUserID: recipient.ID,
RandomID: time.Now().UnixNano(),
Message: "a file worth keeping",
Media: &domain.MessageMedia{
Kind: domain.MessageMediaKindDocument,
Document: &domain.Document{ID: time.Now().UnixNano(), AccessHash: 1, MimeType: "application/octet-stream"},
},
Date: int(time.Now().Unix()),
})
if err != nil {
t.Fatalf("send private media: %v", err)
}
noticeMedia := &domain.MessageMedia{
Kind: domain.MessageMediaKindService,
ServiceAction: &domain.MessageServiceAction{
Kind: domain.MessageServiceActionCustomText,
Text: domain.RetentionPurgeNoticeText,
},
}
req := domain.EditMessageRequest{
OwnerUserID: sender.ID,
Peer: domain.Peer{Type: domain.PeerTypeUser, ID: recipient.ID},
ID: sent.SenderMessage.ID,
SetRichMessage: true,
Media: noticeMedia,
RetentionPurge: true,
}
res, err := messages.EditMessage(ctx, req)
if err != nil {
t.Fatalf("EditMessage(RetentionPurge): %v", err)
}
self := res.Self()
if self.Message.Media == nil || self.Message.Media.Kind != domain.MessageMediaKindService {
t.Fatalf("edited message media = %+v, want service kind", self.Message.Media)
}
if self.Message.Media.ServiceAction == nil || self.Message.Media.ServiceAction.Kind != domain.MessageServiceActionCustomText {
t.Fatalf("edited message service action = %+v, want custom_text", self.Message.Media.ServiceAction)
}
if self.Message.Media.ServiceAction.Text != domain.RetentionPurgeNoticeText {
t.Fatalf("edited message notice text = %q, want %q", self.Message.Media.ServiceAction.Text, domain.RetentionPurgeNoticeText)
}
// A second retention-purge edit is a no-op: the message is already the
// notice, so this must not keep bumping pts / re-writing it forever.
if _, err := messages.EditMessage(ctx, req); err != domain.ErrMessageNotModified {
t.Fatalf("second RetentionPurge edit err = %v, want ErrMessageNotModified", err)
}
// A non-author, non-bypass edit on someone else's message must still be
// rejected -- RetentionPurge only bypasses the author check when it is
// actually set, not for ordinary edits in general.
ordinary := req
ordinary.RetentionPurge = false
ordinary.OwnerUserID = recipient.ID
ordinary.Peer = domain.Peer{Type: domain.PeerTypeUser, ID: sender.ID}
if _, err := messages.EditMessage(ctx, ordinary); err != domain.ErrMessageAuthorRequired {
t.Fatalf("non-bypass edit by non-author err = %v, want ErrMessageAuthorRequired", err)
}
}
// TestRetentionPurgeBypassProducesServiceMessage_ChannelMessage is the
// channel counterpart of the private-message test above: channel messages
// store their service action in a structurally separate Action column (see
// tgChannelMessage), so applyRetentionPurgeChannelMessage must clear body/
// media and set Action instead of replacing Media like the private path.
func TestRetentionPurgeBypassProducesServiceMessage_ChannelMessage(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
suffix := randomSuffix(t)
users := NewUserStore(pool)
owner, err := users.Create(ctx, domain.User{AccessHash: 83, Phone: "+1997" + suffix + "03", FirstName: "RetentionChOwner"})
if err != nil {
t.Fatalf("create owner: %v", err)
}
t.Cleanup(func() { _, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = $1", owner.ID) })
channels := NewChannelStore(pool)
created, err := channels.CreateChannel(ctx, domain.CreateChannelRequest{CreatorUserID: owner.ID, Title: "Retention " + suffix, Megagroup: true, Date: 1700003000})
if err != nil {
t.Fatalf("create channel: %v", err)
}
channelID := created.Channel.ID
t.Cleanup(func() { _, _ = pool.Exec(ctx, "DELETE FROM channels WHERE id = $1", channelID) })
sent, err := channels.SendChannelMessage(ctx, domain.SendChannelMessageRequest{
UserID: owner.ID, ChannelID: channelID, RandomID: 771122, Message: "old file",
Media: &domain.MessageMedia{Kind: domain.MessageMediaKindDocument, Document: &domain.Document{ID: time.Now().UnixNano(), AccessHash: 1}},
Date: 1700003001,
})
if err != nil {
t.Fatalf("send channel message: %v", err)
}
req := domain.EditChannelMessageRequest{
ChannelID: channelID,
ID: sent.Message.ID,
RetentionPurge: true,
RetentionPurgeAction: &domain.ChannelMessageAction{Type: domain.ChannelActionCustomText, Text: domain.RetentionPurgeNoticeText},
}
res, err := channels.EditChannelMessage(ctx, req)
if err != nil {
t.Fatalf("EditChannelMessage(RetentionPurge): %v", err)
}
if res.Message.Action == nil || res.Message.Action.Type != domain.ChannelActionCustomText {
t.Fatalf("edited channel message action = %+v, want custom_text", res.Message.Action)
}
if res.Message.Action.Text != domain.RetentionPurgeNoticeText {
t.Fatalf("edited channel message notice text = %q, want %q", res.Message.Action.Text, domain.RetentionPurgeNoticeText)
}
if res.Message.Body != "" || !res.Message.Media.IsZero() {
t.Fatalf("edited channel message body/media not cleared: body=%q media=%+v", res.Message.Body, res.Message.Media)
}
// Idempotency guard: a second retention-purge edit on an already-service
// message must not succeed again.
if _, err := channels.EditChannelMessage(ctx, req); err != domain.ErrMessageNotModified {
t.Fatalf("second RetentionPurge channel edit err = %v, want ErrMessageNotModified", err)
}
// Read-back via ListChannelHistory renders as a service message too.
hist, err := channels.ListChannelHistory(ctx, owner.ID, domain.ChannelHistoryFilter{ChannelID: channelID, Limit: 10})
if err != nil {
t.Fatalf("list channel history: %v", err)
}
var found bool
for _, m := range hist.Messages {
if m.ID == sent.Message.ID {
found = true
if m.Action == nil || m.Action.Type != domain.ChannelActionCustomText {
t.Fatalf("history action = %+v, want custom_text", m.Action)
}
}
}
if !found {
t.Fatalf("purged message %d not found in channel history", sent.Message.ID)
}
}
// TestDocumentCategoryPopulatedOnSend guards Part 2's schema/write-path
// change: sending a document message stamps the already-computed
// domain.DocumentMediaCategory value directly onto the documents row, at the
// same point the shared message_box_media index is written -- zero new
// classification logic.
func TestDocumentCategoryPopulatedOnSend(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
suffix := randomSuffix(t)
users := NewUserStore(pool)
sender, err := users.Create(ctx, domain.User{AccessHash: 84, Phone: "+1997" + suffix + "04", FirstName: "CatSender"})
if err != nil {
t.Fatalf("create sender: %v", err)
}
recipient, err := users.Create(ctx, domain.User{AccessHash: 85, Phone: "+1997" + suffix + "05", FirstName: "CatRecipient"})
if err != nil {
t.Fatalf("create recipient: %v", err)
}
ids := []int64{sender.ID, recipient.ID}
docID := time.Now().UnixNano()
t.Cleanup(func() {
_, _ = pool.Exec(ctx, "DELETE FROM message_boxes WHERE owner_user_id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM private_messages WHERE sender_user_id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM dialogs WHERE user_id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM users WHERE id = ANY($1::bigint[])", ids)
_, _ = pool.Exec(ctx, "DELETE FROM documents WHERE id = $1", docID)
_, _ = pool.Exec(ctx, "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
})
media := NewMediaStore(pool)
if err := media.PutDocument(ctx, domain.Document{ID: docID, MimeType: "video/mp4", Size: 2048}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
messages := NewMessageStore(pool, WithMessageAllocators(&perUserCounterAllocator{}))
_, err = messages.SendPrivateText(ctx, domain.SendPrivateTextRequest{
SenderUserID: sender.ID,
RecipientUserID: recipient.ID,
RandomID: time.Now().UnixNano(),
Media: &domain.MessageMedia{
Kind: domain.MessageMediaKindDocument,
Document: &domain.Document{
ID: docID, AccessHash: 1, MimeType: "video/mp4",
Attributes: []domain.DocumentAttribute{{Kind: domain.DocAttrVideo, W: 640, H: 480, Duration: 5}},
},
},
Date: int(time.Now().Unix()),
})
if err != nil {
t.Fatalf("send video document: %v", err)
}
var category int16
if err := pool.QueryRow(ctx, "SELECT category FROM documents WHERE id = $1", docID).Scan(&category); err != nil {
t.Fatalf("read document category: %v", err)
}
if domain.MediaCategory(category) != domain.MediaCategoryVideo {
t.Fatalf("document category = %d, want %d (Video)", category, domain.MediaCategoryVideo)
}
}
// TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory guards Part 2's
// query-layer change: ListDocumentIDsForHardRetentionOlderThan now takes a
// category argument and must only return documents in that exact bucket,
// even when another old, blob-owning document sits in a different category.
func TestHardRetentionCategoryFilterOnlyReturnsMatchingCategory(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
videoID := time.Now().UnixNano()
musicID := videoID + 1
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = ANY($1::bigint[])", []int64{videoID, musicID})
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = ANY($1::text[])",
[]string{"doc:" + strconv.FormatInt(videoID, 10), "doc:" + strconv.FormatInt(musicID, 10)})
})
for _, d := range []struct {
id int64
category domain.MediaCategory
}{{videoID, domain.MediaCategoryVideo}, {musicID, domain.MediaCategoryMusic}} {
if err := media.PutDocument(ctx, domain.Document{ID: d.id, MimeType: "application/octet-stream", Size: 512}); err != nil {
t.Fatalf("PutDocument %d: %v", d.id, err)
}
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days', category = $2 WHERE id = $1", d.id, int16(d.category)); err != nil {
t.Fatalf("backdate/categorize document %d: %v", d.id, err)
}
blob := postgresTestBlob("doc:"+strconv.FormatInt(d.id, 10), "cat-filter", 512, "application/octet-stream")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob %d: %v", d.id, err)
}
}
cutoff := time.Now().Add(-24 * time.Hour)
videoIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryVideo, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Video): %v", err)
}
if !containsInt64(videoIDs, videoID) || containsInt64(videoIDs, musicID) {
t.Fatalf("video-category candidates = %v, want to include %d and exclude %d", videoIDs, videoID, musicID)
}
musicIDs, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, domain.MediaCategoryMusic, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan(Music): %v", err)
}
if !containsInt64(musicIDs, musicID) || containsInt64(musicIDs, videoID) {
t.Fatalf("music-category candidates = %v, want to include %d and exclude %d", musicIDs, musicID, videoID)
}
}
// TestEvictionListsOldestAcrossDocumentsAndPhotos guards Part 3's eviction
// query layer: ListOldestMediaForEviction must return candidates from both
// tables (interleaving by created_at is done by the caller in Go, see
// files.Service.EvictOldestMediaOverBudget) so the oldest-overall item can be
// picked regardless of which table it lives in.
func TestEvictionListsOldestAcrossDocumentsAndPhotos(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
docID := time.Now().UnixNano()
photoID := docID + 1
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
_, _ = pool.Exec(context.Background(), "DELETE FROM photos WHERE id = $1", photoID)
_, _ = pool.Exec(context.Background(), "DELETE FROM file_blobs WHERE location_key = ANY($1::text[])",
[]string{"doc:" + strconv.FormatInt(docID, 10), "photo:" + strconv.FormatInt(photoID, 10)})
})
if err := media.PutDocument(ctx, domain.Document{ID: docID, MimeType: "application/octet-stream", Size: 256}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
// The document is the older of the two (200 days vs 100 days) -- it must
// sort before the photo in the merged eviction candidate list.
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '200 days' WHERE id = $1", docID); err != nil {
t.Fatalf("backdate document: %v", err)
}
if err := media.PutFileBlob(ctx, postgresTestBlob("doc:"+strconv.FormatInt(docID, 10), "evict-doc", 256, "application/octet-stream")); err != nil {
t.Fatalf("PutFileBlob doc: %v", err)
}
if err := media.PutPhoto(ctx, domain.Photo{ID: photoID, AccessHash: 1}); err != nil {
t.Fatalf("PutPhoto: %v", err)
}
if _, err := pool.Exec(ctx, "UPDATE photos SET created_at = now() - interval '100 days' WHERE id = $1", photoID); err != nil {
t.Fatalf("backdate photo: %v", err)
}
if err := media.PutFileBlob(ctx, postgresTestBlob("photo:"+strconv.FormatInt(photoID, 10), "evict-photo", 256, "image/jpeg")); err != nil {
t.Fatalf("PutFileBlob photo: %v", err)
}
candidates, err := media.ListOldestMediaForEviction(ctx, 1000)
if err != nil {
t.Fatalf("ListOldestMediaForEviction: %v", err)
}
var doc, photo *domain.EvictionCandidate
for i := range candidates {
c := candidates[i]
switch {
case c.Kind == domain.MediaKindDocument && c.MediaID == docID:
doc = &candidates[i]
case c.Kind == domain.MediaKindPhoto && c.MediaID == photoID:
photo = &candidates[i]
}
}
if doc == nil || photo == nil {
t.Fatalf("eviction candidates missing doc/photo: %+v", candidates)
}
if !doc.CreatedAt.Before(photo.CreatedAt) {
t.Fatalf("document created_at %v not before photo created_at %v (document should be older)", doc.CreatedAt, photo.CreatedAt)
}
}

244
internal/store/postgres/sqlcgen/media.sql.go
View file

@ -825,20 +825,103 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
return items, nil
}
const listAvatarOrphanedPhotoIDsOlderThan = `-- name: ListAvatarOrphanedPhotoIDsOlderThan :many
SELECT p.id FROM photos p
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < $1::timestamptz
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
ORDER BY p.orphaned_at ASC
LIMIT $2::int
`
type ListAvatarOrphanedPhotoIDsOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// Same as ListOrphanedPhotoIDsOlderThan but only photos currently active as
// someone's avatar (profile_photos.active) -- lets the Avatar category carry
// its own retention age. In practice a live avatar is never orphaned (an
// active profile_photos row is itself a media_references entry), so this
// bucket is expected to stay empty under "orphan" mode; kept for symmetry
// with the "hard" mode avatar split, which is the one that actually matters.
func (q *Queries) ListAvatarOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListAvatarOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listAvatarOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listAvatarPhotoIDsForHardRetentionOlderThan = `-- name: ListAvatarPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
AND EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT $2::int
`
type ListAvatarPhotoIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// Same as ListPhotoIDsForHardRetentionOlderThan but only photos currently
// active as someone's avatar (profile_photos.active) -- lets the Avatar
// category carry its own retention age, independent of ordinary shared-media
// photos.
func (q *Queries) ListAvatarPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListAvatarPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listAvatarPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
SELECT d.id FROM documents d
WHERE d.created_at < $1::timestamptz
AND d.category = $2::smallint
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT $2::int
LIMIT $3::int
`
type ListDocumentIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
Category int16
BatchLimit int32
}
@ -848,9 +931,11 @@ type ListDocumentIDsForHardRetentionOlderThanParams struct {
// keeps this sweep from re-selecting the same document forever: once its
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
// rows but deliberately leaves this documents row in place), it naturally
// drops out of this query on the next pass.
// drops out of this query on the next pass. category scopes to one
// documents.category bucket per sweep tick -- see
// ListOrphanedDocumentIDsOlderThan for the 0/None fallback note.
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
if err != nil {
return nil, err
}
@ -917,20 +1002,156 @@ func (q *Queries) ListFileBlobsByLocationPrefix(ctx context.Context, arg ListFil
return items, nil
}
const listMediaReferences = `-- name: ListMediaReferences :many
SELECT media_kind, media_id, ref_kind, ref_key
FROM media_references
WHERE media_kind = $1::text AND media_id = $2::bigint
`
type ListMediaReferencesParams struct {
MediaKind string
MediaID int64
}
type ListMediaReferencesRow struct {
MediaKind string
MediaID int64
RefKind string
RefKey string
}
// Every live reference to a document/photo -- used by the storage retention
// sweep to turn a hard-retention/eviction blob purge into a visible notice on
// every message that still embeds the purged media (see
// files.notifyRetentionPurge). profile_photo/sticker_set/gift refs have no
// message to edit and are filtered by the caller, not this query.
func (q *Queries) ListMediaReferences(ctx context.Context, arg ListMediaReferencesParams) ([]ListMediaReferencesRow, error) {
rows, err := q.db.Query(ctx, listMediaReferences, arg.MediaKind, arg.MediaID)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListMediaReferencesRow
for rows.Next() {
var i ListMediaReferencesRow
if err := rows.Scan(
&i.MediaKind,
&i.MediaID,
&i.RefKind,
&i.RefKey,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listOldestDocumentsForEviction = `-- name: ListOldestDocumentsForEviction :many
SELECT d.id, d.created_at FROM documents d
WHERE EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT $1::int
`
type ListOldestDocumentsForEvictionRow struct {
ID int64
CreatedAt pgtype.Timestamptz
}
// Active eviction (TELESRV_STORAGE_EVICTION_ENABLE): candidates are every
// document that still owns at least one file_blobs row, oldest-uploaded
// first, REGARDLESS of category or age -- unlike the retention sweeps above,
// eviction only cares about reclaiming bytes once the total physical budget
// (TELESRV_STORAGE_MAX_TOTAL_BYTES) is exceeded. created_at is returned so
// the caller can interleave these with ListOldestPhotosForEviction by actual
// age instead of draining one table before touching the other.
func (q *Queries) ListOldestDocumentsForEviction(ctx context.Context, batchLimit int32) ([]ListOldestDocumentsForEvictionRow, error) {
rows, err := q.db.Query(ctx, listOldestDocumentsForEviction, batchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListOldestDocumentsForEvictionRow
for rows.Next() {
var i ListOldestDocumentsForEvictionRow
if err := rows.Scan(&i.ID, &i.CreatedAt); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listOldestPhotosForEviction = `-- name: ListOldestPhotosForEviction :many
SELECT p.id, p.created_at FROM photos p
WHERE EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT $1::int
`
type ListOldestPhotosForEvictionRow struct {
ID int64
CreatedAt pgtype.Timestamptz
}
// See ListOldestDocumentsForEviction -- same oldest-first eviction candidate
// selection, for photos (avatars included: eviction is bytes-only and does
// not honor the Avatar category's separate retention age).
func (q *Queries) ListOldestPhotosForEviction(ctx context.Context, batchLimit int32) ([]ListOldestPhotosForEvictionRow, error) {
rows, err := q.db.Query(ctx, listOldestPhotosForEviction, batchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []ListOldestPhotosForEvictionRow
for rows.Next() {
var i ListOldestPhotosForEvictionRow
if err := rows.Scan(&i.ID, &i.CreatedAt); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listOrphanedDocumentIDsOlderThan = `-- name: ListOrphanedDocumentIDsOlderThan :many
SELECT id FROM documents
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
AND category = $2::smallint
ORDER BY orphaned_at ASC
LIMIT $2::int
LIMIT $3::int
`
type ListOrphanedDocumentIDsOlderThanParams struct {
Cutoff pgtype.Timestamptz
Category int16
BatchLimit int32
}
// category selects one documents.category bucket per sweep tick (per-category
// retention age, see internal/app/files/retention.go) -- 0 (MediaCategoryNone)
// covers unclassified documents (e.g. stickers), which always use the shared
// global age since there is no per-category override for that bucket.
func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg ListOrphanedDocumentIDsOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.BatchLimit)
rows, err := q.db.Query(ctx, listOrphanedDocumentIDsOlderThan, arg.Cutoff, arg.Category, arg.BatchLimit)
if err != nil {
return nil, err
}
@ -950,9 +1171,10 @@ func (q *Queries) ListOrphanedDocumentIDsOlderThan(ctx context.Context, arg List
}
const listOrphanedPhotoIDsOlderThan = `-- name: ListOrphanedPhotoIDsOlderThan :many
SELECT id FROM photos
WHERE orphaned_at IS NOT NULL AND orphaned_at < $1::timestamptz
ORDER BY orphaned_at ASC
SELECT p.id FROM photos p
WHERE p.orphaned_at IS NOT NULL AND p.orphaned_at < $1::timestamptz
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
ORDER BY p.orphaned_at ASC
LIMIT $2::int
`
@ -961,6 +1183,8 @@ type ListOrphanedPhotoIDsOlderThanParams struct {
BatchLimit int32
}
// Excludes photos currently active as someone's avatar -- see
// ListAvatarOrphanedPhotoIDsOlderThan for that split-off bucket.
func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrphanedPhotoIDsOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listOrphanedPhotoIDsOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
@ -984,6 +1208,7 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
AND NOT EXISTS (SELECT 1 FROM profile_photos pp WHERE pp.photo_id = p.id AND pp.active)
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
@ -999,7 +1224,8 @@ type ListPhotoIDsForHardRetentionOlderThanParams struct {
}
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
// candidate selection, for photos.
// candidate selection, for photos. Excludes photos currently active as
// someone's avatar -- see ListAvatarPhotoIDsForHardRetentionOlderThan.
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {

1
internal/store/postgres/sqlcgen/models.go
View file

@ -1252,6 +1252,7 @@ type Document struct {
CreatedAt pgtype.Timestamptz
OwnerUserID int64
OrphanedAt pgtype.Timestamptz
Category int16
}
type EncryptedFile struct {