astra/owpengram-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

adding more functions to media managament system

Browse source
This commit is contained in:
onysd 2026-09-03 00:54:09 +03:00
parent 95e62c2d77
commit 70c0ba44f0
30 changed files with 1494 additions and 104 deletions
Download patch file Download diff file Expand all files Collapse all files

50
.env.example
View file

@ -292,14 +292,33 @@ TELESRV_S3_USE_SSL=false
# MinIO needs this on (bucket in the URL path); AWS S3 does not.
TELESRV_S3_PATH_STYLE=true
# Reject new uploads once storage is nearly full, instead of letting the disk
# fill up. Thresholds live in the Advanced section below.
# fill up. Thresholds are the three fields right below.
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
# Automatically delete old media once it's no longer referenced by any
# message, profile photo, or sticker set (never deletes media still visible
# in a conversation). Off by default -- storage usage is tracked and shown
# in the admin panel either way; enable this once you're comfortable with
# what it will reclaim. Retention age lives in the Advanced section below.
TELESRV_STORAGE_RETENTION_ENABLE=false
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
# Reject new uploads once total tracked blob bytes would exceed this. The only
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
# optional soft cap on localfs too. <=0 disables.
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
# Reject a single upload once its total assembled size (sum of all its parts)
# would exceed this. <=0 disables this check (the protocol's own part-count
# ceiling of ~4GB still applies). Must not exceed that ceiling.
TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=0
# Storage retention sweep mode: "off" (default, nothing auto-deleted --
# storage usage is still tracked and shown in the admin panel either way),
# "orphan" (safe: deletes a document/photo's blob only once it's no longer
# referenced by any message/profile photo/sticker set), or "hard"
# (aggressive: deletes a document/photo's blob once it's old enough,
# REGARDLESS of whether it's still referenced -- old media in active
# conversations will show as unavailable).
TELESRV_STORAGE_RETENTION_MODE=off
# How long a document/photo must have had zero references before the sweep
# deletes it in "orphan" mode above -- or how old the media itself is before
# "hard" mode deletes its bytes regardless of references. Ignored when the
# mode above is "off". The sweep itself runs alongside every other retention
# check on the shared TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH
# cadence (Advanced section below).
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
# ==============================================================================
@ -438,22 +457,9 @@ TELESRV_GIF_SEED_DIR=data/gifs
# read fresh on every request -- editing them takes effect with no restart.
TELESRV_IDENTITY_DIR=data/identity
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above).
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
# Reject new uploads once total tracked blob bytes would exceed this. The only
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
# optional soft cap on localfs too. <=0 disables.
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
# How often the cached free-space/usage gauge behind the guard above refreshes.
# How often the cached free-space/usage gauge behind the low-space guard
# (thresholds now live in the Storage & Media section above) refreshes.
TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m
# Storage retention sweep tuning (master toggle is TELESRV_STORAGE_RETENTION_ENABLE above).
# How long a document/photo must have had zero references before the sweep
# deletes it -- not how old the media itself is, and it never touches media
# still referenced by a live message/profile-photo/sticker-set. The sweep
# itself runs alongside every other retention check on the shared
# TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH cadence above.
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
# New-account perks: free Telegram Premium months.
TELESRV_PREMIUM_GRANT_MONTHS=3

57
cmd/telesrv-admin/readstore.go
View file

@ -2612,11 +2612,25 @@ type AccountStorageRow struct {
FileCount int64 `json:"FileCount,string"`
}
// ListAccountStorageUsage pages the per-account storage breakdown, largest
// user first. Offset-based (not keyset): storage administration on a
// self-hosted deployment doesn't need to support arbitrarily deep pages
// efficiently the way an infinite-scroll feed does.
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) {
// storageUsageSortColumns whitelists the columns ListAccountStorageUsage's
// sortBy may map to -- never interpolate the caller's sort key directly into
// SQL, since unlike a plain value it can't go through a query parameter.
var storageUsageSortColumns = map[string]string{
"bytes": "t.bytes",
"files": "t.file_count",
"user_id": "t.owner_user_id",
"username": "lower(COALESCE(u.username, ''))",
"first_name": "lower(COALESCE(u.first_name, ''))",
}
// ListAccountStorageUsage pages the per-account storage breakdown. sortBy
// selects one of storageUsageSortColumns (falls back to "bytes" for an
// unknown/empty key); sortDesc reverses it. q, if non-empty, filters to
// accounts whose id/username/first name match it. Offset-based (not
// keyset): storage administration on a self-hosted deployment doesn't need
// to support arbitrarily deep pages efficiently the way an infinite-scroll
// feed does.
func (s *readStore) ListAccountStorageUsage(ctx context.Context, q string, sortBy string, sortDesc bool, offset, limit int) ([]AccountStorageRow, bool, error) {
if limit <= 0 {
limit = storageUsageListDefaultLimit
}
@ -2626,6 +2640,32 @@ func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit i
if offset < 0 {
offset = 0
}
column, ok := storageUsageSortColumns[sortBy]
if !ok {
column = storageUsageSortColumns["bytes"]
}
direction := "ASC"
if sortDesc {
direction = "DESC"
}
q = strings.TrimSpace(q)
var whereClause string
args := []any{}
argN := 1
if q != "" {
id := int64(-1)
if n, err := strconv.ParseInt(q, 10, 64); err == nil {
id = n
}
whereClause = fmt.Sprintf(`WHERE t.owner_user_id = $%d
OR lower(COALESCE(u.username, '')) LIKE $%d
OR lower(COALESCE(u.first_name, '')) LIKE $%d`, argN, argN+1, argN+2)
args = append(args, id, "%"+strings.ToLower(q)+"%", "%"+strings.ToLower(q)+"%")
argN += 3
}
offsetArg := argN
limitArg := argN + 1
args = append(args, offset, limit+1)
rows, err := s.pool.Query(ctx, `
WITH totals AS (
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
@ -2636,9 +2676,10 @@ WITH totals AS (
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
FROM totals t
LEFT JOIN users u ON u.id = t.owner_user_id
ORDER BY t.bytes DESC, t.owner_user_id
OFFSET $1
LIMIT $2`, offset, limit+1)
`+whereClause+`
ORDER BY `+column+` `+direction+`, t.owner_user_id
OFFSET $`+strconv.Itoa(offsetArg)+`
LIMIT $`+strconv.Itoa(limitArg), args...)
if err != nil {
return nil, false, fmt.Errorf("list account storage usage: %w", err)
}

9
cmd/telesrv-admin/server.go
View file

@ -2498,7 +2498,14 @@ func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request
writeAPIError(w, http.StatusBadRequest, "invalid limit")
return
}
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit)
sortDesc := query.Get("order") != "asc"
rows, hasMore, err := s.read.ListAccountStorageUsage(
r.Context(),
query.Get("q"),
query.Get("sort"),
sortDesc,
offset,
limit)
if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error())
return

1
cmd/telesrv-admin/web/dist/assets/index-B7hI8ol7.css vendored Normal file
View file

File diff suppressed because one or more lines are too long

9
cmd/telesrv-admin/web/dist/assets/index-CP20QCwX.js vendored
View file

File diff suppressed because one or more lines are too long

9
cmd/telesrv-admin/web/dist/assets/index-IZM8WmmY.js vendored Normal file
View file

File diff suppressed because one or more lines are too long

1
cmd/telesrv-admin/web/dist/assets/index-Wtwg__bv.css vendored
View file

File diff suppressed because one or more lines are too long

4
cmd/telesrv-admin/web/dist/index.html vendored
View file

@ -23,8 +23,8 @@
})();
</script>
<script type="module" crossorigin src="/assets/index-CP20QCwX.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-Wtwg__bv.css">
<script type="module" crossorigin src="/assets/index-IZM8WmmY.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
</head>
<body>
<div id="root"></div>

418
cmd/telesrv-admin/web/src/pages/StoragePage.tsx
View file

@ -1,18 +1,52 @@
import { ChevronDown, Loader2, RefreshCw } from "lucide-react";
import { useEffect, useState } from "react";
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search } from "lucide-react";
import { useEffect, useMemo, useState } from "react";
import { api, errorMessage } from "../api";
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui";
import { ActionButton } from "../components/ActionButton";
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
import type { Navigate } from "../routing";
import type { AccountStorageRow, StorageStatsResponse } from "../types";
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
type StorageSortKey = "user_id" | "username" | "bytes" | "files";
// SortableHeader is a <th> that toggles ascending/descending on click and
// shows which column (and direction) is currently active -- there's no
// existing sortable-table convention elsewhere in this admin panel to
// mirror, so this is a small, self-contained one for this page.
function SortableHeader({
label,
sortKey,
activeKey,
desc,
onSort
}: {
label: string;
sortKey: StorageSortKey;
activeKey: StorageSortKey;
desc: boolean;
onSort: (key: StorageSortKey) => void;
}) {
const active = sortKey === activeKey;
return (
<th>
<button type="button" className="sort-header" onClick={() => onSort(sortKey)}>
{label}
{active ? (desc ? <ArrowDown size={13} /> : <ArrowUp size={13} />) : <ArrowUpDown size={13} className="sort-header-idle" />}
</button>
</th>
);
}
function StorageOverviewTab() {
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
const [rows, setRows] = useState<AccountStorageRow[]>([]);
const [hasMore, setHasMore] = useState(false);
const [offset, setOffset] = useState(0);
const [busy, setBusy] = useState(false);
const [error, setError] = useState("");
const [q, setQ] = useState("");
const [sortKey, setSortKey] = useState<StorageSortKey>("bytes");
const [sortDesc, setSortDesc] = useState(true);
async function loadStats() {
try {
@ -26,7 +60,13 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
setBusy(true);
setError("");
const at = next ? offset : 0;
const params = new URLSearchParams({ limit: "50", offset: String(at) });
const params = new URLSearchParams({
limit: "50",
offset: String(at),
sort: sortKey,
order: sortDesc ? "desc" : "asc"
});
if (q.trim()) params.set("q", q.trim());
try {
const result = await api.storageAccounts(params);
const page = result.rows ?? [];
@ -48,7 +88,16 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
useEffect(() => {
refresh();
// eslint-disable-next-line react-hooks/exhaustive-deps
}, []);
}, [sortKey, sortDesc]);
function toggleSort(key: StorageSortKey) {
if (key === sortKey) {
setSortDesc((current) => !current);
} else {
setSortKey(key);
setSortDesc(true);
}
}
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
// sum of what the per-account table below adds up to. They legitimately
@ -56,15 +105,7 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
return (
<PageFrame
title={"Storage"}
eyebrow={"Media / Storage usage"}
actions={
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
</button>
}
>
<>
{error && <Alert>{error}</Alert>}
<div className="metric-row">
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
@ -83,14 +124,29 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
/>
</div>
<QueryPanel>
<form className="toolbar" onSubmit={(event) => { event.preventDefault(); void loadAccounts(false); }}>
<label className="searchbox">
<Search size={15} />
<input value={q} onChange={(event) => setQ(event.target.value)} placeholder={"User ID / username / name"} />
</label>
<button className="btn primary icon-text" type="submit" disabled={busy}>
{busy ? <Loader2 size={15} className="spin" /> : <Search size={15} />} {"Search"}
</button>
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
</button>
</form>
</QueryPanel>
<div className="table-wrap">
<table className="data-table">
<thead>
<tr>
<th>{"User ID"}</th>
<th>{"Account"}</th>
<th>{"Storage used"}</th>
<th>{"Files"}</th>
<SortableHeader label={"User ID"} sortKey="user_id" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
</tr>
</thead>
<tbody>
@ -113,6 +169,330 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
</button>
</div>
)}
</>
);
}
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
const [tab, setTab] = useState<"overview" | "limits">("overview");
return (
<PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}>
<div className="tab-bar" role="tablist" aria-label={"Storage sections"}>
<button className={`tab-btn ${tab === "overview" ? "active" : ""}`} type="button" role="tab" aria-selected={tab === "overview"} onClick={() => setTab("overview")}>
{"Overview"}
</button>
<button className={`tab-btn ${tab === "limits" ? "active" : ""}`} type="button" role="tab" aria-selected={tab === "limits"} onClick={() => setTab("limits")}>
{"Limits & Retention"}
</button>
</div>
{tab === "overview" ? <StorageOverviewTab /> : <LimitsRetentionSection />}
</PageFrame>
);
}
// --- Limits & Retention --------------------------------------------------
//
// Edits a handful of TELESRV_STORAGE_* keys through the existing generic
// .env editor endpoints (GET /api/server/env, POST
// /api/actions/update-server-env -- see cmd/telesrv-admin/serversettings.go
// and ServerSettingsPage.tsx's EnvSection, which this mirrors for its
// save/reload flow), but with friendly units instead of raw key=value text:
// GB inputs for byte budgets, a mode dropdown + day count for retention.
const STORAGE_ENV_KEYS = {
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE"
} as const;
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
// can never legally exceed (internal/config validates this server-side too;
// this is just an early, friendlier warning in the form).
const PROTOCOL_UPLOAD_CEILING_BYTES = 524288 * 8000;
const BYTE_UNITS: { label: string; bytes: number }[] = [
{ label: "MB", bytes: 1024 ** 2 },
{ label: "GB", bytes: 1024 ** 3 },
{ label: "TB", bytes: 1024 ** 4 }
];
function bestByteUnit(bytes: number): { label: string; bytes: number } {
for (let i = BYTE_UNITS.length - 1; i >= 0; i--) {
if (bytes >= BYTE_UNITS[i].bytes) return BYTE_UNITS[i];
}
return BYTE_UNITS[1]; // default to GB for small/zero values
}
// parseDurationMinutes reads a Go-style duration string (e.g. "720h", "90m",
// "1h30m") and returns the total as minutes. Unrecognized/empty input is 0.
function parseDurationMinutes(value: string): number {
let totalSeconds = 0;
const re = /(\d+(?:\.\d+)?)\s*(h|m|s)/g;
let match: RegExpExecArray | null;
while ((match = re.exec(value)) !== null) {
const amount = parseFloat(match[1]);
const unit = match[2];
totalSeconds += unit === "h" ? amount * 3600 : unit === "m" ? amount * 60 : amount;
}
return totalSeconds / 60;
}
const DURATION_UNITS: { label: string; minutes: number }[] = [
{ label: "Minutes", minutes: 1 },
{ label: "Hours", minutes: 60 },
{ label: "Days", minutes: 1440 }
];
function bestDurationUnit(minutes: number): { label: string; minutes: number } {
for (let i = DURATION_UNITS.length - 1; i >= 0; i--) {
if (minutes >= DURATION_UNITS[i].minutes) return DURATION_UNITS[i];
}
return DURATION_UNITS[0];
}
// DurationField edits one duration env value (stored as total minutes) as a
// friendly "amount + unit" pair, letting TTL be set in minutes, hours, or
// days rather than being locked to whole days.
function DurationField({
label,
help,
minutes,
disabled,
onChange
}: {
label: string;
help: string;
minutes: string;
disabled?: boolean;
onChange: (minutes: string) => void;
}) {
const totalMinutes = Number(minutes || "0");
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
const amount = totalMinutes > 0 ? totalMinutes / unit.minutes : NaN;
function handleAmountChange(raw: string) {
const parsed = Number(raw);
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
onChange("0");
return;
}
onChange(String(Math.max(1, Math.round(parsed * unit.minutes))));
}
return (
<label className="duration-field">
<span>{label}</span>
<div style={{ display: "flex", gap: 8 }}>
<input
type="number"
min="0"
step="any"
value={Number.isNaN(amount) ? "" : amount}
disabled={disabled}
onChange={(event) => handleAmountChange(event.target.value)}
/>
<select value={unitLabel} disabled={disabled} onChange={(event) => setUnitLabel(event.target.value)} style={{ maxWidth: 100 }}>
{DURATION_UNITS.map((u) => (
<option key={u.label} value={u.label}>{u.label}</option>
))}
</select>
</div>
<span className="env-field-desc">{help}</span>
</label>
);
}
// ByteSizeField edits one byte-count env value as a friendly
// "amount + unit" pair. bytes is the raw byte count as a string (what the
// backend stores); an empty/zero value displays as "Unlimited".
function ByteSizeField({
label,
help,
bytes,
onChange
}: {
label: string;
help: string;
bytes: string;
onChange: (bytes: string) => void;
}) {
const numericBytes = Number(bytes || "0");
const [unitLabel, setUnitLabel] = useState(() => bestByteUnit(numericBytes).label);
const unit = BYTE_UNITS.find((u) => u.label === unitLabel) ?? BYTE_UNITS[1];
const amount = numericBytes > 0 ? numericBytes / unit.bytes : NaN;
function handleAmountChange(raw: string) {
const parsed = Number(raw);
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
onChange("0");
return;
}
onChange(String(Math.round(parsed * unit.bytes)));
}
return (
<label className="duration-field">
<span>{label}</span>
<div style={{ display: "flex", gap: 8 }}>
<input
type="number"
min="0"
step="any"
value={Number.isNaN(amount) ? "" : amount}
placeholder={"Unlimited"}
onChange={(event) => handleAmountChange(event.target.value)}
/>
<select value={unitLabel} onChange={(event) => setUnitLabel(event.target.value)} style={{ maxWidth: 90 }}>
{BYTE_UNITS.map((u) => (
<option key={u.label} value={u.label}>{u.label}</option>
))}
</select>
</div>
<span className="env-field-desc">{help}</span>
</label>
);
}
function LimitsRetentionSection() {
const [loaded, setLoaded] = useState(false);
const [error, setError] = useState("");
const [initial, setInitial] = useState<Record<string, string>>({});
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
const [minFreeBytes, setMinFreeBytes] = useState("0");
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
const [retentionMode, setRetentionMode] = useState("off");
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
async function load() {
setError("");
try {
const groups = await api.serverEnv();
const values: Record<string, string> = {};
for (const group of groups) {
for (const field of group.fields) {
values[field.key] = field.value || field.default_value || "";
}
}
setInitial(values);
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
const mode = (values[STORAGE_ENV_KEYS.retentionMode] || "off").trim().toLowerCase();
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
setLoaded(true);
} catch (err) {
setError(errorMessage(err));
}
}
useEffect(() => { void load(); }, []);
// Only the age is re-encoded from its friendly "days" input back to a Go
// duration string; mode/off keeps whatever TELESRV_STORAGE_RETENTION_MAX_AGE
// already was rather than clobbering it with a throwaway placeholder.
const pendingValues = useMemo(() => {
const next: Record<string, string> = {
[STORAGE_ENV_KEYS.maxTotal]: maxTotalBytes || "0",
[STORAGE_ENV_KEYS.minFree]: minFreeBytes || "0",
[STORAGE_ENV_KEYS.maxUploadFile]: maxUploadFileBytes || "0",
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
: `${Math.max(1, Math.round(Number(retentionAgeMinutes || "0")))}m`
};
const changed: Record<string, string> = {};
for (const [key, value] of Object.entries(next)) {
if ((initial[key] ?? "") !== value) changed[key] = value;
}
return changed;
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, initial]);
const hasChanges = Object.keys(pendingValues).length > 0;
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
return (
<section className="section-block">
<SectionHead
title={"Limits & Retention"}
text={"Server-wide storage budget, per-file upload cap, and automatic media cleanup. Saved to .env -- takes effect on the next Restart/Update."}
/>
{error && <Alert>{error}</Alert>}
{!loaded ? (
<p style={{ color: "var(--muted)" }}>{"Loading current settings..."}</p>
) : (
<div className="card-body">
<div className="attr-block">
<ByteSizeField
label={"Max total storage budget"}
help={"Reject new uploads once total tracked blob bytes would exceed this. Empty/0 = unlimited."}
bytes={maxTotalBytes}
onChange={setMaxTotalBytes}
/>
<ByteSizeField
label={"Min free space guard"}
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
bytes={minFreeBytes}
onChange={setMinFreeBytes}
/>
<ByteSizeField
label={"Max single file size"}
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
bytes={maxUploadFileBytes}
onChange={setMaxUploadFileBytes}
/>
{uploadCeilingExceeded && (
<Alert>{"This exceeds the protocol's own ~4GB upload ceiling and will be refused when the server restarts."}</Alert>
)}
</div>
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<label className="duration-field">
<span>{"Retention mode"}</span>
<select value={retentionMode} onChange={(event) => setRetentionMode(event.target.value)}>
<option value="off">{"Off"}</option>
<option value="orphan">{"Delete once no longer used (safe)"}</option>
<option value="hard">{"Delete after a fixed time, even if still in use"}</option>
</select>
</label>
<p className="env-field-desc">
{retentionMode === "off" &&
"No storage sweep runs; nothing is auto-deleted. Storage usage is still tracked and shown above either way."}
{retentionMode === "orphan" &&
"Safe: a document or photo's file is deleted only once it is no longer referenced by any message, profile photo, or sticker set. Media still visible in a conversation is never touched, regardless of age."}
{retentionMode === "hard" &&
"Irreversible and aggressive: a document or photo's file bytes are deleted once old enough, REGARDLESS of whether a message still references it. Old media in active conversations will start showing as unavailable once purged -- only the file is removed, the message itself keeps rendering its placeholder (name, size, thumbnail)."}
</p>
<DurationField
label={"Retention age"}
help={
retentionMode === "hard"
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
: "How long a document or photo must have had zero references before its file is deleted."
}
minutes={retentionAgeMinutes}
disabled={retentionMode === "off"}
onChange={setRetentionAgeMinutes}
/>
</div>
<div className="gift-table-actions env-save-row">
<ActionButton
tone="warn"
label={"Save limits & retention settings"}
path="/api/actions/update-server-env"
payload={() => ({ values: pendingValues })}
disabled={!hasChanges || uploadCeilingExceeded}
onDone={() => void load()}
/>
</div>
</div>
)}
</section>
);
}

21
cmd/telesrv-admin/web/src/styles/02-pages-and-forms.css
View file

@ -779,6 +779,27 @@ textarea:focus {
font-weight: 800;
}
.sort-header {
display: inline-flex;
align-items: center;
gap: 5px;
padding: 0;
border: 0;
background: transparent;
color: inherit;
font: inherit;
font-weight: 800;
cursor: pointer;
}
.sort-header:hover {
color: var(--text);
}
.sort-header-idle {
opacity: 0.45;
}
.data-table tbody tr:hover {
background: var(--panel-subtle);
}

21
cmd/telesrv/main.go
View file

@ -932,6 +932,7 @@ func run(logger *zap.Logger) error {
filesapp.WithUploadPartBackend(localBlobFS),
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
filesapp.WithSpaceGuard(spaceGuard),
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
externalMediaOption(cfg),
webPagePreviewOption(cfg),
@ -1063,11 +1064,7 @@ func run(logger *zap.Logger) error {
readModelVersionStore,
cfg.UserProjectionFactCacheMaxEntries,
)
storageRetentionMaxAge := cfg.StorageRetentionMaxAge
if !cfg.StorageRetentionEnable {
storageRetentionMaxAge = 0
}
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
retentionWorker := maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
cfg.UpdateEventRetention,
cfg.RetentionInterval,
cfg.RetentionBatch,
@ -1080,9 +1077,17 @@ func run(logger *zap.Logger) error {
WithModerationRetention(moderationReportStore).
WithUserUpdateRetention(updateEventStore).
WithChannelUpdateRetention(channelStore).
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention).
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge).
Run(ctx)
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention)
// TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
// the orphan-only (safe) and hard (age-based, ignores live references)
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
switch cfg.StorageRetentionMode {
case config.StorageRetentionModeOrphan:
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, cfg.StorageRetentionMaxAge)
case config.StorageRetentionModeHard:
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, cfg.StorageRetentionMaxAge)
}
go retentionWorker.Run(ctx)
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
cfg.UploadPartTTL,
cfg.UploadPartGCInterval,

33
internal/app/files/blobcache.go
View file

@ -87,8 +87,11 @@ func (c *stickerSetNegativeCache) delete(refs ...domain.StickerSetRef) {
// 每个 chunk 一次 GetFileBlob 的 PG 往返（一个文件按 ≤512KB/1MB 分多次 getFile，热门贴纸/
// reaction/头像更被大量用户重复拉）。
//
// FileBlob 元数据小（约百字节）且内容不可变：location_key 一旦写入即固定指向同一 object_key，
// 新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突，故只读填充、无需失效。
// FileBlob 元数据小（约百字节），新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突，
// 故写入路径只读填充、无需失效。但一个 location_key 的 file_blobs 行可以被存储 retention
// 清理主动删除（尤其是 hard 模式：不等 orphaned，仍被引用的媒体到期也会被清理字节），这种情况
// 下必须调用 delete 使该 key 失效，否则缓存会继续认为它存在，导致后续下载走到已被删除的字节
// 而不是优雅返回 LOCATION_INVALID。
type blobMetaCache struct {
mu sync.Mutex
cap int
@ -141,6 +144,17 @@ func (c *blobMetaCache) put(key string, blob domain.FileBlob) {
}
}
// delete 使一个 location_key 的缓存元数据立即失效（如果存在）。见上方类型注释：
// 存储 retention 清理主动删掉该 location_key 的 file_blobs 行后必须调用。
func (c *blobMetaCache) delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[key]; ok {
c.ll.Remove(el)
delete(c.m, key)
}
}
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
// 缩略图通常只有几 KB 到几十 KB，缓存全量内容可以避开点击历史时的本地磁盘冷读抖动；
// 大媒体仍由 BlobBackend.GetRange 分段读取，避免把大文件放进内存。
@ -220,6 +234,21 @@ func (c *blobBytesCache) put(key string, bytes []byte) {
}
}
// delete evicts a cached object_key's full bytes (if present) and reclaims
// its budget. Called after a retention sweep physically deletes the object
// from its backend, so a later GetFile can't keep serving bytes that no
// longer exist on disk/S3.
func (c *blobBytesCache) delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[key]; ok {
entry := el.Value.(*blobBytesEntry)
c.ll.Remove(el)
delete(c.m, key)
c.used -= entry.size
}
}
type stickerSetFullCache struct {
mu sync.RWMutex
byID map[int64]stickerSetFullEntry

80
internal/app/files/retention.go
View file

@ -24,6 +24,17 @@ type mediaRetentionStore interface {
// OrphanDocumentIfUnreferenced is the immediate (no grace period)
// counterpart to the age-based sweep above -- see its doc comment.
OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)
// -- "hard" retention mode (TELESRV_STORAGE_RETENTION_MODE=hard) --
// Candidate selection ignores media_references entirely: a document/
// photo still referenced by a live message is exactly as eligible as an
// orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
}
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
@ -68,6 +79,63 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
return deleted, nil
}
// DeleteBlobBytesForMediaOlderThan implements
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
// documents/photos whose upload/created_at is older than cutoff, physically
// deletes their blob bytes (main body + thumbnail/rendition variants) from
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
// live message/profile-photo/sticker-set still references them. It
// deliberately never touches the documents/photos metadata row itself: a
// message must still be able to render "here was a photo/document"
// (dimensions, mime type, filename) after its bytes are gone, rather than
// the message breaking outright. A subsequent upload.getFile for the same
// location key finds no file_blobs row and returns LOCATION_INVALID, which
// stock clients already render as a "media unavailable" placeholder.
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, nil
}
purged := 0
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
if err != nil {
return purged, fmt.Errorf("list documents for hard retention: %w", err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
remaining := limit - len(docIDs)
if remaining <= 0 {
return purged, nil
}
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
return purged, nil
}
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
// id right now (skipping the grace period) and, only if that succeeds --
@ -106,6 +174,17 @@ func (s *Service) deleteDocumentNowIfUnreferenced(ctx context.Context, id int64)
// delete it from.
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
for _, b := range blobs {
// The file_blobs row for this exact location_key is already gone
// (the caller deleted it in the same transaction that produced this
// blob list) -- so any cached "found" metadata for it is now wrong
// regardless of whether the underlying bytes turn out to still be
// shared by another row below. Without this, a hot GetFile path that
// had this location_key's metadata cached would keep trying to read
// bytes that may no longer exist (hard retention mode purges blobs
// for actively-referenced, potentially still-hot media), producing
// an internal error instead of the graceful LOCATION_INVALID a stock
// client knows how to render.
s.blobCache.delete(b.LocationKey)
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
if err != nil {
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
@ -123,5 +202,6 @@ func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionS
if err := backend.Delete(ctx, b.ObjectKey); err != nil {
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
}
s.byteCache.delete(b.ObjectKey)
}
}

18
internal/app/files/service.go
View file

@ -72,6 +72,10 @@ type Service struct {
stickerSetNegCache *stickerSetNegativeCache
uploadQuota domain.UploadPartQuota
spaceGuard SpaceGuard
// maxUploadFileBytes caps a single upload's total assembled size (sum of
// all parts). <=0 means unlimited (still bounded by the protocol's own
// MaxUploadPartBytes*MaxUploadParts ceiling). See WithMaxUploadFileBytes.
maxUploadFileBytes int64
mapTiles *mapTileProxy
externalMedia *externalMediaFetcher
webpage *webpageFetcher
@ -143,6 +147,15 @@ func WithSpaceGuard(guard SpaceGuard) Option {
}
}
// WithMaxUploadFileBytes caps a single uploaded file's total assembled size
// (sum of all parts, not any one part) -- TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES.
// <=0 leaves it unlimited (still bounded by MaxUploadPartBytes*MaxUploadParts).
func WithMaxUploadFileBytes(maxBytes int64) Option {
return func(s *Service) {
s.maxUploadFileBytes = maxBytes
}
}
// WithAdditionalBlobBackend registers a non-active blob backend purely for
// reading/deleting rows written while it used to be active. Register the
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
@ -753,6 +766,11 @@ func (s *Service) loadAndValidateUploadParts(ctx context.Context, ownerUserID, f
return nil, 0, domain.ErrFilePartsInvalid
}
}
// Checked against the total assembled size, not any single part --
// MaxUploadPartBytes above already bounds each part individually.
if s.maxUploadFileBytes > 0 && total > s.maxUploadFileBytes {
return nil, 0, domain.ErrFileTooLarge
}
return parts, total, nil
}

83
internal/app/files/upload_parts_test.go
View file

@ -171,6 +171,89 @@ func TestCreatePhotoFromUploadReceiptReplaysAfterPartCleanup(t *testing.T) {
}
}
// TestMaxUploadFileBytesRejectsOversizedAssembledDocument covers
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES enforcement (WithMaxUploadFileBytes):
// the check must fire against the TOTAL assembled size (sum of every part),
// not any single part -- each individual part here is well under the limit,
// only their sum exceeds it.
func TestMaxUploadFileBytesRejectsOversizedAssembledDocument(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
local, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const partSize = 1024
const maxUploadFileBytes = 2 * partSize // exactly 2 parts' worth; 3 parts must be rejected
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
parts := []string{
strings.Repeat("a", partSize),
strings.Repeat("b", partSize),
strings.Repeat("c", partSize),
}
for i, part := range parts {
if _, err := svc.SaveBigFilePart(ctx, 10, 300, i, len(parts), []byte(part)); err != nil {
t.Fatalf("SaveBigFilePart %d: %v", i, err)
}
}
_, err = svc.CreateDocumentFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 300, Parts: len(parts), Name: "big.bin", Big: true},
domain.DocumentSpec{MimeType: "application/octet-stream"},
)
if !errors.Is(err, domain.ErrFileTooLarge) {
t.Fatalf("CreateDocumentFromUpload over max size err = %v, want ErrFileTooLarge", err)
}
}
// TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit ensures the check
// is an upper bound, not an off-by-one trap: a total exactly at the
// configured ceiling must still succeed.
func TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
local, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const partSize = 1024
const maxUploadFileBytes = 2 * partSize
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
parts := []string{strings.Repeat("a", partSize), strings.Repeat("b", partSize)}
for i, part := range parts {
if _, err := svc.SaveBigFilePart(ctx, 10, 301, i, len(parts), []byte(part)); err != nil {
t.Fatalf("SaveBigFilePart %d: %v", i, err)
}
}
doc, err := svc.CreateDocumentFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 301, Parts: len(parts), Name: "exact.bin", Big: true},
domain.DocumentSpec{MimeType: "application/octet-stream"},
)
if err != nil {
t.Fatalf("CreateDocumentFromUpload at exact max size: %v", err)
}
if doc.Size != int64(maxUploadFileBytes) {
t.Fatalf("doc size = %d, want %d", doc.Size, maxUploadFileBytes)
}
}
// TestMaxUploadFileBytesUnlimitedByDefault ensures leaving
// WithMaxUploadFileBytes unset (or 0) never rejects on size -- only the
// protocol's own MaxUploadPartBytes/MaxUploadParts ceiling still applies.
func TestMaxUploadFileBytesUnlimitedByDefault(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
svc, _ := newUploadPartTestService(t, media, domain.UploadPartQuota{})
file := domain.UploadedFileRef{OwnerUserID: 10, FileID: 302, Parts: 1, Name: "photo.jpg"}
if _, err := svc.SaveFilePart(ctx, file.OwnerUserID, file.FileID, 0, []byte(strings.Repeat("z", 4096))); err != nil {
t.Fatalf("SaveFilePart: %v", err)
}
if _, err := svc.CreatePhotoFromUpload(ctx, file); err != nil {
t.Fatalf("CreatePhotoFromUpload with no configured max size: %v", err)
}
}
type countingUploadPartBackend struct {
*LocalFS
getUploadPartCalls int

31
internal/app/maintenance/media_retention.go
View file

@ -19,9 +19,38 @@ type OrphanedMediaRetentionStore interface {
// how long a document/photo must have been orphaned before it's actually
// deleted (not how old the media itself is) -- <=0 leaves the sweep
// disabled even if a store is provided, matching
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
// TELESRV_STORAGE_RETENTION_MODE=off/orphan-with-no-age being the safe
// default. Mutually exclusive with WithHardMediaRetention -- the resolved
// TELESRV_STORAGE_RETENTION_MODE is one of "off"/"orphan"/"hard", never
// both orphan and hard sweeps at once.
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.orphanedMedia = store
w.orphanedMediaMaxAge = maxAge
return w
}
// HardMediaRetentionStore physically deletes a document/photo's blob bytes
// once the media itself (its upload/created time, not how long it has been
// orphaned) is older than the configured age, REGARDLESS of whether a live
// message/profile-photo/sticker-set still references it. Unlike
// OrphanedMediaRetentionStore, it must never delete the document/photo
// metadata row -- only the underlying file_blobs row(s) and, once confirming
// no other row still needs the same content-addressed object, the physical
// bytes. This keeps a message rendering its media placeholder (dimensions,
// mime type, filename) after the bytes are gone; a subsequent download
// resolves to LOCATION_INVALID instead of the message breaking outright.
type HardMediaRetentionStore interface {
DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
}
// WithHardMediaRetention enables the aggressive storage retention sweep
// (TELESRV_STORAGE_RETENTION_MODE=hard). maxAge is how old the media itself
// must be (its created_at, not an orphaned_at grace period) before its blob
// bytes are purged -- <=0 leaves the sweep disabled even if a store is
// provided. Mutually exclusive with WithOrphanedMediaRetention -- call at
// most one of the two, matching the config's single 3-way retention mode.
func (w *RetentionWorker) WithHardMediaRetention(store HardMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.hardMedia = store
w.hardMediaMaxAge = maxAge
return w
}

85
internal/app/maintenance/retention.go
View file

@ -95,6 +95,10 @@ const (
// 继续保留，在线漏推由正常 difference 路径补偿。
defaultOutboxPoisonRetention = time.Minute
defaultOutboxPoisonInterval = 15 * time.Second
// minMediaRetentionInterval floors the derived media-sweep cadence below
// so a very short (e.g. test-only) TELESRV_STORAGE_RETENTION_MAX_AGE
// can't spin the sweep query in a tight loop.
minMediaRetentionInterval = 15 * time.Second
)
// RetentionWorker 周期性回收存储中的死数据。
@ -119,6 +123,7 @@ type RetentionWorker struct {
activeAuthKeys ActiveRawAuthKeyProvider
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
orphanedMedia OrphanedMediaRetentionStore
hardMedia HardMediaRetentionStore
logger *zap.Logger
retention time.Duration
botAPIRetention time.Duration
@ -128,6 +133,7 @@ type RetentionWorker struct {
outboxPoisonRetention time.Duration
outboxPoisonInterval time.Duration
orphanedMediaMaxAge time.Duration
hardMediaMaxAge time.Duration
interval time.Duration
batch int
}
@ -259,6 +265,15 @@ func (w *RetentionWorker) Run(ctx context.Context) {
heartbeatC = heartbeatTicker.C
defer heartbeatTicker.Stop()
}
var (
mediaTicker *time.Ticker
mediaC <-chan time.Time
)
if interval := w.mediaRetentionInterval(); interval > 0 {
mediaTicker = time.NewTicker(interval)
mediaC = mediaTicker.C
defer mediaTicker.Stop()
}
for {
select {
case <-ctx.Done():
@ -269,6 +284,8 @@ func (w *RetentionWorker) Run(ctx context.Context) {
w.runOutboxPoisonOnce(ctx)
case <-heartbeatC:
w.heartbeatActiveAuthKeys(ctx)
case <-mediaC:
w.runMediaRetentionOnce(ctx)
}
}
}
@ -276,6 +293,63 @@ func (w *RetentionWorker) Run(ctx context.Context) {
func (w *RetentionWorker) runOnce(ctx context.Context) {
w.runOutboxPoisonOnce(ctx)
w.runRetentionOnce(ctx)
w.runMediaRetentionOnce(ctx)
}
// mediaRetentionInterval derives how often the storage media sweep
// (orphan/hard) runs, independent of the shared housekeeping w.interval.
// A short TELESRV_STORAGE_RETENTION_MAX_AGE (e.g. 30m) configured under a
// much longer TELESRV_RETENTION_INTERVAL (default 1h) would otherwise let
// media sit for up to age+interval past its cutoff before actually being
// swept -- capping the sweep cadence at the retention age itself bounds
// that worst case to at most 2x the configured age instead.
func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
var maxAge time.Duration
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
maxAge = w.orphanedMediaMaxAge
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
maxAge = w.hardMediaMaxAge
}
if maxAge <= 0 {
return 0
}
interval := w.interval
if interval <= 0 || maxAge < interval {
interval = maxAge
}
if interval < minMediaRetentionInterval {
interval = minMediaRetentionInterval
}
return interval
}
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
// "Hard" mode: purges blob bytes for documents/photos older than
// hardMediaMaxAge regardless of whether a live reference remains --
// the store is required to keep the document/photo metadata row
// intact, only removing file_blobs rows/bytes, so a message still
// renders its media placeholder (LOCATION_INVALID on download)
// instead of breaking outright.
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
} else if hardDeleted > 0 {
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
}
}
}
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
@ -414,17 +488,6 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
}
}
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
}
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {

64
internal/app/maintenance/retention_test.go
View file

@ -393,3 +393,67 @@ func TestRetentionWorkerSkipsOrphanDeleteWhenHeartbeatFails(t *testing.T) {
t.Fatalf("heartbeat failure signals = %+v", entries)
}
}
type fakeHardMediaRetention struct {
calls int
cutoff time.Time
limit int
deleted int
}
func (f *fakeHardMediaRetention) DeleteBlobBytesForMediaOlderThan(_ context.Context, cutoff time.Time, limit int) (int, error) {
f.calls++
f.cutoff = cutoff
f.limit = limit
return f.deleted, nil
}
// TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge guards the fix for
// a real gotcha reported live: with a short storage retention age (e.g. 30m)
// under the default 1h TELESRV_RETENTION_INTERVAL housekeeping cadence,
// media eligible for hard/orphan deletion used to sit for up to age+interval
// before the shared ticker actually got around to sweeping it (a 30m age
// could take up to 1h30m to actually disappear). The sweep now runs on its
// own ticker capped at the configured age, so it never lags by more than
// roughly one age-window.
func TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge(t *testing.T) {
hard := &fakeHardMediaRetention{}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, 30*time.Minute)
if got := w.mediaRetentionInterval(); got != 30*time.Minute {
t.Fatalf("media retention interval = %v, want min(interval=1h, maxAge=30m) = 30m", got)
}
}
func TestRetentionWorkerMediaSweepIntervalFloorsAtMinimum(t *testing.T) {
hard := &fakeHardMediaRetention{}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, time.Second)
if got := w.mediaRetentionInterval(); got != minMediaRetentionInterval {
t.Fatalf("media retention interval = %v, want floor %v", got, minMediaRetentionInterval)
}
}
func TestRetentionWorkerMediaSweepIntervalZeroWhenNoModeConfigured(t *testing.T) {
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50)
if got := w.mediaRetentionInterval(); got != 0 {
t.Fatalf("media retention interval = %v, want 0 (no separate ticker) when retention is off", got)
}
}
// TestRetentionWorkerRunsHardMediaSweepOnStartup confirms runOnce (called
// once immediately when Run starts, matching every other retention check)
// still fires the media sweep too, not just the new dedicated ticker.
func TestRetentionWorkerRunsHardMediaSweepOnStartup(t *testing.T) {
hard := &fakeHardMediaRetention{deleted: 4}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, 30*time.Minute)
w.runOnce(context.Background())
if hard.calls != 1 {
t.Fatalf("hard media sweep calls = %d, want 1", hard.calls)
}
}

86
internal/config/config.go
View file

@ -13,11 +13,20 @@ import (
"golang.org/x/text/language"
"telesrv/internal/app/files"
"telesrv/internal/branding"
"telesrv/internal/domain"
"telesrv/internal/links"
)
// Storage retention modes for TELESRV_STORAGE_RETENTION_MODE. See
// Config.StorageRetentionMode for what each one does.
const (
StorageRetentionModeOff = "off"
StorageRetentionModeOrphan = "orphan"
StorageRetentionModeHard = "hard"
)
const (
defaultConfigFile = ".env"
defaultCountryCode = "CN"
@ -311,19 +320,41 @@ type Config struct {
// the s3 backend (no OS-level free space concept), and usable as an
// optional soft budget cap on localfs too. <=0 disables this check.
StorageMaxTotalBytes int64
// StorageMaxUploadFileBytes caps the total assembled size of a single
// uploaded file (sum of all its parts, not any one part) -- rejected at
// upload-finalize time with a FILE_TOO_BIG rpc error. <=0 disables this
// check (the only remaining ceiling is then the protocol's own part-count
// limit, files.MaxUploadPartBytes*files.MaxUploadParts, ~4GB). Validated
// at config-load time to never exceed that protocol ceiling -- a larger
// value could never actually be reached, so it's refused as a
// startup-time misconfiguration rather than silently accepted as a no-op.
StorageMaxUploadFileBytes int64
// StorageUsageRefreshInterval is how often the cached free-space/budget
// usage gauge refreshes; <=0 uses a 1 minute default.
StorageUsageRefreshInterval time.Duration
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
// default: orphaned media (no live message/profile-photo/sticker-set
// reference) is tracked and visible in the admin panel, but nothing is
// auto-deleted until the operator explicitly opts in.
StorageRetentionEnable bool
// StorageRetentionMaxAge is how long a document/photo must have been
// orphaned (not how old the media itself is) before the sweep deletes
// it. Never deletes media that still has a live reference, regardless
// of age. <=0 uses a 30 day default. The sweep itself runs on the
// shared RetentionInterval/RetentionBatch cadence alongside every other
// StorageRetentionMode selects the storage retention sweep behavior:
// - "off" (default): no sweep at all.
// - "orphan": safe mode -- deletes a document/photo's blob only once it
// has had no live message/profile-photo/sticker-set reference for at
// least StorageRetentionMaxAge. Never touches media still referenced,
// regardless of age.
// - "hard": aggressive mode -- deletes a document/photo's blob bytes
// once the media itself (its upload/created time, not how long it's
// been orphaned) is older than StorageRetentionMaxAge, REGARDLESS of
// whether it's still referenced by a live message. Only the blob
// bytes are removed; the document/photo metadata row is kept so the
// message still renders (dimensions, mime type, filename) --
// subsequent downloads get LOCATION_INVALID ("media no longer
// available") instead of the message breaking outright. This is
// irreversible and can surprise users if misunderstood: old media in
// active conversations WILL disappear.
// Any other value fails config validation.
StorageRetentionMode string
// StorageRetentionMaxAge is the shared age threshold for both "orphan"
// and "hard" modes -- see StorageRetentionMode for how its meaning
// differs between them. Ignored when StorageRetentionMode is "off". <=0
// uses a 30 day default. The sweep itself runs on the shared
// RetentionInterval/RetentionBatch cadence alongside every other
// retention check (see maintenance.RetentionWorker).
StorageRetentionMaxAge time.Duration
// StickerSeedDir 是 reaction / sticker 资源种子目录（导入到 documents/sticker_sets + blob）。
@ -945,8 +976,9 @@ func Load() (Config, error) {
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
StorageMaxUploadFileBytes: envInt64Or("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", 0),
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
@ -1285,6 +1317,9 @@ func validateBlobStorageConfig(cfg Config) error {
if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
}
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
return err
}
if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled")
}
@ -1474,14 +1509,39 @@ func validateStorageConfig(cfg Config) error {
if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
}
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
return err
}
if cfg.StorageUsageRefreshInterval < 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
}
if cfg.StorageRetentionMaxAge < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
}
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
switch cfg.StorageRetentionMode {
case StorageRetentionModeOff:
// no sweep; StorageRetentionMaxAge is ignored.
case StorageRetentionModeOrphan, StorageRetentionModeHard:
if cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
}
default:
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
}
return nil
}
// validateMaxUploadFileBytes checks TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: it
// must be non-negative, and if set (>0) must not exceed the protocol's own
// part-count upload ceiling -- a larger configured value could never actually
// be hit, so it's refused as a startup-time misconfiguration.
func validateMaxUploadFileBytes(v int64) error {
if v < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES must be non-negative")
}
const protocolCeiling = int64(files.MaxUploadPartBytes) * int64(files.MaxUploadParts)
if v > protocolCeiling {
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES (%d) must not exceed the protocol upload ceiling of %d bytes (%d parts x %d bytes)", v, protocolCeiling, files.MaxUploadParts, files.MaxUploadPartBytes)
}
return nil
}

96
internal/config/config_test.go
View file

@ -1,6 +1,7 @@
package config
import (
"fmt"
"os"
"path/filepath"
"testing"
@ -467,6 +468,101 @@ func TestLoadRejectsInvalidStorageCapacityConfig(t *testing.T) {
}
}
func TestLoadMaxUploadFileBytesDefaultsToUnlimited(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageMaxUploadFileBytes != 0 {
t.Fatalf("expected default StorageMaxUploadFileBytes=0 (unlimited), got %d", cfg.StorageMaxUploadFileBytes)
}
}
func TestLoadAcceptsMaxUploadFileBytesAtProtocolCeiling(t *testing.T) {
disableDefaultConfigFile(t)
const ceiling = int64(524288) * 8000 // files.MaxUploadPartBytes * files.MaxUploadParts
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", fmt.Sprint(ceiling))
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageMaxUploadFileBytes != ceiling {
t.Fatalf("expected StorageMaxUploadFileBytes=%d, got %d", ceiling, cfg.StorageMaxUploadFileBytes)
}
}
func TestLoadRejectsInvalidMaxUploadFileBytes(t *testing.T) {
const ceiling = int64(524288) * 8000
for _, item := range []struct {
name string
value string
}{
{"negative", "-1"},
{"exceeds protocol ceiling", fmt.Sprint(ceiling + 1)},
} {
t.Run(item.name, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", item.value)
if _, err := Load(); err == nil {
t.Fatalf("Load accepted invalid TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=%s", item.value)
}
})
}
}
func TestLoadStorageRetentionModeDefaultsToOff(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageRetentionMode != StorageRetentionModeOff {
t.Fatalf("expected default StorageRetentionMode=%q, got %q", StorageRetentionModeOff, cfg.StorageRetentionMode)
}
}
func TestLoadStorageRetentionModeOrphanAndHard(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "48h")
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageRetentionMode != mode {
t.Fatalf("expected StorageRetentionMode=%q, got %q", mode, cfg.StorageRetentionMode)
}
if cfg.StorageRetentionMaxAge != 48*time.Hour {
t.Fatalf("expected StorageRetentionMaxAge=48h, got %v", cfg.StorageRetentionMaxAge)
}
})
}
}
func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", "sometimes")
if _, err := Load(); err == nil {
t.Fatal("Load accepted an unknown TELESRV_STORAGE_RETENTION_MODE")
}
}
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
if _, err := Load(); err == nil {
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
}
})
}
}
func TestLoadUpdateServiceConfig(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/")

5
internal/domain/media_errors.go
View file

@ -9,6 +9,11 @@ var (
ErrFilePartsInvalid = errors.New("file parts invalid")
ErrFilePartTooBig = errors.New("file part too big")
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
// ErrFileTooLarge is returned when a file's total assembled size (sum of
// all its parts, not any single part) exceeds the configured
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES ceiling. Distinct from
// ErrFilePartTooBig, which is a single oversized part.
ErrFileTooLarge = errors.New("file too large")
ErrPhotoInvalid = errors.New("photo invalid")
ErrDocumentInvalid = errors.New("document invalid")
// ErrStorageFull is returned when the configured low-space guard rejects a

53
internal/procctl/env_test.go
View file

@ -133,3 +133,56 @@ func TestWriteEnvValuesRoundTrip(t *testing.T) {
t.Error(".env lost .env.example's comment lines on write")
}
}
// TestWriteEnvValuesPreservesUnrelatedCustomValues guards against a real
// regression: a save that only touches one section's keys used to reset
// every OTHER already-customized key (e.g. TELESRV_ADMIN_UI_PASSWORD) back
// to .env.example's bare template default, because the old implementation
// fell back to the template line instead of the current .env value for any
// key missing from that save's payload.
func TestWriteEnvValuesPreservesUnrelatedCustomValues(t *testing.T) {
root := findRepoRoot(t)
tmpl, err := os.ReadFile(filepath.Join(root, ".env.example"))
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, ".env.example"), tmpl, 0o644); err != nil {
t.Fatal(err)
}
m := NewManager(dir)
// First save sets the admin password, as a one-time setup step would.
if err := m.WriteEnvValues(map[string]string{
"TELESRV_ADMIN_UI_PASSWORD": "s3cret",
}); err != nil {
t.Fatal(err)
}
// A later, unrelated save (e.g. the Storage page) that never mentions
// the password must not disturb it.
if err := m.WriteEnvValues(map[string]string{
"TELESRV_STORAGE_MAX_TOTAL_BYTES": "209715200",
}); err != nil {
t.Fatal(err)
}
groups, err := m.ReadEnvGroups()
if err != nil {
t.Fatal(err)
}
values := map[string]string{}
for _, g := range groups {
for _, f := range g.Fields {
values[f.Key] = f.Value
}
}
if values["TELESRV_ADMIN_UI_PASSWORD"] != "s3cret" {
t.Errorf("TELESRV_ADMIN_UI_PASSWORD = %q, want it preserved as \"s3cret\" after an unrelated save", values["TELESRV_ADMIN_UI_PASSWORD"])
}
if values["TELESRV_STORAGE_MAX_TOTAL_BYTES"] != "209715200" {
t.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES = %q, want 209715200", values["TELESRV_STORAGE_MAX_TOTAL_BYTES"])
}
}

37
internal/procctl/procctl.go
View file

@ -637,15 +637,25 @@ func (m *Manager) readEnvFile() (map[string]string, error) {
// WriteEnvValues rewrites .env from .env.example's exact text, substituting
// each known key's value in place -- see save_env()'s docstring in
// server-panel.py for why this (not a fresh key=value dump) is what
// preserves comments/layout. Only keys present in values are touched; a
// template-commented optional field is uncommented when given a non-empty
// value and left as-is when given an empty one.
// preserves comments/layout. Only keys present in values are set to a new
// value; every other key keeps whatever is already in the current .env
// (falling back to the template's own default only for a key .env never
// set) -- previously this fell straight back to the template default for
// any key not in this particular save's payload, silently wiping out every
// other customized setting (e.g. TELESRV_ADMIN_UI_PASSWORD) on every save
// that only touches one section's keys. A template-commented optional field
// is uncommented when given a non-empty value and left as-is when given an
// empty one.
func (m *Manager) WriteEnvValues(values map[string]string) error {
tmplPath := filepath.Join(m.Root, ".env.example")
tmplData, err := os.ReadFile(tmplPath)
if err != nil {
return fmt.Errorf("read .env.example: %w", err)
}
existing, err := m.readEnvFile()
if err != nil {
return err
}
lines := strings.Split(string(tmplData), "\n")
// Split() on a trailing "\n" leaves one empty trailing element; drop it
// so the join below doesn't add a spurious blank line before the final
@ -657,15 +667,20 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
seen := map[string]bool{}
for _, raw := range lines {
line := strings.TrimSpace(raw)
if a := activeFieldRe.FindStringSubmatch(line); a != nil {
if v, ok := values[a[1]]; ok && !seen[a[1]] {
if a := activeFieldRe.FindStringSubmatch(line); a != nil && !seen[a[1]] {
if v, ok := values[a[1]]; ok {
seen[a[1]] = true
out = append(out, a[1]+"="+v)
continue
}
if v, ok := existing[a[1]]; ok {
seen[a[1]] = true
out = append(out, a[1]+"="+v)
continue
}
}
if c := commentedFieldRe.FindStringSubmatch(line); c != nil {
if v, ok := values[c[1]]; ok && !seen[c[1]] {
if c := commentedFieldRe.FindStringSubmatch(line); c != nil && !seen[c[1]] {
if v, ok := values[c[1]]; ok {
seen[c[1]] = true
if v != "" {
out = append(out, c[1]+"="+v)
@ -674,6 +689,14 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
}
continue
}
// A previously-enabled optional field shows up in the current
// .env as an active line even though the template still has it
// commented out -- keep it enabled with its existing value.
if v, ok := existing[c[1]]; ok {
seen[c[1]] = true
out = append(out, c[1]+"="+v)
continue
}
}
out = append(out, raw)
}

9
internal/rpc/errors.go
View file

@ -107,6 +107,15 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
// fileTooBigErr surfaces TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: the upload's
// total assembled size (not any single part) exceeds the configured
// per-file ceiling. FILE_TOO_BIG is not an official Telegram desktop-RPC
// error string, but it's already this codebase's own convention for the
// same concept on the Bot API surface (see internal/botapi/server.go) --
// reused here for consistency rather than inventing a second name for the
// same condition.
func fileTooBigErr() error { return tgerr.New(400, "FILE_TOO_BIG") }
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
// the client to retry shortly would be misleading.

2
internal/rpc/send_media.go
View file

@ -1152,6 +1152,8 @@ func mediaUploadErr(err error) error {
switch {
case errors.Is(err, domain.ErrFilePartsInvalid):
return filePartsInvalidErr()
case errors.Is(err, domain.ErrFileTooLarge):
return fileTooBigErr()
case errors.Is(err, domain.ErrPhotoInvalid):
return photoInvalidErr()
case errors.Is(err, domain.ErrDocumentInvalid):

2
internal/rpc/upload.go
View file

@ -348,6 +348,8 @@ func fileSaveErr(err error) error {
return filePartsInvalidErr()
case errors.Is(err, domain.ErrFilePartTooBig):
return filePartTooBigErr()
case errors.Is(err, domain.ErrFileTooLarge):
return fileTooBigErr()
case errors.Is(err, domain.ErrUploadQuotaExceeded):
return floodWaitErr(60)
case errors.Is(err, domain.ErrStorageFull):

105
internal/store/postgres/media_hard_retention_integration_test.go Normal file
View file

@ -0,0 +1,105 @@
package postgres
import (
"context"
"strconv"
"testing"
"time"
"telesrv/internal/domain"
)
// TestHardRetentionPurgesBlobBytesButKeepsMetadataRow exercises the "hard"
// storage retention mode's store methods end-to-end against a real
// Postgres: a document old enough (by created_at, not orphaned_at) is a
// candidate for ListDocumentIDsForHardRetentionOlderThan REGARDLESS of
// still having a live media_references row, DeleteFileBlobsForDocument
// removes only its file_blobs row (never the documents row itself), and a
// second sweep pass no longer finds it a candidate since it no longer owns
// any file_blobs row. This is the correctness property the whole "hard"
// mode design hinges on: a message must still be able to render its media
// placeholder after the bytes are gone.
func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
docID := time.Now().UnixNano()
locationKey := "doc:" + strconv.FormatInt(docID, 10)
if err := media.PutDocument(ctx, domain.Document{
ID: docID,
MimeType: "application/octet-stream",
Size: 1024,
}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
})
// Backdate created_at well past any plausible test cutoff -- PutDocument
// always stamps now() and has no created_at parameter.
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days' WHERE id = $1", docID); err != nil {
t.Fatalf("backdate document: %v", err)
}
blob := postgresTestBlob(locationKey, "hard-retention-doc", 1024, "application/octet-stream")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
// A LIVE reference (as if a message still embeds this document) must not
// exempt it from "hard" mode -- that's the entire point of the mode,
// unlike the orphan-only sweep.
if err := addMediaReferencesTx(ctx, pool, &domain.MessageMedia{
Kind: domain.MessageMediaKindDocument,
Document: &domain.Document{ID: docID},
}, domain.MediaRefKindMessageBox, "hard-retention-test:1"); err != nil {
t.Fatalf("add media reference: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE ref_key = 'hard-retention-test:1'")
})
cutoff := time.Now().Add(-24 * time.Hour)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
}
if !containsInt64(ids, docID) {
t.Fatalf("hard retention candidates = %v, want to include still-referenced but old document %d", ids, docID)
}
blobs, err := media.DeleteFileBlobsForDocument(ctx, docID)
if err != nil {
t.Fatalf("DeleteFileBlobsForDocument: %v", err)
}
if len(blobs) != 1 || blobs[0].LocationKey != locationKey {
t.Fatalf("deleted blobs = %+v, want exactly one for %q", blobs, locationKey)
}
// The documents row itself must survive -- a message referencing it
// still needs to render its placeholder (mime type, size, filename).
if _, found, err := media.GetDocument(ctx, docID); err != nil || !found {
t.Fatalf("document metadata row missing after hard blob purge: found=%v err=%v", found, err)
}
// The file_blobs row is gone: a subsequent download attempt resolves via
// GetFileBlob (files.Service.GetFile's path) to not-found, which the rpc
// layer already maps to LOCATION_INVALID.
if _, found, err := media.GetFileBlob(ctx, locationKey); err != nil || found {
t.Fatalf("file_blobs row still present after hard purge: found=%v err=%v", found, err)
}
// Idempotent / self-terminating: a second sweep pass no longer selects
// this document, since it no longer owns any file_blobs row.
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
}
if containsInt64(ids, docID) {
t.Fatalf("hard retention re-selected already-purged document %d", docID)
}
}

95
internal/store/postgres/media_refs.go
View file

@ -197,6 +197,101 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
return blobs, nil
}
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
// cutoff (by upload/created_at) that still own at least one file_blobs row,
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
// entirely: a document still referenced by a live message is exactly as
// eligible as an orphaned one.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
// ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
// (main body + thumbnail variants), returning what was deleted so the caller
// can physically remove each object from its backend once confirming (via
// CountFileBlobRefs, after this call) no other row still needs it. Unlike
// DeleteDocumentAndBlobs, this deliberately does NOT delete the documents
// row itself -- "hard" retention mode keeps the metadata (dimensions, mime
// type, filename) so a message can still render "this document is no longer
// available" instead of disappearing outright. A subsequent
// upload.getFile/GetFileBlob lookup for a location key this call removed
// correctly finds nothing and reports not-found.
func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete document blob bytes (hard retention)", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("doc:%d", id),
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list document blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}
// DeleteFileBlobsForPhoto is the photo counterpart of
// DeleteFileBlobsForDocument -- see its doc comment. Deliberately does not
// delete the photos row.
func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete photo blob bytes (hard retention)", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("photo:%d", id),
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list photo blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
// (one per rendition size), returning what was deleted so the caller can
// physically remove each object from its backend once confirming (via

31
internal/store/postgres/queries/media.sql
View file

@ -237,6 +237,37 @@ WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
ORDER BY orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListDocumentIDsForHardRetentionOlderThan :many
-- "Hard" retention mode: candidates are documents older than cutoff (by
-- upload/created_at, NOT orphaned_at -- a live reference does not exempt
-- them) that still own at least one file_blobs row. The EXISTS check is what
-- keeps this sweep from re-selecting the same document forever: once its
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
-- rows but deliberately leaves this documents row in place), it naturally
-- drops out of this query on the next pass.
SELECT d.id FROM documents d
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListPhotoIDsForHardRetentionOlderThan :many
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
-- candidate selection, for photos.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;

83
internal/store/postgres/sqlcgen/media.sql.go
View file

@ -825,6 +825,50 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
return items, nil
}
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
SELECT d.id FROM documents d
WHERE d.created_at < $1::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT $2::int
`
type ListDocumentIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// "Hard" retention mode: candidates are documents older than cutoff (by
// upload/created_at, NOT orphaned_at -- a live reference does not exempt
// them) that still own at least one file_blobs row. The EXISTS check is what
// keeps this sweep from re-selecting the same document forever: once its
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
// rows but deliberately leaves this documents row in place), it naturally
// drops out of this query on the next pass.
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
SELECT location_key, backend, object_key, size
FROM file_blobs
@ -937,6 +981,45 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
return items, nil
}
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT $2::int
`
type ListPhotoIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
// candidate selection, for photos.
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listProfilePhotos = `-- name: ListProfilePhotos :many
SELECT photo_id
FROM profile_photos