adding more functions to media managament system

This commit is contained in:
onysd 2026-09-03 00:54:09 +03:00
parent 95e62c2d77
commit 70c0ba44f0
30 changed files with 1494 additions and 104 deletions

View file

@ -292,14 +292,33 @@ TELESRV_S3_USE_SSL=false
# MinIO needs this on (bucket in the URL path); AWS S3 does not. # MinIO needs this on (bucket in the URL path); AWS S3 does not.
TELESRV_S3_PATH_STYLE=true TELESRV_S3_PATH_STYLE=true
# Reject new uploads once storage is nearly full, instead of letting the disk # Reject new uploads once storage is nearly full, instead of letting the disk
# fill up. Thresholds live in the Advanced section below. # fill up. Thresholds are the three fields right below.
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
# Automatically delete old media once it's no longer referenced by any # localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
# message, profile photo, or sticker set (never deletes media still visible TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
# in a conversation). Off by default -- storage usage is tracked and shown # Reject new uploads once total tracked blob bytes would exceed this. The only
# in the admin panel either way; enable this once you're comfortable with # meaningful "low space" signal on the s3 backend (no OS free-space concept);
# what it will reclaim. Retention age lives in the Advanced section below. # optional soft cap on localfs too. <=0 disables.
TELESRV_STORAGE_RETENTION_ENABLE=false TELESRV_STORAGE_MAX_TOTAL_BYTES=0
# Reject a single upload once its total assembled size (sum of all its parts)
# would exceed this. <=0 disables this check (the protocol's own part-count
# ceiling of ~4GB still applies). Must not exceed that ceiling.
TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=0
# Storage retention sweep mode: "off" (default, nothing auto-deleted --
# storage usage is still tracked and shown in the admin panel either way),
# "orphan" (safe: deletes a document/photo's blob only once it's no longer
# referenced by any message/profile photo/sticker set), or "hard"
# (aggressive: deletes a document/photo's blob once it's old enough,
# REGARDLESS of whether it's still referenced -- old media in active
# conversations will show as unavailable).
TELESRV_STORAGE_RETENTION_MODE=off
# How long a document/photo must have had zero references before the sweep
# deletes it in "orphan" mode above -- or how old the media itself is before
# "hard" mode deletes its bytes regardless of references. Ignored when the
# mode above is "off". The sweep itself runs alongside every other retention
# check on the shared TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH
# cadence (Advanced section below).
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
# ============================================================================== # ==============================================================================
@ -438,22 +457,9 @@ TELESRV_GIF_SEED_DIR=data/gifs
# read fresh on every request -- editing them takes effect with no restart. # read fresh on every request -- editing them takes effect with no restart.
TELESRV_IDENTITY_DIR=data/identity TELESRV_IDENTITY_DIR=data/identity
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above). # How often the cached free-space/usage gauge behind the low-space guard
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables. # (thresholds now live in the Storage & Media section above) refreshes.
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
# Reject new uploads once total tracked blob bytes would exceed this. The only
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
# optional soft cap on localfs too. <=0 disables.
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
# How often the cached free-space/usage gauge behind the guard above refreshes.
TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m
# Storage retention sweep tuning (master toggle is TELESRV_STORAGE_RETENTION_ENABLE above).
# How long a document/photo must have had zero references before the sweep
# deletes it -- not how old the media itself is, and it never touches media
# still referenced by a live message/profile-photo/sticker-set. The sweep
# itself runs alongside every other retention check on the shared
# TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH cadence above.
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
# New-account perks: free Telegram Premium months. # New-account perks: free Telegram Premium months.
TELESRV_PREMIUM_GRANT_MONTHS=3 TELESRV_PREMIUM_GRANT_MONTHS=3

View file

@ -2612,11 +2612,25 @@ type AccountStorageRow struct {
FileCount int64 `json:"FileCount,string"` FileCount int64 `json:"FileCount,string"`
} }
// ListAccountStorageUsage pages the per-account storage breakdown, largest // storageUsageSortColumns whitelists the columns ListAccountStorageUsage's
// user first. Offset-based (not keyset): storage administration on a // sortBy may map to -- never interpolate the caller's sort key directly into
// self-hosted deployment doesn't need to support arbitrarily deep pages // SQL, since unlike a plain value it can't go through a query parameter.
// efficiently the way an infinite-scroll feed does. var storageUsageSortColumns = map[string]string{
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) { "bytes": "t.bytes",
"files": "t.file_count",
"user_id": "t.owner_user_id",
"username": "lower(COALESCE(u.username, ''))",
"first_name": "lower(COALESCE(u.first_name, ''))",
}
// ListAccountStorageUsage pages the per-account storage breakdown. sortBy
// selects one of storageUsageSortColumns (falls back to "bytes" for an
// unknown/empty key); sortDesc reverses it. q, if non-empty, filters to
// accounts whose id/username/first name match it. Offset-based (not
// keyset): storage administration on a self-hosted deployment doesn't need
// to support arbitrarily deep pages efficiently the way an infinite-scroll
// feed does.
func (s *readStore) ListAccountStorageUsage(ctx context.Context, q string, sortBy string, sortDesc bool, offset, limit int) ([]AccountStorageRow, bool, error) {
if limit <= 0 { if limit <= 0 {
limit = storageUsageListDefaultLimit limit = storageUsageListDefaultLimit
} }
@ -2626,6 +2640,32 @@ func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit i
if offset < 0 { if offset < 0 {
offset = 0 offset = 0
} }
column, ok := storageUsageSortColumns[sortBy]
if !ok {
column = storageUsageSortColumns["bytes"]
}
direction := "ASC"
if sortDesc {
direction = "DESC"
}
q = strings.TrimSpace(q)
var whereClause string
args := []any{}
argN := 1
if q != "" {
id := int64(-1)
if n, err := strconv.ParseInt(q, 10, 64); err == nil {
id = n
}
whereClause = fmt.Sprintf(`WHERE t.owner_user_id = $%d
OR lower(COALESCE(u.username, '')) LIKE $%d
OR lower(COALESCE(u.first_name, '')) LIKE $%d`, argN, argN+1, argN+2)
args = append(args, id, "%"+strings.ToLower(q)+"%", "%"+strings.ToLower(q)+"%")
argN += 3
}
offsetArg := argN
limitArg := argN + 1
args = append(args, offset, limit+1)
rows, err := s.pool.Query(ctx, ` rows, err := s.pool.Query(ctx, `
WITH totals AS ( WITH totals AS (
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
@ -2636,9 +2676,10 @@ WITH totals AS (
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
FROM totals t FROM totals t
LEFT JOIN users u ON u.id = t.owner_user_id LEFT JOIN users u ON u.id = t.owner_user_id
ORDER BY t.bytes DESC, t.owner_user_id `+whereClause+`
OFFSET $1 ORDER BY `+column+` `+direction+`, t.owner_user_id
LIMIT $2`, offset, limit+1) OFFSET $`+strconv.Itoa(offsetArg)+`
LIMIT $`+strconv.Itoa(limitArg), args...)
if err != nil { if err != nil {
return nil, false, fmt.Errorf("list account storage usage: %w", err) return nil, false, fmt.Errorf("list account storage usage: %w", err)
} }

View file

@ -2498,7 +2498,14 @@ func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request
writeAPIError(w, http.StatusBadRequest, "invalid limit") writeAPIError(w, http.StatusBadRequest, "invalid limit")
return return
} }
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit) sortDesc := query.Get("order") != "asc"
rows, hasMore, err := s.read.ListAccountStorageUsage(
r.Context(),
query.Get("q"),
query.Get("sort"),
sortDesc,
offset,
limit)
if err != nil { if err != nil {
writeAPIError(w, http.StatusInternalServerError, err.Error()) writeAPIError(w, http.StatusInternalServerError, err.Error())
return return

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View file

@ -23,8 +23,8 @@
})(); })();
</script> </script>
<script type="module" crossorigin src="/assets/index-CP20QCwX.js"></script> <script type="module" crossorigin src="/assets/index-IZM8WmmY.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-Wtwg__bv.css"> <link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
</head> </head>
<body> <body>
<div id="root"></div> <div id="root"></div>

View file

@ -1,18 +1,52 @@
import { ChevronDown, Loader2, RefreshCw } from "lucide-react"; import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search } from "lucide-react";
import { useEffect, useState } from "react"; import { useEffect, useMemo, useState } from "react";
import { api, errorMessage } from "../api"; import { api, errorMessage } from "../api";
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui"; import { ActionButton } from "../components/ActionButton";
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
import { displayUsername, formatBytes, formatQuantity } from "../lib/format"; import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
import type { Navigate } from "../routing"; import type { Navigate } from "../routing";
import type { AccountStorageRow, StorageStatsResponse } from "../types"; import type { AccountStorageRow, StorageStatsResponse } from "../types";
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) { type StorageSortKey = "user_id" | "username" | "bytes" | "files";
// SortableHeader is a <th> that toggles ascending/descending on click and
// shows which column (and direction) is currently active -- there's no
// existing sortable-table convention elsewhere in this admin panel to
// mirror, so this is a small, self-contained one for this page.
function SortableHeader({
label,
sortKey,
activeKey,
desc,
onSort
}: {
label: string;
sortKey: StorageSortKey;
activeKey: StorageSortKey;
desc: boolean;
onSort: (key: StorageSortKey) => void;
}) {
const active = sortKey === activeKey;
return (
<th>
<button type="button" className="sort-header" onClick={() => onSort(sortKey)}>
{label}
{active ? (desc ? <ArrowDown size={13} /> : <ArrowUp size={13} />) : <ArrowUpDown size={13} className="sort-header-idle" />}
</button>
</th>
);
}
function StorageOverviewTab() {
const [stats, setStats] = useState<StorageStatsResponse | null>(null); const [stats, setStats] = useState<StorageStatsResponse | null>(null);
const [rows, setRows] = useState<AccountStorageRow[]>([]); const [rows, setRows] = useState<AccountStorageRow[]>([]);
const [hasMore, setHasMore] = useState(false); const [hasMore, setHasMore] = useState(false);
const [offset, setOffset] = useState(0); const [offset, setOffset] = useState(0);
const [busy, setBusy] = useState(false); const [busy, setBusy] = useState(false);
const [error, setError] = useState(""); const [error, setError] = useState("");
const [q, setQ] = useState("");
const [sortKey, setSortKey] = useState<StorageSortKey>("bytes");
const [sortDesc, setSortDesc] = useState(true);
async function loadStats() { async function loadStats() {
try { try {
@ -26,7 +60,13 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
setBusy(true); setBusy(true);
setError(""); setError("");
const at = next ? offset : 0; const at = next ? offset : 0;
const params = new URLSearchParams({ limit: "50", offset: String(at) }); const params = new URLSearchParams({
limit: "50",
offset: String(at),
sort: sortKey,
order: sortDesc ? "desc" : "asc"
});
if (q.trim()) params.set("q", q.trim());
try { try {
const result = await api.storageAccounts(params); const result = await api.storageAccounts(params);
const page = result.rows ?? []; const page = result.rows ?? [];
@ -48,7 +88,16 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
useEffect(() => { useEffect(() => {
refresh(); refresh();
// eslint-disable-next-line react-hooks/exhaustive-deps // eslint-disable-next-line react-hooks/exhaustive-deps
}, []); }, [sortKey, sortDesc]);
function toggleSort(key: StorageSortKey) {
if (key === sortKey) {
setSortDesc((current) => !current);
} else {
setSortKey(key);
setSortDesc(true);
}
}
// Physical is what actually consumes disk/S3 (deduplicated); logical is the // Physical is what actually consumes disk/S3 (deduplicated); logical is the
// sum of what the per-account table below adds up to. They legitimately // sum of what the per-account table below adds up to. They legitimately
@ -56,15 +105,7 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0; const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
return ( return (
<PageFrame <>
title={"Storage"}
eyebrow={"Media / Storage usage"}
actions={
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
</button>
}
>
{error && <Alert>{error}</Alert>} {error && <Alert>{error}</Alert>}
<div className="metric-row"> <div className="metric-row">
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} /> <Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
@ -83,14 +124,29 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
/> />
</div> </div>
<QueryPanel>
<form className="toolbar" onSubmit={(event) => { event.preventDefault(); void loadAccounts(false); }}>
<label className="searchbox">
<Search size={15} />
<input value={q} onChange={(event) => setQ(event.target.value)} placeholder={"User ID / username / name"} />
</label>
<button className="btn primary icon-text" type="submit" disabled={busy}>
{busy ? <Loader2 size={15} className="spin" /> : <Search size={15} />} {"Search"}
</button>
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
</button>
</form>
</QueryPanel>
<div className="table-wrap"> <div className="table-wrap">
<table className="data-table"> <table className="data-table">
<thead> <thead>
<tr> <tr>
<th>{"User ID"}</th> <SortableHeader label={"User ID"} sortKey="user_id" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<th>{"Account"}</th> <SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<th>{"Storage used"}</th> <SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
<th>{"Files"}</th> <SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
@ -113,6 +169,330 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
</button> </button>
</div> </div>
)} )}
</>
);
}
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
const [tab, setTab] = useState<"overview" | "limits">("overview");
return (
<PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}>
<div className="tab-bar" role="tablist" aria-label={"Storage sections"}>
<button className={`tab-btn ${tab === "overview" ? "active" : ""}`} type="button" role="tab" aria-selected={tab === "overview"} onClick={() => setTab("overview")}>
{"Overview"}
</button>
<button className={`tab-btn ${tab === "limits" ? "active" : ""}`} type="button" role="tab" aria-selected={tab === "limits"} onClick={() => setTab("limits")}>
{"Limits & Retention"}
</button>
</div>
{tab === "overview" ? <StorageOverviewTab /> : <LimitsRetentionSection />}
</PageFrame> </PageFrame>
); );
} }
// --- Limits & Retention --------------------------------------------------
//
// Edits a handful of TELESRV_STORAGE_* keys through the existing generic
// .env editor endpoints (GET /api/server/env, POST
// /api/actions/update-server-env -- see cmd/telesrv-admin/serversettings.go
// and ServerSettingsPage.tsx's EnvSection, which this mirrors for its
// save/reload flow), but with friendly units instead of raw key=value text:
// GB inputs for byte budgets, a mode dropdown + day count for retention.
const STORAGE_ENV_KEYS = {
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE"
} as const;
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
// can never legally exceed (internal/config validates this server-side too;
// this is just an early, friendlier warning in the form).
const PROTOCOL_UPLOAD_CEILING_BYTES = 524288 * 8000;
const BYTE_UNITS: { label: string; bytes: number }[] = [
{ label: "MB", bytes: 1024 ** 2 },
{ label: "GB", bytes: 1024 ** 3 },
{ label: "TB", bytes: 1024 ** 4 }
];
function bestByteUnit(bytes: number): { label: string; bytes: number } {
for (let i = BYTE_UNITS.length - 1; i >= 0; i--) {
if (bytes >= BYTE_UNITS[i].bytes) return BYTE_UNITS[i];
}
return BYTE_UNITS[1]; // default to GB for small/zero values
}
// parseDurationMinutes reads a Go-style duration string (e.g. "720h", "90m",
// "1h30m") and returns the total as minutes. Unrecognized/empty input is 0.
function parseDurationMinutes(value: string): number {
let totalSeconds = 0;
const re = /(\d+(?:\.\d+)?)\s*(h|m|s)/g;
let match: RegExpExecArray | null;
while ((match = re.exec(value)) !== null) {
const amount = parseFloat(match[1]);
const unit = match[2];
totalSeconds += unit === "h" ? amount * 3600 : unit === "m" ? amount * 60 : amount;
}
return totalSeconds / 60;
}
const DURATION_UNITS: { label: string; minutes: number }[] = [
{ label: "Minutes", minutes: 1 },
{ label: "Hours", minutes: 60 },
{ label: "Days", minutes: 1440 }
];
function bestDurationUnit(minutes: number): { label: string; minutes: number } {
for (let i = DURATION_UNITS.length - 1; i >= 0; i--) {
if (minutes >= DURATION_UNITS[i].minutes) return DURATION_UNITS[i];
}
return DURATION_UNITS[0];
}
// DurationField edits one duration env value (stored as total minutes) as a
// friendly "amount + unit" pair, letting TTL be set in minutes, hours, or
// days rather than being locked to whole days.
function DurationField({
label,
help,
minutes,
disabled,
onChange
}: {
label: string;
help: string;
minutes: string;
disabled?: boolean;
onChange: (minutes: string) => void;
}) {
const totalMinutes = Number(minutes || "0");
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
const amount = totalMinutes > 0 ? totalMinutes / unit.minutes : NaN;
function handleAmountChange(raw: string) {
const parsed = Number(raw);
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
onChange("0");
return;
}
onChange(String(Math.max(1, Math.round(parsed * unit.minutes))));
}
return (
<label className="duration-field">
<span>{label}</span>
<div style={{ display: "flex", gap: 8 }}>
<input
type="number"
min="0"
step="any"
value={Number.isNaN(amount) ? "" : amount}
disabled={disabled}
onChange={(event) => handleAmountChange(event.target.value)}
/>
<select value={unitLabel} disabled={disabled} onChange={(event) => setUnitLabel(event.target.value)} style={{ maxWidth: 100 }}>
{DURATION_UNITS.map((u) => (
<option key={u.label} value={u.label}>{u.label}</option>
))}
</select>
</div>
<span className="env-field-desc">{help}</span>
</label>
);
}
// ByteSizeField edits one byte-count env value as a friendly
// "amount + unit" pair. bytes is the raw byte count as a string (what the
// backend stores); an empty/zero value displays as "Unlimited".
function ByteSizeField({
label,
help,
bytes,
onChange
}: {
label: string;
help: string;
bytes: string;
onChange: (bytes: string) => void;
}) {
const numericBytes = Number(bytes || "0");
const [unitLabel, setUnitLabel] = useState(() => bestByteUnit(numericBytes).label);
const unit = BYTE_UNITS.find((u) => u.label === unitLabel) ?? BYTE_UNITS[1];
const amount = numericBytes > 0 ? numericBytes / unit.bytes : NaN;
function handleAmountChange(raw: string) {
const parsed = Number(raw);
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
onChange("0");
return;
}
onChange(String(Math.round(parsed * unit.bytes)));
}
return (
<label className="duration-field">
<span>{label}</span>
<div style={{ display: "flex", gap: 8 }}>
<input
type="number"
min="0"
step="any"
value={Number.isNaN(amount) ? "" : amount}
placeholder={"Unlimited"}
onChange={(event) => handleAmountChange(event.target.value)}
/>
<select value={unitLabel} onChange={(event) => setUnitLabel(event.target.value)} style={{ maxWidth: 90 }}>
{BYTE_UNITS.map((u) => (
<option key={u.label} value={u.label}>{u.label}</option>
))}
</select>
</div>
<span className="env-field-desc">{help}</span>
</label>
);
}
function LimitsRetentionSection() {
const [loaded, setLoaded] = useState(false);
const [error, setError] = useState("");
const [initial, setInitial] = useState<Record<string, string>>({});
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
const [minFreeBytes, setMinFreeBytes] = useState("0");
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
const [retentionMode, setRetentionMode] = useState("off");
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
async function load() {
setError("");
try {
const groups = await api.serverEnv();
const values: Record<string, string> = {};
for (const group of groups) {
for (const field of group.fields) {
values[field.key] = field.value || field.default_value || "";
}
}
setInitial(values);
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
const mode = (values[STORAGE_ENV_KEYS.retentionMode] || "off").trim().toLowerCase();
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
setLoaded(true);
} catch (err) {
setError(errorMessage(err));
}
}
useEffect(() => { void load(); }, []);
// Only the age is re-encoded from its friendly "days" input back to a Go
// duration string; mode/off keeps whatever TELESRV_STORAGE_RETENTION_MAX_AGE
// already was rather than clobbering it with a throwaway placeholder.
const pendingValues = useMemo(() => {
const next: Record<string, string> = {
[STORAGE_ENV_KEYS.maxTotal]: maxTotalBytes || "0",
[STORAGE_ENV_KEYS.minFree]: minFreeBytes || "0",
[STORAGE_ENV_KEYS.maxUploadFile]: maxUploadFileBytes || "0",
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
: `${Math.max(1, Math.round(Number(retentionAgeMinutes || "0")))}m`
};
const changed: Record<string, string> = {};
for (const [key, value] of Object.entries(next)) {
if ((initial[key] ?? "") !== value) changed[key] = value;
}
return changed;
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, initial]);
const hasChanges = Object.keys(pendingValues).length > 0;
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
return (
<section className="section-block">
<SectionHead
title={"Limits & Retention"}
text={"Server-wide storage budget, per-file upload cap, and automatic media cleanup. Saved to .env -- takes effect on the next Restart/Update."}
/>
{error && <Alert>{error}</Alert>}
{!loaded ? (
<p style={{ color: "var(--muted)" }}>{"Loading current settings..."}</p>
) : (
<div className="card-body">
<div className="attr-block">
<ByteSizeField
label={"Max total storage budget"}
help={"Reject new uploads once total tracked blob bytes would exceed this. Empty/0 = unlimited."}
bytes={maxTotalBytes}
onChange={setMaxTotalBytes}
/>
<ByteSizeField
label={"Min free space guard"}
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
bytes={minFreeBytes}
onChange={setMinFreeBytes}
/>
<ByteSizeField
label={"Max single file size"}
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
bytes={maxUploadFileBytes}
onChange={setMaxUploadFileBytes}
/>
{uploadCeilingExceeded && (
<Alert>{"This exceeds the protocol's own ~4GB upload ceiling and will be refused when the server restarts."}</Alert>
)}
</div>
<div className="attr-block" style={{ marginTop: "1.5em" }}>
<label className="duration-field">
<span>{"Retention mode"}</span>
<select value={retentionMode} onChange={(event) => setRetentionMode(event.target.value)}>
<option value="off">{"Off"}</option>
<option value="orphan">{"Delete once no longer used (safe)"}</option>
<option value="hard">{"Delete after a fixed time, even if still in use"}</option>
</select>
</label>
<p className="env-field-desc">
{retentionMode === "off" &&
"No storage sweep runs; nothing is auto-deleted. Storage usage is still tracked and shown above either way."}
{retentionMode === "orphan" &&
"Safe: a document or photo's file is deleted only once it is no longer referenced by any message, profile photo, or sticker set. Media still visible in a conversation is never touched, regardless of age."}
{retentionMode === "hard" &&
"Irreversible and aggressive: a document or photo's file bytes are deleted once old enough, REGARDLESS of whether a message still references it. Old media in active conversations will start showing as unavailable once purged -- only the file is removed, the message itself keeps rendering its placeholder (name, size, thumbnail)."}
</p>
<DurationField
label={"Retention age"}
help={
retentionMode === "hard"
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
: "How long a document or photo must have had zero references before its file is deleted."
}
minutes={retentionAgeMinutes}
disabled={retentionMode === "off"}
onChange={setRetentionAgeMinutes}
/>
</div>
<div className="gift-table-actions env-save-row">
<ActionButton
tone="warn"
label={"Save limits & retention settings"}
path="/api/actions/update-server-env"
payload={() => ({ values: pendingValues })}
disabled={!hasChanges || uploadCeilingExceeded}
onDone={() => void load()}
/>
</div>
</div>
)}
</section>
);
}

View file

@ -779,6 +779,27 @@ textarea:focus {
font-weight: 800; font-weight: 800;
} }
.sort-header {
display: inline-flex;
align-items: center;
gap: 5px;
padding: 0;
border: 0;
background: transparent;
color: inherit;
font: inherit;
font-weight: 800;
cursor: pointer;
}
.sort-header:hover {
color: var(--text);
}
.sort-header-idle {
opacity: 0.45;
}
.data-table tbody tr:hover { .data-table tbody tr:hover {
background: var(--panel-subtle); background: var(--panel-subtle);
} }

View file

@ -932,6 +932,7 @@ func run(logger *zap.Logger) error {
filesapp.WithUploadPartBackend(localBlobFS), filesapp.WithUploadPartBackend(localBlobFS),
filesapp.WithAdditionalBlobBackend(additionalBlobBackend), filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
filesapp.WithSpaceGuard(spaceGuard), filesapp.WithSpaceGuard(spaceGuard),
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir), filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
externalMediaOption(cfg), externalMediaOption(cfg),
webPagePreviewOption(cfg), webPagePreviewOption(cfg),
@ -1063,11 +1064,7 @@ func run(logger *zap.Logger) error {
readModelVersionStore, readModelVersionStore,
cfg.UserProjectionFactCacheMaxEntries, cfg.UserProjectionFactCacheMaxEntries,
) )
storageRetentionMaxAge := cfg.StorageRetentionMaxAge retentionWorker := maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
if !cfg.StorageRetentionEnable {
storageRetentionMaxAge = 0
}
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
cfg.UpdateEventRetention, cfg.UpdateEventRetention,
cfg.RetentionInterval, cfg.RetentionInterval,
cfg.RetentionBatch, cfg.RetentionBatch,
@ -1080,9 +1077,17 @@ func run(logger *zap.Logger) error {
WithModerationRetention(moderationReportStore). WithModerationRetention(moderationReportStore).
WithUserUpdateRetention(updateEventStore). WithUserUpdateRetention(updateEventStore).
WithChannelUpdateRetention(channelStore). WithChannelUpdateRetention(channelStore).
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention). WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention)
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge). // TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
Run(ctx) // the orphan-only (safe) and hard (age-based, ignores live references)
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
switch cfg.StorageRetentionMode {
case config.StorageRetentionModeOrphan:
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, cfg.StorageRetentionMaxAge)
case config.StorageRetentionModeHard:
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, cfg.StorageRetentionMaxAge)
}
go retentionWorker.Run(ctx)
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"), go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
cfg.UploadPartTTL, cfg.UploadPartTTL,
cfg.UploadPartGCInterval, cfg.UploadPartGCInterval,

View file

@ -87,8 +87,11 @@ func (c *stickerSetNegativeCache) delete(refs ...domain.StickerSetRef) {
// 每个 chunk 一次 GetFileBlob 的 PG 往返(一个文件按 ≤512KB/1MB 分多次 getFile热门贴纸/ // 每个 chunk 一次 GetFileBlob 的 PG 往返(一个文件按 ≤512KB/1MB 分多次 getFile热门贴纸/
// reaction/头像更被大量用户重复拉)。 // reaction/头像更被大量用户重复拉)。
// //
// FileBlob 元数据小约百字节且内容不可变location_key 一旦写入即固定指向同一 object_key // FileBlob 元数据小(约百字节),新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,
// 新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,故只读填充、无需失效。 // 故写入路径只读填充、无需失效。但一个 location_key 的 file_blobs 行可以被存储 retention
// 清理主动删除(尤其是 hard 模式:不等 orphaned仍被引用的媒体到期也会被清理字节这种情况
// 下必须调用 delete 使该 key 失效,否则缓存会继续认为它存在,导致后续下载走到已被删除的字节
// 而不是优雅返回 LOCATION_INVALID。
type blobMetaCache struct { type blobMetaCache struct {
mu sync.Mutex mu sync.Mutex
cap int cap int
@ -141,6 +144,17 @@ func (c *blobMetaCache) put(key string, blob domain.FileBlob) {
} }
} }
// delete 使一个 location_key 的缓存元数据立即失效(如果存在)。见上方类型注释:
// 存储 retention 清理主动删掉该 location_key 的 file_blobs 行后必须调用。
func (c *blobMetaCache) delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[key]; ok {
c.ll.Remove(el)
delete(c.m, key)
}
}
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction / // blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
// 缩略图通常只有几 KB 到几十 KB缓存全量内容可以避开点击历史时的本地磁盘冷读抖动 // 缩略图通常只有几 KB 到几十 KB缓存全量内容可以避开点击历史时的本地磁盘冷读抖动
// 大媒体仍由 BlobBackend.GetRange 分段读取,避免把大文件放进内存。 // 大媒体仍由 BlobBackend.GetRange 分段读取,避免把大文件放进内存。
@ -220,6 +234,21 @@ func (c *blobBytesCache) put(key string, bytes []byte) {
} }
} }
// delete evicts a cached object_key's full bytes (if present) and reclaims
// its budget. Called after a retention sweep physically deletes the object
// from its backend, so a later GetFile can't keep serving bytes that no
// longer exist on disk/S3.
func (c *blobBytesCache) delete(key string) {
c.mu.Lock()
defer c.mu.Unlock()
if el, ok := c.m[key]; ok {
entry := el.Value.(*blobBytesEntry)
c.ll.Remove(el)
delete(c.m, key)
c.used -= entry.size
}
}
type stickerSetFullCache struct { type stickerSetFullCache struct {
mu sync.RWMutex mu sync.RWMutex
byID map[int64]stickerSetFullEntry byID map[int64]stickerSetFullEntry

View file

@ -24,6 +24,17 @@ type mediaRetentionStore interface {
// OrphanDocumentIfUnreferenced is the immediate (no grace period) // OrphanDocumentIfUnreferenced is the immediate (no grace period)
// counterpart to the age-based sweep above -- see its doc comment. // counterpart to the age-based sweep above -- see its doc comment.
OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error) OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)
// -- "hard" retention mode (TELESRV_STORAGE_RETENTION_MODE=hard) --
// Candidate selection ignores media_references entirely: a document/
// photo still referenced by a live message is exactly as eligible as an
// orphaned one once it's old enough. The delete methods physically
// remove only the file_blobs row(s)/bytes, never the document/photo
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
} }
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore: // DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
@ -68,6 +79,63 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
return deleted, nil return deleted, nil
} }
// DeleteBlobBytesForMediaOlderThan implements
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
// documents/photos whose upload/created_at is older than cutoff, physically
// deletes their blob bytes (main body + thumbnail/rendition variants) from
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
// live message/profile-photo/sticker-set still references them. It
// deliberately never touches the documents/photos metadata row itself: a
// message must still be able to render "here was a photo/document"
// (dimensions, mime type, filename) after its bytes are gone, rather than
// the message breaking outright. A subsequent upload.getFile for the same
// location key finds no file_blobs row and returns LOCATION_INVALID, which
// stock clients already render as a "media unavailable" placeholder.
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
store, ok := s.media.(mediaRetentionStore)
if !ok || limit <= 0 {
return 0, nil
}
purged := 0
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
if err != nil {
return purged, fmt.Errorf("list documents for hard retention: %w", err)
}
for _, id := range docIDs {
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
if err != nil {
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
remaining := limit - len(docIDs)
if remaining <= 0 {
return purged, nil
}
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
if err != nil {
return purged, fmt.Errorf("list photos for hard retention: %w", err)
}
for _, id := range photoIDs {
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
if err != nil {
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
continue
}
if len(blobs) == 0 {
continue
}
s.deleteOrphanedBlobs(ctx, store, blobs)
purged++
}
return purged, nil
}
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the // deleteDocumentNowIfUnreferenced is the immediate counterpart to the
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans // age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
// id right now (skipping the grace period) and, only if that succeeds -- // id right now (skipping the grace period) and, only if that succeeds --
@ -106,6 +174,17 @@ func (s *Service) deleteDocumentNowIfUnreferenced(ctx context.Context, id int64)
// delete it from. // delete it from.
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) { func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
for _, b := range blobs { for _, b := range blobs {
// The file_blobs row for this exact location_key is already gone
// (the caller deleted it in the same transaction that produced this
// blob list) -- so any cached "found" metadata for it is now wrong
// regardless of whether the underlying bytes turn out to still be
// shared by another row below. Without this, a hot GetFile path that
// had this location_key's metadata cached would keep trying to read
// bytes that may no longer exist (hard retention mode purges blobs
// for actively-referenced, potentially still-hot media), producing
// an internal error instead of the graceful LOCATION_INVALID a stock
// client knows how to render.
s.blobCache.delete(b.LocationKey)
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey) refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
if err != nil { if err != nil {
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err)) s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
@ -123,5 +202,6 @@ func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionS
if err := backend.Delete(ctx, b.ObjectKey); err != nil { if err := backend.Delete(ctx, b.ObjectKey); err != nil {
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err)) s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
} }
s.byteCache.delete(b.ObjectKey)
} }
} }

View file

@ -72,6 +72,10 @@ type Service struct {
stickerSetNegCache *stickerSetNegativeCache stickerSetNegCache *stickerSetNegativeCache
uploadQuota domain.UploadPartQuota uploadQuota domain.UploadPartQuota
spaceGuard SpaceGuard spaceGuard SpaceGuard
// maxUploadFileBytes caps a single upload's total assembled size (sum of
// all parts). <=0 means unlimited (still bounded by the protocol's own
// MaxUploadPartBytes*MaxUploadParts ceiling). See WithMaxUploadFileBytes.
maxUploadFileBytes int64
mapTiles *mapTileProxy mapTiles *mapTileProxy
externalMedia *externalMediaFetcher externalMedia *externalMediaFetcher
webpage *webpageFetcher webpage *webpageFetcher
@ -143,6 +147,15 @@ func WithSpaceGuard(guard SpaceGuard) Option {
} }
} }
// WithMaxUploadFileBytes caps a single uploaded file's total assembled size
// (sum of all parts, not any one part) -- TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES.
// <=0 leaves it unlimited (still bounded by MaxUploadPartBytes*MaxUploadParts).
func WithMaxUploadFileBytes(maxBytes int64) Option {
return func(s *Service) {
s.maxUploadFileBytes = maxBytes
}
}
// WithAdditionalBlobBackend registers a non-active blob backend purely for // WithAdditionalBlobBackend registers a non-active blob backend purely for
// reading/deleting rows written while it used to be active. Register the // reading/deleting rows written while it used to be active. Register the
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old // previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
@ -753,6 +766,11 @@ func (s *Service) loadAndValidateUploadParts(ctx context.Context, ownerUserID, f
return nil, 0, domain.ErrFilePartsInvalid return nil, 0, domain.ErrFilePartsInvalid
} }
} }
// Checked against the total assembled size, not any single part --
// MaxUploadPartBytes above already bounds each part individually.
if s.maxUploadFileBytes > 0 && total > s.maxUploadFileBytes {
return nil, 0, domain.ErrFileTooLarge
}
return parts, total, nil return parts, total, nil
} }

View file

@ -171,6 +171,89 @@ func TestCreatePhotoFromUploadReceiptReplaysAfterPartCleanup(t *testing.T) {
} }
} }
// TestMaxUploadFileBytesRejectsOversizedAssembledDocument covers
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES enforcement (WithMaxUploadFileBytes):
// the check must fire against the TOTAL assembled size (sum of every part),
// not any single part -- each individual part here is well under the limit,
// only their sum exceeds it.
func TestMaxUploadFileBytesRejectsOversizedAssembledDocument(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
local, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const partSize = 1024
const maxUploadFileBytes = 2 * partSize // exactly 2 parts' worth; 3 parts must be rejected
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
parts := []string{
strings.Repeat("a", partSize),
strings.Repeat("b", partSize),
strings.Repeat("c", partSize),
}
for i, part := range parts {
if _, err := svc.SaveBigFilePart(ctx, 10, 300, i, len(parts), []byte(part)); err != nil {
t.Fatalf("SaveBigFilePart %d: %v", i, err)
}
}
_, err = svc.CreateDocumentFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 300, Parts: len(parts), Name: "big.bin", Big: true},
domain.DocumentSpec{MimeType: "application/octet-stream"},
)
if !errors.Is(err, domain.ErrFileTooLarge) {
t.Fatalf("CreateDocumentFromUpload over max size err = %v, want ErrFileTooLarge", err)
}
}
// TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit ensures the check
// is an upper bound, not an off-by-one trap: a total exactly at the
// configured ceiling must still succeed.
func TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
local, err := NewLocalFS(t.TempDir())
if err != nil {
t.Fatalf("NewLocalFS: %v", err)
}
const partSize = 1024
const maxUploadFileBytes = 2 * partSize
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
parts := []string{strings.Repeat("a", partSize), strings.Repeat("b", partSize)}
for i, part := range parts {
if _, err := svc.SaveBigFilePart(ctx, 10, 301, i, len(parts), []byte(part)); err != nil {
t.Fatalf("SaveBigFilePart %d: %v", i, err)
}
}
doc, err := svc.CreateDocumentFromUpload(ctx,
domain.UploadedFileRef{OwnerUserID: 10, FileID: 301, Parts: len(parts), Name: "exact.bin", Big: true},
domain.DocumentSpec{MimeType: "application/octet-stream"},
)
if err != nil {
t.Fatalf("CreateDocumentFromUpload at exact max size: %v", err)
}
if doc.Size != int64(maxUploadFileBytes) {
t.Fatalf("doc size = %d, want %d", doc.Size, maxUploadFileBytes)
}
}
// TestMaxUploadFileBytesUnlimitedByDefault ensures leaving
// WithMaxUploadFileBytes unset (or 0) never rejects on size -- only the
// protocol's own MaxUploadPartBytes/MaxUploadParts ceiling still applies.
func TestMaxUploadFileBytesUnlimitedByDefault(t *testing.T) {
ctx := context.Background()
media := newFakeMediaStore()
svc, _ := newUploadPartTestService(t, media, domain.UploadPartQuota{})
file := domain.UploadedFileRef{OwnerUserID: 10, FileID: 302, Parts: 1, Name: "photo.jpg"}
if _, err := svc.SaveFilePart(ctx, file.OwnerUserID, file.FileID, 0, []byte(strings.Repeat("z", 4096))); err != nil {
t.Fatalf("SaveFilePart: %v", err)
}
if _, err := svc.CreatePhotoFromUpload(ctx, file); err != nil {
t.Fatalf("CreatePhotoFromUpload with no configured max size: %v", err)
}
}
type countingUploadPartBackend struct { type countingUploadPartBackend struct {
*LocalFS *LocalFS
getUploadPartCalls int getUploadPartCalls int

View file

@ -19,9 +19,38 @@ type OrphanedMediaRetentionStore interface {
// how long a document/photo must have been orphaned before it's actually // how long a document/photo must have been orphaned before it's actually
// deleted (not how old the media itself is) -- <=0 leaves the sweep // deleted (not how old the media itself is) -- <=0 leaves the sweep
// disabled even if a store is provided, matching // disabled even if a store is provided, matching
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default. // TELESRV_STORAGE_RETENTION_MODE=off/orphan-with-no-age being the safe
// default. Mutually exclusive with WithHardMediaRetention -- the resolved
// TELESRV_STORAGE_RETENTION_MODE is one of "off"/"orphan"/"hard", never
// both orphan and hard sweeps at once.
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker { func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.orphanedMedia = store w.orphanedMedia = store
w.orphanedMediaMaxAge = maxAge w.orphanedMediaMaxAge = maxAge
return w return w
} }
// HardMediaRetentionStore physically deletes a document/photo's blob bytes
// once the media itself (its upload/created time, not how long it has been
// orphaned) is older than the configured age, REGARDLESS of whether a live
// message/profile-photo/sticker-set still references it. Unlike
// OrphanedMediaRetentionStore, it must never delete the document/photo
// metadata row -- only the underlying file_blobs row(s) and, once confirming
// no other row still needs the same content-addressed object, the physical
// bytes. This keeps a message rendering its media placeholder (dimensions,
// mime type, filename) after the bytes are gone; a subsequent download
// resolves to LOCATION_INVALID instead of the message breaking outright.
type HardMediaRetentionStore interface {
DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
}
// WithHardMediaRetention enables the aggressive storage retention sweep
// (TELESRV_STORAGE_RETENTION_MODE=hard). maxAge is how old the media itself
// must be (its created_at, not an orphaned_at grace period) before its blob
// bytes are purged -- <=0 leaves the sweep disabled even if a store is
// provided. Mutually exclusive with WithOrphanedMediaRetention -- call at
// most one of the two, matching the config's single 3-way retention mode.
func (w *RetentionWorker) WithHardMediaRetention(store HardMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
w.hardMedia = store
w.hardMediaMaxAge = maxAge
return w
}

View file

@ -95,6 +95,10 @@ const (
// 继续保留,在线漏推由正常 difference 路径补偿。 // 继续保留,在线漏推由正常 difference 路径补偿。
defaultOutboxPoisonRetention = time.Minute defaultOutboxPoisonRetention = time.Minute
defaultOutboxPoisonInterval = 15 * time.Second defaultOutboxPoisonInterval = 15 * time.Second
// minMediaRetentionInterval floors the derived media-sweep cadence below
// so a very short (e.g. test-only) TELESRV_STORAGE_RETENTION_MAX_AGE
// can't spin the sweep query in a tight loop.
minMediaRetentionInterval = 15 * time.Second
) )
// RetentionWorker 周期性回收存储中的死数据。 // RetentionWorker 周期性回收存储中的死数据。
@ -119,6 +123,7 @@ type RetentionWorker struct {
activeAuthKeys ActiveRawAuthKeyProvider activeAuthKeys ActiveRawAuthKeyProvider
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
orphanedMedia OrphanedMediaRetentionStore orphanedMedia OrphanedMediaRetentionStore
hardMedia HardMediaRetentionStore
logger *zap.Logger logger *zap.Logger
retention time.Duration retention time.Duration
botAPIRetention time.Duration botAPIRetention time.Duration
@ -128,6 +133,7 @@ type RetentionWorker struct {
outboxPoisonRetention time.Duration outboxPoisonRetention time.Duration
outboxPoisonInterval time.Duration outboxPoisonInterval time.Duration
orphanedMediaMaxAge time.Duration orphanedMediaMaxAge time.Duration
hardMediaMaxAge time.Duration
interval time.Duration interval time.Duration
batch int batch int
} }
@ -259,6 +265,15 @@ func (w *RetentionWorker) Run(ctx context.Context) {
heartbeatC = heartbeatTicker.C heartbeatC = heartbeatTicker.C
defer heartbeatTicker.Stop() defer heartbeatTicker.Stop()
} }
var (
mediaTicker *time.Ticker
mediaC <-chan time.Time
)
if interval := w.mediaRetentionInterval(); interval > 0 {
mediaTicker = time.NewTicker(interval)
mediaC = mediaTicker.C
defer mediaTicker.Stop()
}
for { for {
select { select {
case <-ctx.Done(): case <-ctx.Done():
@ -269,6 +284,8 @@ func (w *RetentionWorker) Run(ctx context.Context) {
w.runOutboxPoisonOnce(ctx) w.runOutboxPoisonOnce(ctx)
case <-heartbeatC: case <-heartbeatC:
w.heartbeatActiveAuthKeys(ctx) w.heartbeatActiveAuthKeys(ctx)
case <-mediaC:
w.runMediaRetentionOnce(ctx)
} }
} }
} }
@ -276,6 +293,63 @@ func (w *RetentionWorker) Run(ctx context.Context) {
func (w *RetentionWorker) runOnce(ctx context.Context) { func (w *RetentionWorker) runOnce(ctx context.Context) {
w.runOutboxPoisonOnce(ctx) w.runOutboxPoisonOnce(ctx)
w.runRetentionOnce(ctx) w.runRetentionOnce(ctx)
w.runMediaRetentionOnce(ctx)
}
// mediaRetentionInterval derives how often the storage media sweep
// (orphan/hard) runs, independent of the shared housekeeping w.interval.
// A short TELESRV_STORAGE_RETENTION_MAX_AGE (e.g. 30m) configured under a
// much longer TELESRV_RETENTION_INTERVAL (default 1h) would otherwise let
// media sit for up to age+interval past its cutoff before actually being
// swept -- capping the sweep cadence at the retention age itself bounds
// that worst case to at most 2x the configured age instead.
func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
var maxAge time.Duration
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
maxAge = w.orphanedMediaMaxAge
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
maxAge = w.hardMediaMaxAge
}
if maxAge <= 0 {
return 0
}
interval := w.interval
if interval <= 0 || maxAge < interval {
interval = maxAge
}
if interval < minMediaRetentionInterval {
interval = minMediaRetentionInterval
}
return interval
}
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
// "Hard" mode: purges blob bytes for documents/photos older than
// hardMediaMaxAge regardless of whether a live reference remains --
// the store is required to keep the document/photo metadata row
// intact, only removing file_blobs rows/bytes, so a message still
// renders its media placeholder (LOCATION_INVALID on download)
// instead of breaking outright.
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
} else if hardDeleted > 0 {
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
}
}
} }
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) { func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
@ -414,17 +488,6 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted)) w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
} }
} }
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
// Only ever touches documents/photos already marked orphaned (no live
// message/profile-photo/sticker-set reference remains) -- media still
// visible in a conversation is never a candidate, regardless of age.
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
if err != nil {
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
} else if mediaDeleted > 0 {
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
}
}
} }
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration { func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {

View file

@ -393,3 +393,67 @@ func TestRetentionWorkerSkipsOrphanDeleteWhenHeartbeatFails(t *testing.T) {
t.Fatalf("heartbeat failure signals = %+v", entries) t.Fatalf("heartbeat failure signals = %+v", entries)
} }
} }
type fakeHardMediaRetention struct {
calls int
cutoff time.Time
limit int
deleted int
}
func (f *fakeHardMediaRetention) DeleteBlobBytesForMediaOlderThan(_ context.Context, cutoff time.Time, limit int) (int, error) {
f.calls++
f.cutoff = cutoff
f.limit = limit
return f.deleted, nil
}
// TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge guards the fix for
// a real gotcha reported live: with a short storage retention age (e.g. 30m)
// under the default 1h TELESRV_RETENTION_INTERVAL housekeeping cadence,
// media eligible for hard/orphan deletion used to sit for up to age+interval
// before the shared ticker actually got around to sweeping it (a 30m age
// could take up to 1h30m to actually disappear). The sweep now runs on its
// own ticker capped at the configured age, so it never lags by more than
// roughly one age-window.
func TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge(t *testing.T) {
hard := &fakeHardMediaRetention{}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, 30*time.Minute)
if got := w.mediaRetentionInterval(); got != 30*time.Minute {
t.Fatalf("media retention interval = %v, want min(interval=1h, maxAge=30m) = 30m", got)
}
}
func TestRetentionWorkerMediaSweepIntervalFloorsAtMinimum(t *testing.T) {
hard := &fakeHardMediaRetention{}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, time.Second)
if got := w.mediaRetentionInterval(); got != minMediaRetentionInterval {
t.Fatalf("media retention interval = %v, want floor %v", got, minMediaRetentionInterval)
}
}
func TestRetentionWorkerMediaSweepIntervalZeroWhenNoModeConfigured(t *testing.T) {
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50)
if got := w.mediaRetentionInterval(); got != 0 {
t.Fatalf("media retention interval = %v, want 0 (no separate ticker) when retention is off", got)
}
}
// TestRetentionWorkerRunsHardMediaSweepOnStartup confirms runOnce (called
// once immediately when Run starts, matching every other retention check)
// still fires the media sweep too, not just the new dedicated ticker.
func TestRetentionWorkerRunsHardMediaSweepOnStartup(t *testing.T) {
hard := &fakeHardMediaRetention{deleted: 4}
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
WithHardMediaRetention(hard, 30*time.Minute)
w.runOnce(context.Background())
if hard.calls != 1 {
t.Fatalf("hard media sweep calls = %d, want 1", hard.calls)
}
}

View file

@ -13,11 +13,20 @@ import (
"golang.org/x/text/language" "golang.org/x/text/language"
"telesrv/internal/app/files"
"telesrv/internal/branding" "telesrv/internal/branding"
"telesrv/internal/domain" "telesrv/internal/domain"
"telesrv/internal/links" "telesrv/internal/links"
) )
// Storage retention modes for TELESRV_STORAGE_RETENTION_MODE. See
// Config.StorageRetentionMode for what each one does.
const (
StorageRetentionModeOff = "off"
StorageRetentionModeOrphan = "orphan"
StorageRetentionModeHard = "hard"
)
const ( const (
defaultConfigFile = ".env" defaultConfigFile = ".env"
defaultCountryCode = "CN" defaultCountryCode = "CN"
@ -311,19 +320,41 @@ type Config struct {
// the s3 backend (no OS-level free space concept), and usable as an // the s3 backend (no OS-level free space concept), and usable as an
// optional soft budget cap on localfs too. <=0 disables this check. // optional soft budget cap on localfs too. <=0 disables this check.
StorageMaxTotalBytes int64 StorageMaxTotalBytes int64
// StorageMaxUploadFileBytes caps the total assembled size of a single
// uploaded file (sum of all its parts, not any one part) -- rejected at
// upload-finalize time with a FILE_TOO_BIG rpc error. <=0 disables this
// check (the only remaining ceiling is then the protocol's own part-count
// limit, files.MaxUploadPartBytes*files.MaxUploadParts, ~4GB). Validated
// at config-load time to never exceed that protocol ceiling -- a larger
// value could never actually be reached, so it's refused as a
// startup-time misconfiguration rather than silently accepted as a no-op.
StorageMaxUploadFileBytes int64
// StorageUsageRefreshInterval is how often the cached free-space/budget // StorageUsageRefreshInterval is how often the cached free-space/budget
// usage gauge refreshes; <=0 uses a 1 minute default. // usage gauge refreshes; <=0 uses a 1 minute default.
StorageUsageRefreshInterval time.Duration StorageUsageRefreshInterval time.Duration
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by // StorageRetentionMode selects the storage retention sweep behavior:
// default: orphaned media (no live message/profile-photo/sticker-set // - "off" (default): no sweep at all.
// reference) is tracked and visible in the admin panel, but nothing is // - "orphan": safe mode -- deletes a document/photo's blob only once it
// auto-deleted until the operator explicitly opts in. // has had no live message/profile-photo/sticker-set reference for at
StorageRetentionEnable bool // least StorageRetentionMaxAge. Never touches media still referenced,
// StorageRetentionMaxAge is how long a document/photo must have been // regardless of age.
// orphaned (not how old the media itself is) before the sweep deletes // - "hard": aggressive mode -- deletes a document/photo's blob bytes
// it. Never deletes media that still has a live reference, regardless // once the media itself (its upload/created time, not how long it's
// of age. <=0 uses a 30 day default. The sweep itself runs on the // been orphaned) is older than StorageRetentionMaxAge, REGARDLESS of
// shared RetentionInterval/RetentionBatch cadence alongside every other // whether it's still referenced by a live message. Only the blob
// bytes are removed; the document/photo metadata row is kept so the
// message still renders (dimensions, mime type, filename) --
// subsequent downloads get LOCATION_INVALID ("media no longer
// available") instead of the message breaking outright. This is
// irreversible and can surprise users if misunderstood: old media in
// active conversations WILL disappear.
// Any other value fails config validation.
StorageRetentionMode string
// StorageRetentionMaxAge is the shared age threshold for both "orphan"
// and "hard" modes -- see StorageRetentionMode for how its meaning
// differs between them. Ignored when StorageRetentionMode is "off". <=0
// uses a 30 day default. The sweep itself runs on the shared
// RetentionInterval/RetentionBatch cadence alongside every other
// retention check (see maintenance.RetentionWorker). // retention check (see maintenance.RetentionWorker).
StorageRetentionMaxAge time.Duration StorageRetentionMaxAge time.Duration
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob // StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob
@ -945,8 +976,9 @@ func Load() (Config, error) {
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true), StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30), StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0), StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
StorageMaxUploadFileBytes: envInt64Or("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", 0),
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute), StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false), StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour), StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"), StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300), StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
@ -1285,6 +1317,9 @@ func validateBlobStorageConfig(cfg Config) error {
if cfg.StorageMaxTotalBytes < 0 { if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative") return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
} }
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
return err
}
if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 { if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled") return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled")
} }
@ -1474,14 +1509,39 @@ func validateStorageConfig(cfg Config) error {
if cfg.StorageMaxTotalBytes < 0 { if cfg.StorageMaxTotalBytes < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative") return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
} }
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
return err
}
if cfg.StorageUsageRefreshInterval < 0 { if cfg.StorageUsageRefreshInterval < 0 {
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative") return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
} }
if cfg.StorageRetentionMaxAge < 0 { if cfg.StorageRetentionMaxAge < 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative") return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
} }
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 { switch cfg.StorageRetentionMode {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true") case StorageRetentionModeOff:
// no sweep; StorageRetentionMaxAge is ignored.
case StorageRetentionModeOrphan, StorageRetentionModeHard:
if cfg.StorageRetentionMaxAge <= 0 {
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
}
default:
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
}
return nil
}
// validateMaxUploadFileBytes checks TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: it
// must be non-negative, and if set (>0) must not exceed the protocol's own
// part-count upload ceiling -- a larger configured value could never actually
// be hit, so it's refused as a startup-time misconfiguration.
func validateMaxUploadFileBytes(v int64) error {
if v < 0 {
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES must be non-negative")
}
const protocolCeiling = int64(files.MaxUploadPartBytes) * int64(files.MaxUploadParts)
if v > protocolCeiling {
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES (%d) must not exceed the protocol upload ceiling of %d bytes (%d parts x %d bytes)", v, protocolCeiling, files.MaxUploadParts, files.MaxUploadPartBytes)
} }
return nil return nil
} }

View file

@ -1,6 +1,7 @@
package config package config
import ( import (
"fmt"
"os" "os"
"path/filepath" "path/filepath"
"testing" "testing"
@ -467,6 +468,101 @@ func TestLoadRejectsInvalidStorageCapacityConfig(t *testing.T) {
} }
} }
func TestLoadMaxUploadFileBytesDefaultsToUnlimited(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageMaxUploadFileBytes != 0 {
t.Fatalf("expected default StorageMaxUploadFileBytes=0 (unlimited), got %d", cfg.StorageMaxUploadFileBytes)
}
}
func TestLoadAcceptsMaxUploadFileBytesAtProtocolCeiling(t *testing.T) {
disableDefaultConfigFile(t)
const ceiling = int64(524288) * 8000 // files.MaxUploadPartBytes * files.MaxUploadParts
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", fmt.Sprint(ceiling))
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageMaxUploadFileBytes != ceiling {
t.Fatalf("expected StorageMaxUploadFileBytes=%d, got %d", ceiling, cfg.StorageMaxUploadFileBytes)
}
}
func TestLoadRejectsInvalidMaxUploadFileBytes(t *testing.T) {
const ceiling = int64(524288) * 8000
for _, item := range []struct {
name string
value string
}{
{"negative", "-1"},
{"exceeds protocol ceiling", fmt.Sprint(ceiling + 1)},
} {
t.Run(item.name, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", item.value)
if _, err := Load(); err == nil {
t.Fatalf("Load accepted invalid TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=%s", item.value)
}
})
}
}
func TestLoadStorageRetentionModeDefaultsToOff(t *testing.T) {
disableDefaultConfigFile(t)
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageRetentionMode != StorageRetentionModeOff {
t.Fatalf("expected default StorageRetentionMode=%q, got %q", StorageRetentionModeOff, cfg.StorageRetentionMode)
}
}
func TestLoadStorageRetentionModeOrphanAndHard(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "48h")
cfg, err := Load()
if err != nil {
t.Fatalf("Load: %v", err)
}
if cfg.StorageRetentionMode != mode {
t.Fatalf("expected StorageRetentionMode=%q, got %q", mode, cfg.StorageRetentionMode)
}
if cfg.StorageRetentionMaxAge != 48*time.Hour {
t.Fatalf("expected StorageRetentionMaxAge=48h, got %v", cfg.StorageRetentionMaxAge)
}
})
}
}
func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", "sometimes")
if _, err := Load(); err == nil {
t.Fatal("Load accepted an unknown TELESRV_STORAGE_RETENTION_MODE")
}
}
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
t.Run(mode, func(t *testing.T) {
disableDefaultConfigFile(t)
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
if _, err := Load(); err == nil {
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
}
})
}
}
func TestLoadUpdateServiceConfig(t *testing.T) { func TestLoadUpdateServiceConfig(t *testing.T) {
disableDefaultConfigFile(t) disableDefaultConfigFile(t)
t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/") t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/")

View file

@ -9,6 +9,11 @@ var (
ErrFilePartsInvalid = errors.New("file parts invalid") ErrFilePartsInvalid = errors.New("file parts invalid")
ErrFilePartTooBig = errors.New("file part too big") ErrFilePartTooBig = errors.New("file part too big")
ErrUploadQuotaExceeded = errors.New("upload quota exceeded") ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
// ErrFileTooLarge is returned when a file's total assembled size (sum of
// all its parts, not any single part) exceeds the configured
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES ceiling. Distinct from
// ErrFilePartTooBig, which is a single oversized part.
ErrFileTooLarge = errors.New("file too large")
ErrPhotoInvalid = errors.New("photo invalid") ErrPhotoInvalid = errors.New("photo invalid")
ErrDocumentInvalid = errors.New("document invalid") ErrDocumentInvalid = errors.New("document invalid")
// ErrStorageFull is returned when the configured low-space guard rejects a // ErrStorageFull is returned when the configured low-space guard rejects a

View file

@ -133,3 +133,56 @@ func TestWriteEnvValuesRoundTrip(t *testing.T) {
t.Error(".env lost .env.example's comment lines on write") t.Error(".env lost .env.example's comment lines on write")
} }
} }
// TestWriteEnvValuesPreservesUnrelatedCustomValues guards against a real
// regression: a save that only touches one section's keys used to reset
// every OTHER already-customized key (e.g. TELESRV_ADMIN_UI_PASSWORD) back
// to .env.example's bare template default, because the old implementation
// fell back to the template line instead of the current .env value for any
// key missing from that save's payload.
func TestWriteEnvValuesPreservesUnrelatedCustomValues(t *testing.T) {
root := findRepoRoot(t)
tmpl, err := os.ReadFile(filepath.Join(root, ".env.example"))
if err != nil {
t.Fatal(err)
}
dir := t.TempDir()
if err := os.WriteFile(filepath.Join(dir, ".env.example"), tmpl, 0o644); err != nil {
t.Fatal(err)
}
m := NewManager(dir)
// First save sets the admin password, as a one-time setup step would.
if err := m.WriteEnvValues(map[string]string{
"TELESRV_ADMIN_UI_PASSWORD": "s3cret",
}); err != nil {
t.Fatal(err)
}
// A later, unrelated save (e.g. the Storage page) that never mentions
// the password must not disturb it.
if err := m.WriteEnvValues(map[string]string{
"TELESRV_STORAGE_MAX_TOTAL_BYTES": "209715200",
}); err != nil {
t.Fatal(err)
}
groups, err := m.ReadEnvGroups()
if err != nil {
t.Fatal(err)
}
values := map[string]string{}
for _, g := range groups {
for _, f := range g.Fields {
values[f.Key] = f.Value
}
}
if values["TELESRV_ADMIN_UI_PASSWORD"] != "s3cret" {
t.Errorf("TELESRV_ADMIN_UI_PASSWORD = %q, want it preserved as \"s3cret\" after an unrelated save", values["TELESRV_ADMIN_UI_PASSWORD"])
}
if values["TELESRV_STORAGE_MAX_TOTAL_BYTES"] != "209715200" {
t.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES = %q, want 209715200", values["TELESRV_STORAGE_MAX_TOTAL_BYTES"])
}
}

View file

@ -637,15 +637,25 @@ func (m *Manager) readEnvFile() (map[string]string, error) {
// WriteEnvValues rewrites .env from .env.example's exact text, substituting // WriteEnvValues rewrites .env from .env.example's exact text, substituting
// each known key's value in place -- see save_env()'s docstring in // each known key's value in place -- see save_env()'s docstring in
// server-panel.py for why this (not a fresh key=value dump) is what // server-panel.py for why this (not a fresh key=value dump) is what
// preserves comments/layout. Only keys present in values are touched; a // preserves comments/layout. Only keys present in values are set to a new
// template-commented optional field is uncommented when given a non-empty // value; every other key keeps whatever is already in the current .env
// value and left as-is when given an empty one. // (falling back to the template's own default only for a key .env never
// set) -- previously this fell straight back to the template default for
// any key not in this particular save's payload, silently wiping out every
// other customized setting (e.g. TELESRV_ADMIN_UI_PASSWORD) on every save
// that only touches one section's keys. A template-commented optional field
// is uncommented when given a non-empty value and left as-is when given an
// empty one.
func (m *Manager) WriteEnvValues(values map[string]string) error { func (m *Manager) WriteEnvValues(values map[string]string) error {
tmplPath := filepath.Join(m.Root, ".env.example") tmplPath := filepath.Join(m.Root, ".env.example")
tmplData, err := os.ReadFile(tmplPath) tmplData, err := os.ReadFile(tmplPath)
if err != nil { if err != nil {
return fmt.Errorf("read .env.example: %w", err) return fmt.Errorf("read .env.example: %w", err)
} }
existing, err := m.readEnvFile()
if err != nil {
return err
}
lines := strings.Split(string(tmplData), "\n") lines := strings.Split(string(tmplData), "\n")
// Split() on a trailing "\n" leaves one empty trailing element; drop it // Split() on a trailing "\n" leaves one empty trailing element; drop it
// so the join below doesn't add a spurious blank line before the final // so the join below doesn't add a spurious blank line before the final
@ -657,15 +667,20 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
seen := map[string]bool{} seen := map[string]bool{}
for _, raw := range lines { for _, raw := range lines {
line := strings.TrimSpace(raw) line := strings.TrimSpace(raw)
if a := activeFieldRe.FindStringSubmatch(line); a != nil { if a := activeFieldRe.FindStringSubmatch(line); a != nil && !seen[a[1]] {
if v, ok := values[a[1]]; ok && !seen[a[1]] { if v, ok := values[a[1]]; ok {
seen[a[1]] = true
out = append(out, a[1]+"="+v)
continue
}
if v, ok := existing[a[1]]; ok {
seen[a[1]] = true seen[a[1]] = true
out = append(out, a[1]+"="+v) out = append(out, a[1]+"="+v)
continue continue
} }
} }
if c := commentedFieldRe.FindStringSubmatch(line); c != nil { if c := commentedFieldRe.FindStringSubmatch(line); c != nil && !seen[c[1]] {
if v, ok := values[c[1]]; ok && !seen[c[1]] { if v, ok := values[c[1]]; ok {
seen[c[1]] = true seen[c[1]] = true
if v != "" { if v != "" {
out = append(out, c[1]+"="+v) out = append(out, c[1]+"="+v)
@ -674,6 +689,14 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
} }
continue continue
} }
// A previously-enabled optional field shows up in the current
// .env as an active line even though the template still has it
// commented out -- keep it enabled with its existing value.
if v, ok := existing[c[1]]; ok {
seen[c[1]] = true
out = append(out, c[1]+"="+v)
continue
}
} }
out = append(out, raw) out = append(out, raw)
} }

View file

@ -107,6 +107,15 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") } func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") } func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
// fileTooBigErr surfaces TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: the upload's
// total assembled size (not any single part) exceeds the configured
// per-file ceiling. FILE_TOO_BIG is not an official Telegram desktop-RPC
// error string, but it's already this codebase's own convention for the
// same concept on the Bot API surface (see internal/botapi/server.go) --
// reused here for consistency rather than inventing a second name for the
// same condition.
func fileTooBigErr() error { return tgerr.New(400, "FILE_TOO_BIG") }
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not // storageFullErr surfaces the low-disk-space upload guard. Deliberately not
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling // a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
// the client to retry shortly would be misleading. // the client to retry shortly would be misleading.

View file

@ -1152,6 +1152,8 @@ func mediaUploadErr(err error) error {
switch { switch {
case errors.Is(err, domain.ErrFilePartsInvalid): case errors.Is(err, domain.ErrFilePartsInvalid):
return filePartsInvalidErr() return filePartsInvalidErr()
case errors.Is(err, domain.ErrFileTooLarge):
return fileTooBigErr()
case errors.Is(err, domain.ErrPhotoInvalid): case errors.Is(err, domain.ErrPhotoInvalid):
return photoInvalidErr() return photoInvalidErr()
case errors.Is(err, domain.ErrDocumentInvalid): case errors.Is(err, domain.ErrDocumentInvalid):

View file

@ -348,6 +348,8 @@ func fileSaveErr(err error) error {
return filePartsInvalidErr() return filePartsInvalidErr()
case errors.Is(err, domain.ErrFilePartTooBig): case errors.Is(err, domain.ErrFilePartTooBig):
return filePartTooBigErr() return filePartTooBigErr()
case errors.Is(err, domain.ErrFileTooLarge):
return fileTooBigErr()
case errors.Is(err, domain.ErrUploadQuotaExceeded): case errors.Is(err, domain.ErrUploadQuotaExceeded):
return floodWaitErr(60) return floodWaitErr(60)
case errors.Is(err, domain.ErrStorageFull): case errors.Is(err, domain.ErrStorageFull):

View file

@ -0,0 +1,105 @@
package postgres
import (
"context"
"strconv"
"testing"
"time"
"telesrv/internal/domain"
)
// TestHardRetentionPurgesBlobBytesButKeepsMetadataRow exercises the "hard"
// storage retention mode's store methods end-to-end against a real
// Postgres: a document old enough (by created_at, not orphaned_at) is a
// candidate for ListDocumentIDsForHardRetentionOlderThan REGARDLESS of
// still having a live media_references row, DeleteFileBlobsForDocument
// removes only its file_blobs row (never the documents row itself), and a
// second sweep pass no longer finds it a candidate since it no longer owns
// any file_blobs row. This is the correctness property the whole "hard"
// mode design hinges on: a message must still be able to render its media
// placeholder after the bytes are gone.
func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
pool := testPool(t)
ctx := context.Background()
media := NewMediaStore(pool)
docID := time.Now().UnixNano()
locationKey := "doc:" + strconv.FormatInt(docID, 10)
if err := media.PutDocument(ctx, domain.Document{
ID: docID,
MimeType: "application/octet-stream",
Size: 1024,
}); err != nil {
t.Fatalf("PutDocument: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
})
// Backdate created_at well past any plausible test cutoff -- PutDocument
// always stamps now() and has no created_at parameter.
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days' WHERE id = $1", docID); err != nil {
t.Fatalf("backdate document: %v", err)
}
blob := postgresTestBlob(locationKey, "hard-retention-doc", 1024, "application/octet-stream")
if err := media.PutFileBlob(ctx, blob); err != nil {
t.Fatalf("PutFileBlob: %v", err)
}
// A LIVE reference (as if a message still embeds this document) must not
// exempt it from "hard" mode -- that's the entire point of the mode,
// unlike the orphan-only sweep.
if err := addMediaReferencesTx(ctx, pool, &domain.MessageMedia{
Kind: domain.MessageMediaKindDocument,
Document: &domain.Document{ID: docID},
}, domain.MediaRefKindMessageBox, "hard-retention-test:1"); err != nil {
t.Fatalf("add media reference: %v", err)
}
t.Cleanup(func() {
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE ref_key = 'hard-retention-test:1'")
})
cutoff := time.Now().Add(-24 * time.Hour)
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
}
if !containsInt64(ids, docID) {
t.Fatalf("hard retention candidates = %v, want to include still-referenced but old document %d", ids, docID)
}
blobs, err := media.DeleteFileBlobsForDocument(ctx, docID)
if err != nil {
t.Fatalf("DeleteFileBlobsForDocument: %v", err)
}
if len(blobs) != 1 || blobs[0].LocationKey != locationKey {
t.Fatalf("deleted blobs = %+v, want exactly one for %q", blobs, locationKey)
}
// The documents row itself must survive -- a message referencing it
// still needs to render its placeholder (mime type, size, filename).
if _, found, err := media.GetDocument(ctx, docID); err != nil || !found {
t.Fatalf("document metadata row missing after hard blob purge: found=%v err=%v", found, err)
}
// The file_blobs row is gone: a subsequent download attempt resolves via
// GetFileBlob (files.Service.GetFile's path) to not-found, which the rpc
// layer already maps to LOCATION_INVALID.
if _, found, err := media.GetFileBlob(ctx, locationKey); err != nil || found {
t.Fatalf("file_blobs row still present after hard purge: found=%v err=%v", found, err)
}
// Idempotent / self-terminating: a second sweep pass no longer selects
// this document, since it no longer owns any file_blobs row.
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
if err != nil {
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
}
if containsInt64(ids, docID) {
t.Fatalf("hard retention re-selected already-purged document %d", docID)
}
}

View file

@ -197,6 +197,101 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
return blobs, nil return blobs, nil
} }
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
// cutoff (by upload/created_at) that still own at least one file_blobs row,
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
// entirely: a document still referenced by a live message is exactly as
// eligible as an orphaned one.
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
// ListDocumentIDsForHardRetentionOlderThan.
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
if limit <= 0 {
return nil, nil
}
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
BatchLimit: int32(limit),
})
}
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
// (main body + thumbnail variants), returning what was deleted so the caller
// can physically remove each object from its backend once confirming (via
// CountFileBlobRefs, after this call) no other row still needs it. Unlike
// DeleteDocumentAndBlobs, this deliberately does NOT delete the documents
// row itself -- "hard" retention mode keeps the metadata (dimensions, mime
// type, filename) so a message can still render "this document is no longer
// available" instead of disappearing outright. A subsequent
// upload.getFile/GetFileBlob lookup for a location key this call removed
// correctly finds nothing and reports not-found.
func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete document blob bytes (hard retention)", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("doc:%d", id),
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list document blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}
// DeleteFileBlobsForPhoto is the photo counterpart of
// DeleteFileBlobsForDocument -- see its doc comment. Deliberately does not
// delete the photos row.
func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) {
var blobs []domain.FileBlob
err := withTx(ctx, s.db, "delete photo blob bytes (hard retention)", func(tx pgx.Tx) error {
qtx := s.q.WithTx(tx)
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
ExactKey: fmt.Sprintf("photo:%d", id),
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
})
if err != nil {
return fmt.Errorf("list photo blobs: %w", err)
}
for _, r := range rows {
blobs = append(blobs, domain.FileBlob{
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
})
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
return fmt.Errorf("delete file blob row: %w", err)
}
}
return nil
})
if err != nil {
return nil, err
}
return blobs, nil
}
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns // DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
// (one per rendition size), returning what was deleted so the caller can // (one per rendition size), returning what was deleted so the caller can
// physically remove each object from its backend once confirming (via // physically remove each object from its backend once confirming (via

View file

@ -237,6 +237,37 @@ WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
ORDER BY orphaned_at ASC ORDER BY orphaned_at ASC
LIMIT sqlc.arg(batch_limit)::int; LIMIT sqlc.arg(batch_limit)::int;
-- name: ListDocumentIDsForHardRetentionOlderThan :many
-- "Hard" retention mode: candidates are documents older than cutoff (by
-- upload/created_at, NOT orphaned_at -- a live reference does not exempt
-- them) that still own at least one file_blobs row. The EXISTS check is what
-- keeps this sweep from re-selecting the same document forever: once its
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
-- rows but deliberately leaves this documents row in place), it naturally
-- drops out of this query on the next pass.
SELECT d.id FROM documents d
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: ListPhotoIDsForHardRetentionOlderThan :many
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
-- candidate selection, for photos.
SELECT p.id FROM photos p
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT sqlc.arg(batch_limit)::int;
-- name: CountFileBlobRefs :one -- name: CountFileBlobRefs :one
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text; SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;

View file

@ -825,6 +825,50 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
return items, nil return items, nil
} }
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
SELECT d.id FROM documents d
WHERE d.created_at < $1::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'doc:' || d.id::text
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
)
ORDER BY d.created_at ASC
LIMIT $2::int
`
type ListDocumentIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// "Hard" retention mode: candidates are documents older than cutoff (by
// upload/created_at, NOT orphaned_at -- a live reference does not exempt
// them) that still own at least one file_blobs row. The EXISTS check is what
// keeps this sweep from re-selecting the same document forever: once its
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
// rows but deliberately leaves this documents row in place), it naturally
// drops out of this query on the next pass.
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
SELECT location_key, backend, object_key, size SELECT location_key, backend, object_key, size
FROM file_blobs FROM file_blobs
@ -937,6 +981,45 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
return items, nil return items, nil
} }
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
SELECT p.id FROM photos p
WHERE p.created_at < $1::timestamptz
AND EXISTS (
SELECT 1 FROM file_blobs fb
WHERE fb.location_key = 'photo:' || p.id::text
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
)
ORDER BY p.created_at ASC
LIMIT $2::int
`
type ListPhotoIDsForHardRetentionOlderThanParams struct {
Cutoff pgtype.Timestamptz
BatchLimit int32
}
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
// candidate selection, for photos.
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
if err != nil {
return nil, err
}
defer rows.Close()
var items []int64
for rows.Next() {
var id int64
if err := rows.Scan(&id); err != nil {
return nil, err
}
items = append(items, id)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const listProfilePhotos = `-- name: ListProfilePhotos :many const listProfilePhotos = `-- name: ListProfilePhotos :many
SELECT photo_id SELECT photo_id
FROM profile_photos FROM profile_photos