adding more functions to media managament system
This commit is contained in:
parent
95e62c2d77
commit
70c0ba44f0
30 changed files with 1494 additions and 104 deletions
50
.env.example
50
.env.example
|
|
@ -292,14 +292,33 @@ TELESRV_S3_USE_SSL=false
|
||||||
# MinIO needs this on (bucket in the URL path); AWS S3 does not.
|
# MinIO needs this on (bucket in the URL path); AWS S3 does not.
|
||||||
TELESRV_S3_PATH_STYLE=true
|
TELESRV_S3_PATH_STYLE=true
|
||||||
# Reject new uploads once storage is nearly full, instead of letting the disk
|
# Reject new uploads once storage is nearly full, instead of letting the disk
|
||||||
# fill up. Thresholds live in the Advanced section below.
|
# fill up. Thresholds are the three fields right below.
|
||||||
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
|
TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE=true
|
||||||
# Automatically delete old media once it's no longer referenced by any
|
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
|
||||||
# message, profile photo, or sticker set (never deletes media still visible
|
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
|
||||||
# in a conversation). Off by default -- storage usage is tracked and shown
|
# Reject new uploads once total tracked blob bytes would exceed this. The only
|
||||||
# in the admin panel either way; enable this once you're comfortable with
|
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
|
||||||
# what it will reclaim. Retention age lives in the Advanced section below.
|
# optional soft cap on localfs too. <=0 disables.
|
||||||
TELESRV_STORAGE_RETENTION_ENABLE=false
|
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
|
||||||
|
# Reject a single upload once its total assembled size (sum of all its parts)
|
||||||
|
# would exceed this. <=0 disables this check (the protocol's own part-count
|
||||||
|
# ceiling of ~4GB still applies). Must not exceed that ceiling.
|
||||||
|
TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=0
|
||||||
|
# Storage retention sweep mode: "off" (default, nothing auto-deleted --
|
||||||
|
# storage usage is still tracked and shown in the admin panel either way),
|
||||||
|
# "orphan" (safe: deletes a document/photo's blob only once it's no longer
|
||||||
|
# referenced by any message/profile photo/sticker set), or "hard"
|
||||||
|
# (aggressive: deletes a document/photo's blob once it's old enough,
|
||||||
|
# REGARDLESS of whether it's still referenced -- old media in active
|
||||||
|
# conversations will show as unavailable).
|
||||||
|
TELESRV_STORAGE_RETENTION_MODE=off
|
||||||
|
# How long a document/photo must have had zero references before the sweep
|
||||||
|
# deletes it in "orphan" mode above -- or how old the media itself is before
|
||||||
|
# "hard" mode deletes its bytes regardless of references. Ignored when the
|
||||||
|
# mode above is "off". The sweep itself runs alongside every other retention
|
||||||
|
# check on the shared TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH
|
||||||
|
# cadence (Advanced section below).
|
||||||
|
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
|
||||||
|
|
||||||
|
|
||||||
# ==============================================================================
|
# ==============================================================================
|
||||||
|
|
@ -438,22 +457,9 @@ TELESRV_GIF_SEED_DIR=data/gifs
|
||||||
# read fresh on every request -- editing them takes effect with no restart.
|
# read fresh on every request -- editing them takes effect with no restart.
|
||||||
TELESRV_IDENTITY_DIR=data/identity
|
TELESRV_IDENTITY_DIR=data/identity
|
||||||
|
|
||||||
# Storage low-space guard thresholds (master toggle is TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE above).
|
# How often the cached free-space/usage gauge behind the low-space guard
|
||||||
# localfs: reject new uploads once real free disk bytes fall below this; <=0 disables.
|
# (thresholds now live in the Storage & Media section above) refreshes.
|
||||||
TELESRV_STORAGE_MIN_FREE_BYTES=1073741824
|
|
||||||
# Reject new uploads once total tracked blob bytes would exceed this. The only
|
|
||||||
# meaningful "low space" signal on the s3 backend (no OS free-space concept);
|
|
||||||
# optional soft cap on localfs too. <=0 disables.
|
|
||||||
TELESRV_STORAGE_MAX_TOTAL_BYTES=0
|
|
||||||
# How often the cached free-space/usage gauge behind the guard above refreshes.
|
|
||||||
TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m
|
TELESRV_STORAGE_USAGE_REFRESH_INTERVAL=1m
|
||||||
# Storage retention sweep tuning (master toggle is TELESRV_STORAGE_RETENTION_ENABLE above).
|
|
||||||
# How long a document/photo must have had zero references before the sweep
|
|
||||||
# deletes it -- not how old the media itself is, and it never touches media
|
|
||||||
# still referenced by a live message/profile-photo/sticker-set. The sweep
|
|
||||||
# itself runs alongside every other retention check on the shared
|
|
||||||
# TELESRV_RETENTION_INTERVAL/TELESRV_RETENTION_BATCH cadence above.
|
|
||||||
TELESRV_STORAGE_RETENTION_MAX_AGE=720h
|
|
||||||
|
|
||||||
# New-account perks: free Telegram Premium months.
|
# New-account perks: free Telegram Premium months.
|
||||||
TELESRV_PREMIUM_GRANT_MONTHS=3
|
TELESRV_PREMIUM_GRANT_MONTHS=3
|
||||||
|
|
|
||||||
|
|
@ -2612,11 +2612,25 @@ type AccountStorageRow struct {
|
||||||
FileCount int64 `json:"FileCount,string"`
|
FileCount int64 `json:"FileCount,string"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ListAccountStorageUsage pages the per-account storage breakdown, largest
|
// storageUsageSortColumns whitelists the columns ListAccountStorageUsage's
|
||||||
// user first. Offset-based (not keyset): storage administration on a
|
// sortBy may map to -- never interpolate the caller's sort key directly into
|
||||||
// self-hosted deployment doesn't need to support arbitrarily deep pages
|
// SQL, since unlike a plain value it can't go through a query parameter.
|
||||||
// efficiently the way an infinite-scroll feed does.
|
var storageUsageSortColumns = map[string]string{
|
||||||
func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit int) ([]AccountStorageRow, bool, error) {
|
"bytes": "t.bytes",
|
||||||
|
"files": "t.file_count",
|
||||||
|
"user_id": "t.owner_user_id",
|
||||||
|
"username": "lower(COALESCE(u.username, ''))",
|
||||||
|
"first_name": "lower(COALESCE(u.first_name, ''))",
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListAccountStorageUsage pages the per-account storage breakdown. sortBy
|
||||||
|
// selects one of storageUsageSortColumns (falls back to "bytes" for an
|
||||||
|
// unknown/empty key); sortDesc reverses it. q, if non-empty, filters to
|
||||||
|
// accounts whose id/username/first name match it. Offset-based (not
|
||||||
|
// keyset): storage administration on a self-hosted deployment doesn't need
|
||||||
|
// to support arbitrarily deep pages efficiently the way an infinite-scroll
|
||||||
|
// feed does.
|
||||||
|
func (s *readStore) ListAccountStorageUsage(ctx context.Context, q string, sortBy string, sortDesc bool, offset, limit int) ([]AccountStorageRow, bool, error) {
|
||||||
if limit <= 0 {
|
if limit <= 0 {
|
||||||
limit = storageUsageListDefaultLimit
|
limit = storageUsageListDefaultLimit
|
||||||
}
|
}
|
||||||
|
|
@ -2626,6 +2640,32 @@ func (s *readStore) ListAccountStorageUsage(ctx context.Context, offset, limit i
|
||||||
if offset < 0 {
|
if offset < 0 {
|
||||||
offset = 0
|
offset = 0
|
||||||
}
|
}
|
||||||
|
column, ok := storageUsageSortColumns[sortBy]
|
||||||
|
if !ok {
|
||||||
|
column = storageUsageSortColumns["bytes"]
|
||||||
|
}
|
||||||
|
direction := "ASC"
|
||||||
|
if sortDesc {
|
||||||
|
direction = "DESC"
|
||||||
|
}
|
||||||
|
q = strings.TrimSpace(q)
|
||||||
|
var whereClause string
|
||||||
|
args := []any{}
|
||||||
|
argN := 1
|
||||||
|
if q != "" {
|
||||||
|
id := int64(-1)
|
||||||
|
if n, err := strconv.ParseInt(q, 10, 64); err == nil {
|
||||||
|
id = n
|
||||||
|
}
|
||||||
|
whereClause = fmt.Sprintf(`WHERE t.owner_user_id = $%d
|
||||||
|
OR lower(COALESCE(u.username, '')) LIKE $%d
|
||||||
|
OR lower(COALESCE(u.first_name, '')) LIKE $%d`, argN, argN+1, argN+2)
|
||||||
|
args = append(args, id, "%"+strings.ToLower(q)+"%", "%"+strings.ToLower(q)+"%")
|
||||||
|
argN += 3
|
||||||
|
}
|
||||||
|
offsetArg := argN
|
||||||
|
limitArg := argN + 1
|
||||||
|
args = append(args, offset, limit+1)
|
||||||
rows, err := s.pool.Query(ctx, `
|
rows, err := s.pool.Query(ctx, `
|
||||||
WITH totals AS (
|
WITH totals AS (
|
||||||
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
|
SELECT owner_user_id, SUM(size)::bigint AS bytes, COUNT(*)::bigint AS file_count
|
||||||
|
|
@ -2636,9 +2676,10 @@ WITH totals AS (
|
||||||
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
|
SELECT t.owner_user_id, COALESCE(u.username, ''), COALESCE(u.first_name, ''), t.bytes, t.file_count
|
||||||
FROM totals t
|
FROM totals t
|
||||||
LEFT JOIN users u ON u.id = t.owner_user_id
|
LEFT JOIN users u ON u.id = t.owner_user_id
|
||||||
ORDER BY t.bytes DESC, t.owner_user_id
|
`+whereClause+`
|
||||||
OFFSET $1
|
ORDER BY `+column+` `+direction+`, t.owner_user_id
|
||||||
LIMIT $2`, offset, limit+1)
|
OFFSET $`+strconv.Itoa(offsetArg)+`
|
||||||
|
LIMIT $`+strconv.Itoa(limitArg), args...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, false, fmt.Errorf("list account storage usage: %w", err)
|
return nil, false, fmt.Errorf("list account storage usage: %w", err)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -2498,7 +2498,14 @@ func (s *server) handleStorageAccountsAPI(w http.ResponseWriter, r *http.Request
|
||||||
writeAPIError(w, http.StatusBadRequest, "invalid limit")
|
writeAPIError(w, http.StatusBadRequest, "invalid limit")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
rows, hasMore, err := s.read.ListAccountStorageUsage(r.Context(), offset, limit)
|
sortDesc := query.Get("order") != "asc"
|
||||||
|
rows, hasMore, err := s.read.ListAccountStorageUsage(
|
||||||
|
r.Context(),
|
||||||
|
query.Get("q"),
|
||||||
|
query.Get("sort"),
|
||||||
|
sortDesc,
|
||||||
|
offset,
|
||||||
|
limit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeAPIError(w, http.StatusInternalServerError, err.Error())
|
writeAPIError(w, http.StatusInternalServerError, err.Error())
|
||||||
return
|
return
|
||||||
|
|
|
||||||
1
cmd/telesrv-admin/web/dist/assets/index-B7hI8ol7.css
vendored
Normal file
1
cmd/telesrv-admin/web/dist/assets/index-B7hI8ol7.css
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
9
cmd/telesrv-admin/web/dist/assets/index-IZM8WmmY.js
vendored
Normal file
9
cmd/telesrv-admin/web/dist/assets/index-IZM8WmmY.js
vendored
Normal file
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
4
cmd/telesrv-admin/web/dist/index.html
vendored
4
cmd/telesrv-admin/web/dist/index.html
vendored
|
|
@ -23,8 +23,8 @@
|
||||||
})();
|
})();
|
||||||
</script>
|
</script>
|
||||||
|
|
||||||
<script type="module" crossorigin src="/assets/index-CP20QCwX.js"></script>
|
<script type="module" crossorigin src="/assets/index-IZM8WmmY.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-Wtwg__bv.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-B7hI8ol7.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div id="root"></div>
|
<div id="root"></div>
|
||||||
|
|
|
||||||
|
|
@ -1,18 +1,52 @@
|
||||||
import { ChevronDown, Loader2, RefreshCw } from "lucide-react";
|
import { ArrowDown, ArrowUp, ArrowUpDown, ChevronDown, Loader2, RefreshCw, Search } from "lucide-react";
|
||||||
import { useEffect, useState } from "react";
|
import { useEffect, useMemo, useState } from "react";
|
||||||
import { api, errorMessage } from "../api";
|
import { api, errorMessage } from "../api";
|
||||||
import { Alert, EmptyRow, Metric, PageFrame } from "../components/ui";
|
import { ActionButton } from "../components/ActionButton";
|
||||||
|
import { Alert, EmptyRow, Metric, PageFrame, QueryPanel, SectionHead } from "../components/ui";
|
||||||
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
|
import { displayUsername, formatBytes, formatQuantity } from "../lib/format";
|
||||||
import type { Navigate } from "../routing";
|
import type { Navigate } from "../routing";
|
||||||
import type { AccountStorageRow, StorageStatsResponse } from "../types";
|
import type { AccountStorageRow, StorageStatsResponse } from "../types";
|
||||||
|
|
||||||
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
type StorageSortKey = "user_id" | "username" | "bytes" | "files";
|
||||||
|
|
||||||
|
// SortableHeader is a <th> that toggles ascending/descending on click and
|
||||||
|
// shows which column (and direction) is currently active -- there's no
|
||||||
|
// existing sortable-table convention elsewhere in this admin panel to
|
||||||
|
// mirror, so this is a small, self-contained one for this page.
|
||||||
|
function SortableHeader({
|
||||||
|
label,
|
||||||
|
sortKey,
|
||||||
|
activeKey,
|
||||||
|
desc,
|
||||||
|
onSort
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
sortKey: StorageSortKey;
|
||||||
|
activeKey: StorageSortKey;
|
||||||
|
desc: boolean;
|
||||||
|
onSort: (key: StorageSortKey) => void;
|
||||||
|
}) {
|
||||||
|
const active = sortKey === activeKey;
|
||||||
|
return (
|
||||||
|
<th>
|
||||||
|
<button type="button" className="sort-header" onClick={() => onSort(sortKey)}>
|
||||||
|
{label}
|
||||||
|
{active ? (desc ? <ArrowDown size={13} /> : <ArrowUp size={13} />) : <ArrowUpDown size={13} className="sort-header-idle" />}
|
||||||
|
</button>
|
||||||
|
</th>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function StorageOverviewTab() {
|
||||||
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
|
const [stats, setStats] = useState<StorageStatsResponse | null>(null);
|
||||||
const [rows, setRows] = useState<AccountStorageRow[]>([]);
|
const [rows, setRows] = useState<AccountStorageRow[]>([]);
|
||||||
const [hasMore, setHasMore] = useState(false);
|
const [hasMore, setHasMore] = useState(false);
|
||||||
const [offset, setOffset] = useState(0);
|
const [offset, setOffset] = useState(0);
|
||||||
const [busy, setBusy] = useState(false);
|
const [busy, setBusy] = useState(false);
|
||||||
const [error, setError] = useState("");
|
const [error, setError] = useState("");
|
||||||
|
const [q, setQ] = useState("");
|
||||||
|
const [sortKey, setSortKey] = useState<StorageSortKey>("bytes");
|
||||||
|
const [sortDesc, setSortDesc] = useState(true);
|
||||||
|
|
||||||
async function loadStats() {
|
async function loadStats() {
|
||||||
try {
|
try {
|
||||||
|
|
@ -26,7 +60,13 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
setBusy(true);
|
setBusy(true);
|
||||||
setError("");
|
setError("");
|
||||||
const at = next ? offset : 0;
|
const at = next ? offset : 0;
|
||||||
const params = new URLSearchParams({ limit: "50", offset: String(at) });
|
const params = new URLSearchParams({
|
||||||
|
limit: "50",
|
||||||
|
offset: String(at),
|
||||||
|
sort: sortKey,
|
||||||
|
order: sortDesc ? "desc" : "asc"
|
||||||
|
});
|
||||||
|
if (q.trim()) params.set("q", q.trim());
|
||||||
try {
|
try {
|
||||||
const result = await api.storageAccounts(params);
|
const result = await api.storageAccounts(params);
|
||||||
const page = result.rows ?? [];
|
const page = result.rows ?? [];
|
||||||
|
|
@ -48,7 +88,16 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
useEffect(() => {
|
useEffect(() => {
|
||||||
refresh();
|
refresh();
|
||||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
}, []);
|
}, [sortKey, sortDesc]);
|
||||||
|
|
||||||
|
function toggleSort(key: StorageSortKey) {
|
||||||
|
if (key === sortKey) {
|
||||||
|
setSortDesc((current) => !current);
|
||||||
|
} else {
|
||||||
|
setSortKey(key);
|
||||||
|
setSortDesc(true);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
|
// Physical is what actually consumes disk/S3 (deduplicated); logical is the
|
||||||
// sum of what the per-account table below adds up to. They legitimately
|
// sum of what the per-account table below adds up to. They legitimately
|
||||||
|
|
@ -56,15 +105,7 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
|
const dedupBytes = stats ? Math.max(0, Number(stats.LogicalBytes) - Number(stats.PhysicalBytes)) : 0;
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<PageFrame
|
<>
|
||||||
title={"Storage"}
|
|
||||||
eyebrow={"Media / Storage usage"}
|
|
||||||
actions={
|
|
||||||
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
|
|
||||||
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
|
|
||||||
</button>
|
|
||||||
}
|
|
||||||
>
|
|
||||||
{error && <Alert>{error}</Alert>}
|
{error && <Alert>{error}</Alert>}
|
||||||
<div className="metric-row">
|
<div className="metric-row">
|
||||||
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
|
<Metric label={"Physical usage (on disk / S3)"} value={stats ? formatBytes(stats.PhysicalBytes) : "-"} />
|
||||||
|
|
@ -83,14 +124,29 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
<QueryPanel>
|
||||||
|
<form className="toolbar" onSubmit={(event) => { event.preventDefault(); void loadAccounts(false); }}>
|
||||||
|
<label className="searchbox">
|
||||||
|
<Search size={15} />
|
||||||
|
<input value={q} onChange={(event) => setQ(event.target.value)} placeholder={"User ID / username / name"} />
|
||||||
|
</label>
|
||||||
|
<button className="btn primary icon-text" type="submit" disabled={busy}>
|
||||||
|
{busy ? <Loader2 size={15} className="spin" /> : <Search size={15} />} {"Search"}
|
||||||
|
</button>
|
||||||
|
<button className="btn icon-text" type="button" onClick={refresh} disabled={busy}>
|
||||||
|
<RefreshCw size={15} className={busy ? "spin" : ""} /> {"Refresh"}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
</QueryPanel>
|
||||||
|
|
||||||
<div className="table-wrap">
|
<div className="table-wrap">
|
||||||
<table className="data-table">
|
<table className="data-table">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<th>{"User ID"}</th>
|
<SortableHeader label={"User ID"} sortKey="user_id" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
|
||||||
<th>{"Account"}</th>
|
<SortableHeader label={"Account"} sortKey="username" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
|
||||||
<th>{"Storage used"}</th>
|
<SortableHeader label={"Storage used"} sortKey="bytes" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
|
||||||
<th>{"Files"}</th>
|
<SortableHeader label={"Files"} sortKey="files" activeKey={sortKey} desc={sortDesc} onSort={toggleSort} />
|
||||||
</tr>
|
</tr>
|
||||||
</thead>
|
</thead>
|
||||||
<tbody>
|
<tbody>
|
||||||
|
|
@ -113,6 +169,330 @@ export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export function StoragePage({ navigate: _navigate }: { navigate: Navigate }) {
|
||||||
|
const [tab, setTab] = useState<"overview" | "limits">("overview");
|
||||||
|
return (
|
||||||
|
<PageFrame title={"Storage"} eyebrow={"Media / Storage usage"}>
|
||||||
|
<div className="tab-bar" role="tablist" aria-label={"Storage sections"}>
|
||||||
|
<button className={`tab-btn ${tab === "overview" ? "active" : ""}`} type="button" role="tab" aria-selected={tab === "overview"} onClick={() => setTab("overview")}>
|
||||||
|
{"Overview"}
|
||||||
|
</button>
|
||||||
|
<button className={`tab-btn ${tab === "limits" ? "active" : ""}`} type="button" role="tab" aria-selected={tab === "limits"} onClick={() => setTab("limits")}>
|
||||||
|
{"Limits & Retention"}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{tab === "overview" ? <StorageOverviewTab /> : <LimitsRetentionSection />}
|
||||||
</PageFrame>
|
</PageFrame>
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// --- Limits & Retention --------------------------------------------------
|
||||||
|
//
|
||||||
|
// Edits a handful of TELESRV_STORAGE_* keys through the existing generic
|
||||||
|
// .env editor endpoints (GET /api/server/env, POST
|
||||||
|
// /api/actions/update-server-env -- see cmd/telesrv-admin/serversettings.go
|
||||||
|
// and ServerSettingsPage.tsx's EnvSection, which this mirrors for its
|
||||||
|
// save/reload flow), but with friendly units instead of raw key=value text:
|
||||||
|
// GB inputs for byte budgets, a mode dropdown + day count for retention.
|
||||||
|
|
||||||
|
const STORAGE_ENV_KEYS = {
|
||||||
|
maxTotal: "TELESRV_STORAGE_MAX_TOTAL_BYTES",
|
||||||
|
minFree: "TELESRV_STORAGE_MIN_FREE_BYTES",
|
||||||
|
maxUploadFile: "TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES",
|
||||||
|
retentionMode: "TELESRV_STORAGE_RETENTION_MODE",
|
||||||
|
retentionMaxAge: "TELESRV_STORAGE_RETENTION_MAX_AGE"
|
||||||
|
} as const;
|
||||||
|
|
||||||
|
// Mirrors internal/app/files.MaxUploadPartBytes * MaxUploadParts -- the
|
||||||
|
// protocol's own upload-part-count ceiling that TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES
|
||||||
|
// can never legally exceed (internal/config validates this server-side too;
|
||||||
|
// this is just an early, friendlier warning in the form).
|
||||||
|
const PROTOCOL_UPLOAD_CEILING_BYTES = 524288 * 8000;
|
||||||
|
|
||||||
|
const BYTE_UNITS: { label: string; bytes: number }[] = [
|
||||||
|
{ label: "MB", bytes: 1024 ** 2 },
|
||||||
|
{ label: "GB", bytes: 1024 ** 3 },
|
||||||
|
{ label: "TB", bytes: 1024 ** 4 }
|
||||||
|
];
|
||||||
|
|
||||||
|
function bestByteUnit(bytes: number): { label: string; bytes: number } {
|
||||||
|
for (let i = BYTE_UNITS.length - 1; i >= 0; i--) {
|
||||||
|
if (bytes >= BYTE_UNITS[i].bytes) return BYTE_UNITS[i];
|
||||||
|
}
|
||||||
|
return BYTE_UNITS[1]; // default to GB for small/zero values
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseDurationMinutes reads a Go-style duration string (e.g. "720h", "90m",
|
||||||
|
// "1h30m") and returns the total as minutes. Unrecognized/empty input is 0.
|
||||||
|
function parseDurationMinutes(value: string): number {
|
||||||
|
let totalSeconds = 0;
|
||||||
|
const re = /(\d+(?:\.\d+)?)\s*(h|m|s)/g;
|
||||||
|
let match: RegExpExecArray | null;
|
||||||
|
while ((match = re.exec(value)) !== null) {
|
||||||
|
const amount = parseFloat(match[1]);
|
||||||
|
const unit = match[2];
|
||||||
|
totalSeconds += unit === "h" ? amount * 3600 : unit === "m" ? amount * 60 : amount;
|
||||||
|
}
|
||||||
|
return totalSeconds / 60;
|
||||||
|
}
|
||||||
|
|
||||||
|
const DURATION_UNITS: { label: string; minutes: number }[] = [
|
||||||
|
{ label: "Minutes", minutes: 1 },
|
||||||
|
{ label: "Hours", minutes: 60 },
|
||||||
|
{ label: "Days", minutes: 1440 }
|
||||||
|
];
|
||||||
|
|
||||||
|
function bestDurationUnit(minutes: number): { label: string; minutes: number } {
|
||||||
|
for (let i = DURATION_UNITS.length - 1; i >= 0; i--) {
|
||||||
|
if (minutes >= DURATION_UNITS[i].minutes) return DURATION_UNITS[i];
|
||||||
|
}
|
||||||
|
return DURATION_UNITS[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
// DurationField edits one duration env value (stored as total minutes) as a
|
||||||
|
// friendly "amount + unit" pair, letting TTL be set in minutes, hours, or
|
||||||
|
// days rather than being locked to whole days.
|
||||||
|
function DurationField({
|
||||||
|
label,
|
||||||
|
help,
|
||||||
|
minutes,
|
||||||
|
disabled,
|
||||||
|
onChange
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
help: string;
|
||||||
|
minutes: string;
|
||||||
|
disabled?: boolean;
|
||||||
|
onChange: (minutes: string) => void;
|
||||||
|
}) {
|
||||||
|
const totalMinutes = Number(minutes || "0");
|
||||||
|
const [unitLabel, setUnitLabel] = useState(() => bestDurationUnit(totalMinutes).label);
|
||||||
|
const unit = DURATION_UNITS.find((u) => u.label === unitLabel) ?? DURATION_UNITS[2];
|
||||||
|
const amount = totalMinutes > 0 ? totalMinutes / unit.minutes : NaN;
|
||||||
|
|
||||||
|
function handleAmountChange(raw: string) {
|
||||||
|
const parsed = Number(raw);
|
||||||
|
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
|
||||||
|
onChange("0");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onChange(String(Math.max(1, Math.round(parsed * unit.minutes))));
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<label className="duration-field">
|
||||||
|
<span>{label}</span>
|
||||||
|
<div style={{ display: "flex", gap: 8 }}>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min="0"
|
||||||
|
step="any"
|
||||||
|
value={Number.isNaN(amount) ? "" : amount}
|
||||||
|
disabled={disabled}
|
||||||
|
onChange={(event) => handleAmountChange(event.target.value)}
|
||||||
|
/>
|
||||||
|
<select value={unitLabel} disabled={disabled} onChange={(event) => setUnitLabel(event.target.value)} style={{ maxWidth: 100 }}>
|
||||||
|
{DURATION_UNITS.map((u) => (
|
||||||
|
<option key={u.label} value={u.label}>{u.label}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<span className="env-field-desc">{help}</span>
|
||||||
|
</label>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ByteSizeField edits one byte-count env value as a friendly
|
||||||
|
// "amount + unit" pair. bytes is the raw byte count as a string (what the
|
||||||
|
// backend stores); an empty/zero value displays as "Unlimited".
|
||||||
|
function ByteSizeField({
|
||||||
|
label,
|
||||||
|
help,
|
||||||
|
bytes,
|
||||||
|
onChange
|
||||||
|
}: {
|
||||||
|
label: string;
|
||||||
|
help: string;
|
||||||
|
bytes: string;
|
||||||
|
onChange: (bytes: string) => void;
|
||||||
|
}) {
|
||||||
|
const numericBytes = Number(bytes || "0");
|
||||||
|
const [unitLabel, setUnitLabel] = useState(() => bestByteUnit(numericBytes).label);
|
||||||
|
const unit = BYTE_UNITS.find((u) => u.label === unitLabel) ?? BYTE_UNITS[1];
|
||||||
|
const amount = numericBytes > 0 ? numericBytes / unit.bytes : NaN;
|
||||||
|
|
||||||
|
function handleAmountChange(raw: string) {
|
||||||
|
const parsed = Number(raw);
|
||||||
|
if (!raw.trim() || Number.isNaN(parsed) || parsed <= 0) {
|
||||||
|
onChange("0");
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onChange(String(Math.round(parsed * unit.bytes)));
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<label className="duration-field">
|
||||||
|
<span>{label}</span>
|
||||||
|
<div style={{ display: "flex", gap: 8 }}>
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
min="0"
|
||||||
|
step="any"
|
||||||
|
value={Number.isNaN(amount) ? "" : amount}
|
||||||
|
placeholder={"Unlimited"}
|
||||||
|
onChange={(event) => handleAmountChange(event.target.value)}
|
||||||
|
/>
|
||||||
|
<select value={unitLabel} onChange={(event) => setUnitLabel(event.target.value)} style={{ maxWidth: 90 }}>
|
||||||
|
{BYTE_UNITS.map((u) => (
|
||||||
|
<option key={u.label} value={u.label}>{u.label}</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
<span className="env-field-desc">{help}</span>
|
||||||
|
</label>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function LimitsRetentionSection() {
|
||||||
|
const [loaded, setLoaded] = useState(false);
|
||||||
|
const [error, setError] = useState("");
|
||||||
|
const [initial, setInitial] = useState<Record<string, string>>({});
|
||||||
|
const [maxTotalBytes, setMaxTotalBytes] = useState("0");
|
||||||
|
const [minFreeBytes, setMinFreeBytes] = useState("0");
|
||||||
|
const [maxUploadFileBytes, setMaxUploadFileBytes] = useState("0");
|
||||||
|
const [retentionMode, setRetentionMode] = useState("off");
|
||||||
|
const [retentionAgeMinutes, setRetentionAgeMinutes] = useState("43200");
|
||||||
|
|
||||||
|
async function load() {
|
||||||
|
setError("");
|
||||||
|
try {
|
||||||
|
const groups = await api.serverEnv();
|
||||||
|
const values: Record<string, string> = {};
|
||||||
|
for (const group of groups) {
|
||||||
|
for (const field of group.fields) {
|
||||||
|
values[field.key] = field.value || field.default_value || "";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setInitial(values);
|
||||||
|
setMaxTotalBytes(values[STORAGE_ENV_KEYS.maxTotal] || "0");
|
||||||
|
setMinFreeBytes(values[STORAGE_ENV_KEYS.minFree] || "0");
|
||||||
|
setMaxUploadFileBytes(values[STORAGE_ENV_KEYS.maxUploadFile] || "0");
|
||||||
|
const mode = (values[STORAGE_ENV_KEYS.retentionMode] || "off").trim().toLowerCase();
|
||||||
|
setRetentionMode(mode === "orphan" || mode === "hard" ? mode : "off");
|
||||||
|
const mins = parseDurationMinutes(values[STORAGE_ENV_KEYS.retentionMaxAge] || "720h");
|
||||||
|
setRetentionAgeMinutes(mins > 0 ? String(Math.max(1, Math.round(mins))) : "43200");
|
||||||
|
setLoaded(true);
|
||||||
|
} catch (err) {
|
||||||
|
setError(errorMessage(err));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
useEffect(() => { void load(); }, []);
|
||||||
|
|
||||||
|
// Only the age is re-encoded from its friendly "days" input back to a Go
|
||||||
|
// duration string; mode/off keeps whatever TELESRV_STORAGE_RETENTION_MAX_AGE
|
||||||
|
// already was rather than clobbering it with a throwaway placeholder.
|
||||||
|
const pendingValues = useMemo(() => {
|
||||||
|
const next: Record<string, string> = {
|
||||||
|
[STORAGE_ENV_KEYS.maxTotal]: maxTotalBytes || "0",
|
||||||
|
[STORAGE_ENV_KEYS.minFree]: minFreeBytes || "0",
|
||||||
|
[STORAGE_ENV_KEYS.maxUploadFile]: maxUploadFileBytes || "0",
|
||||||
|
[STORAGE_ENV_KEYS.retentionMode]: retentionMode,
|
||||||
|
[STORAGE_ENV_KEYS.retentionMaxAge]: retentionMode === "off"
|
||||||
|
? (initial[STORAGE_ENV_KEYS.retentionMaxAge] || "720h")
|
||||||
|
: `${Math.max(1, Math.round(Number(retentionAgeMinutes || "0")))}m`
|
||||||
|
};
|
||||||
|
const changed: Record<string, string> = {};
|
||||||
|
for (const [key, value] of Object.entries(next)) {
|
||||||
|
if ((initial[key] ?? "") !== value) changed[key] = value;
|
||||||
|
}
|
||||||
|
return changed;
|
||||||
|
}, [maxTotalBytes, minFreeBytes, maxUploadFileBytes, retentionMode, retentionAgeMinutes, initial]);
|
||||||
|
|
||||||
|
const hasChanges = Object.keys(pendingValues).length > 0;
|
||||||
|
const uploadCeilingExceeded = Number(maxUploadFileBytes || "0") > PROTOCOL_UPLOAD_CEILING_BYTES;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<section className="section-block">
|
||||||
|
<SectionHead
|
||||||
|
title={"Limits & Retention"}
|
||||||
|
text={"Server-wide storage budget, per-file upload cap, and automatic media cleanup. Saved to .env -- takes effect on the next Restart/Update."}
|
||||||
|
/>
|
||||||
|
{error && <Alert>{error}</Alert>}
|
||||||
|
{!loaded ? (
|
||||||
|
<p style={{ color: "var(--muted)" }}>{"Loading current settings..."}</p>
|
||||||
|
) : (
|
||||||
|
<div className="card-body">
|
||||||
|
<div className="attr-block">
|
||||||
|
<ByteSizeField
|
||||||
|
label={"Max total storage budget"}
|
||||||
|
help={"Reject new uploads once total tracked blob bytes would exceed this. Empty/0 = unlimited."}
|
||||||
|
bytes={maxTotalBytes}
|
||||||
|
onChange={setMaxTotalBytes}
|
||||||
|
/>
|
||||||
|
<ByteSizeField
|
||||||
|
label={"Min free space guard"}
|
||||||
|
help={"localfs backend only: reject new uploads once real free disk space falls below this. Empty/0 disables the check."}
|
||||||
|
bytes={minFreeBytes}
|
||||||
|
onChange={setMinFreeBytes}
|
||||||
|
/>
|
||||||
|
<ByteSizeField
|
||||||
|
label={"Max single file size"}
|
||||||
|
help={"Reject a single upload once its total assembled size exceeds this. Empty/0 = unlimited, bounded only by the protocol's own ~4GB per-file ceiling."}
|
||||||
|
bytes={maxUploadFileBytes}
|
||||||
|
onChange={setMaxUploadFileBytes}
|
||||||
|
/>
|
||||||
|
{uploadCeilingExceeded && (
|
||||||
|
<Alert>{"This exceeds the protocol's own ~4GB upload ceiling and will be refused when the server restarts."}</Alert>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="attr-block" style={{ marginTop: "1.5em" }}>
|
||||||
|
<label className="duration-field">
|
||||||
|
<span>{"Retention mode"}</span>
|
||||||
|
<select value={retentionMode} onChange={(event) => setRetentionMode(event.target.value)}>
|
||||||
|
<option value="off">{"Off"}</option>
|
||||||
|
<option value="orphan">{"Delete once no longer used (safe)"}</option>
|
||||||
|
<option value="hard">{"Delete after a fixed time, even if still in use"}</option>
|
||||||
|
</select>
|
||||||
|
</label>
|
||||||
|
<p className="env-field-desc">
|
||||||
|
{retentionMode === "off" &&
|
||||||
|
"No storage sweep runs; nothing is auto-deleted. Storage usage is still tracked and shown above either way."}
|
||||||
|
{retentionMode === "orphan" &&
|
||||||
|
"Safe: a document or photo's file is deleted only once it is no longer referenced by any message, profile photo, or sticker set. Media still visible in a conversation is never touched, regardless of age."}
|
||||||
|
{retentionMode === "hard" &&
|
||||||
|
"Irreversible and aggressive: a document or photo's file bytes are deleted once old enough, REGARDLESS of whether a message still references it. Old media in active conversations will start showing as unavailable once purged -- only the file is removed, the message itself keeps rendering its placeholder (name, size, thumbnail)."}
|
||||||
|
</p>
|
||||||
|
<DurationField
|
||||||
|
label={"Retention age"}
|
||||||
|
help={
|
||||||
|
retentionMode === "hard"
|
||||||
|
? "How old the media itself must be, counted from when it was uploaded, before its bytes are purged."
|
||||||
|
: "How long a document or photo must have had zero references before its file is deleted."
|
||||||
|
}
|
||||||
|
minutes={retentionAgeMinutes}
|
||||||
|
disabled={retentionMode === "off"}
|
||||||
|
onChange={setRetentionAgeMinutes}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="gift-table-actions env-save-row">
|
||||||
|
<ActionButton
|
||||||
|
tone="warn"
|
||||||
|
label={"Save limits & retention settings"}
|
||||||
|
path="/api/actions/update-server-env"
|
||||||
|
payload={() => ({ values: pendingValues })}
|
||||||
|
disabled={!hasChanges || uploadCeilingExceeded}
|
||||||
|
onDone={() => void load()}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</section>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -779,6 +779,27 @@ textarea:focus {
|
||||||
font-weight: 800;
|
font-weight: 800;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.sort-header {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 5px;
|
||||||
|
padding: 0;
|
||||||
|
border: 0;
|
||||||
|
background: transparent;
|
||||||
|
color: inherit;
|
||||||
|
font: inherit;
|
||||||
|
font-weight: 800;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.sort-header:hover {
|
||||||
|
color: var(--text);
|
||||||
|
}
|
||||||
|
|
||||||
|
.sort-header-idle {
|
||||||
|
opacity: 0.45;
|
||||||
|
}
|
||||||
|
|
||||||
.data-table tbody tr:hover {
|
.data-table tbody tr:hover {
|
||||||
background: var(--panel-subtle);
|
background: var(--panel-subtle);
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -932,6 +932,7 @@ func run(logger *zap.Logger) error {
|
||||||
filesapp.WithUploadPartBackend(localBlobFS),
|
filesapp.WithUploadPartBackend(localBlobFS),
|
||||||
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
|
filesapp.WithAdditionalBlobBackend(additionalBlobBackend),
|
||||||
filesapp.WithSpaceGuard(spaceGuard),
|
filesapp.WithSpaceGuard(spaceGuard),
|
||||||
|
filesapp.WithMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes),
|
||||||
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
filesapp.WithMapboxMapTiles(cfg.MapboxToken, cfg.MapTileCacheDir),
|
||||||
externalMediaOption(cfg),
|
externalMediaOption(cfg),
|
||||||
webPagePreviewOption(cfg),
|
webPagePreviewOption(cfg),
|
||||||
|
|
@ -1063,11 +1064,7 @@ func run(logger *zap.Logger) error {
|
||||||
readModelVersionStore,
|
readModelVersionStore,
|
||||||
cfg.UserProjectionFactCacheMaxEntries,
|
cfg.UserProjectionFactCacheMaxEntries,
|
||||||
)
|
)
|
||||||
storageRetentionMaxAge := cfg.StorageRetentionMaxAge
|
retentionWorker := maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
|
||||||
if !cfg.StorageRetentionEnable {
|
|
||||||
storageRetentionMaxAge = 0
|
|
||||||
}
|
|
||||||
go maintenance.NewRetentionWorker(dispatchOutboxStore, tempAuthKeyStore, logger.Named("maintenance").Named("retention"),
|
|
||||||
cfg.UpdateEventRetention,
|
cfg.UpdateEventRetention,
|
||||||
cfg.RetentionInterval,
|
cfg.RetentionInterval,
|
||||||
cfg.RetentionBatch,
|
cfg.RetentionBatch,
|
||||||
|
|
@ -1080,9 +1077,17 @@ func run(logger *zap.Logger) error {
|
||||||
WithModerationRetention(moderationReportStore).
|
WithModerationRetention(moderationReportStore).
|
||||||
WithUserUpdateRetention(updateEventStore).
|
WithUserUpdateRetention(updateEventStore).
|
||||||
WithChannelUpdateRetention(channelStore).
|
WithChannelUpdateRetention(channelStore).
|
||||||
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention).
|
WithOrphanAuthKeyRetention(authKeyStore, activeSessions, cfg.OrphanAuthKeyRetention)
|
||||||
WithOrphanedMediaRetention(filesService, storageRetentionMaxAge).
|
// TELESRV_STORAGE_RETENTION_MODE is a single 3-way switch: at most one of
|
||||||
Run(ctx)
|
// the orphan-only (safe) and hard (age-based, ignores live references)
|
||||||
|
// media sweeps is ever wired in, matching "off"/"orphan"/"hard".
|
||||||
|
switch cfg.StorageRetentionMode {
|
||||||
|
case config.StorageRetentionModeOrphan:
|
||||||
|
retentionWorker = retentionWorker.WithOrphanedMediaRetention(filesService, cfg.StorageRetentionMaxAge)
|
||||||
|
case config.StorageRetentionModeHard:
|
||||||
|
retentionWorker = retentionWorker.WithHardMediaRetention(filesService, cfg.StorageRetentionMaxAge)
|
||||||
|
}
|
||||||
|
go retentionWorker.Run(ctx)
|
||||||
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
|
go filesapp.NewUploadPartGCWorker(filesService, logger.Named("files").Named("upload_gc"),
|
||||||
cfg.UploadPartTTL,
|
cfg.UploadPartTTL,
|
||||||
cfg.UploadPartGCInterval,
|
cfg.UploadPartGCInterval,
|
||||||
|
|
|
||||||
|
|
@ -87,8 +87,11 @@ func (c *stickerSetNegativeCache) delete(refs ...domain.StickerSetRef) {
|
||||||
// 每个 chunk 一次 GetFileBlob 的 PG 往返(一个文件按 ≤512KB/1MB 分多次 getFile,热门贴纸/
|
// 每个 chunk 一次 GetFileBlob 的 PG 往返(一个文件按 ≤512KB/1MB 分多次 getFile,热门贴纸/
|
||||||
// reaction/头像更被大量用户重复拉)。
|
// reaction/头像更被大量用户重复拉)。
|
||||||
//
|
//
|
||||||
// FileBlob 元数据小(约百字节)且内容不可变:location_key 一旦写入即固定指向同一 object_key,
|
// FileBlob 元数据小(约百字节),新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,
|
||||||
// 新建 blob 用随机 id 生成 location_key 不会与已缓存项冲突,故只读填充、无需失效。
|
// 故写入路径只读填充、无需失效。但一个 location_key 的 file_blobs 行可以被存储 retention
|
||||||
|
// 清理主动删除(尤其是 hard 模式:不等 orphaned,仍被引用的媒体到期也会被清理字节),这种情况
|
||||||
|
// 下必须调用 delete 使该 key 失效,否则缓存会继续认为它存在,导致后续下载走到已被删除的字节
|
||||||
|
// 而不是优雅返回 LOCATION_INVALID。
|
||||||
type blobMetaCache struct {
|
type blobMetaCache struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
cap int
|
cap int
|
||||||
|
|
@ -141,6 +144,17 @@ func (c *blobMetaCache) put(key string, blob domain.FileBlob) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// delete 使一个 location_key 的缓存元数据立即失效(如果存在)。见上方类型注释:
|
||||||
|
// 存储 retention 清理主动删掉该 location_key 的 file_blobs 行后必须调用。
|
||||||
|
func (c *blobMetaCache) delete(key string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if el, ok := c.m[key]; ok {
|
||||||
|
c.ll.Remove(el)
|
||||||
|
delete(c.m, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
|
// blobBytesCache 是 object_key → 小 blob 全量字节的 LRU。Sticker / reaction /
|
||||||
// 缩略图通常只有几 KB 到几十 KB,缓存全量内容可以避开点击历史时的本地磁盘冷读抖动;
|
// 缩略图通常只有几 KB 到几十 KB,缓存全量内容可以避开点击历史时的本地磁盘冷读抖动;
|
||||||
// 大媒体仍由 BlobBackend.GetRange 分段读取,避免把大文件放进内存。
|
// 大媒体仍由 BlobBackend.GetRange 分段读取,避免把大文件放进内存。
|
||||||
|
|
@ -220,6 +234,21 @@ func (c *blobBytesCache) put(key string, bytes []byte) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// delete evicts a cached object_key's full bytes (if present) and reclaims
|
||||||
|
// its budget. Called after a retention sweep physically deletes the object
|
||||||
|
// from its backend, so a later GetFile can't keep serving bytes that no
|
||||||
|
// longer exist on disk/S3.
|
||||||
|
func (c *blobBytesCache) delete(key string) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
if el, ok := c.m[key]; ok {
|
||||||
|
entry := el.Value.(*blobBytesEntry)
|
||||||
|
c.ll.Remove(el)
|
||||||
|
delete(c.m, key)
|
||||||
|
c.used -= entry.size
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type stickerSetFullCache struct {
|
type stickerSetFullCache struct {
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
byID map[int64]stickerSetFullEntry
|
byID map[int64]stickerSetFullEntry
|
||||||
|
|
|
||||||
|
|
@ -24,6 +24,17 @@ type mediaRetentionStore interface {
|
||||||
// OrphanDocumentIfUnreferenced is the immediate (no grace period)
|
// OrphanDocumentIfUnreferenced is the immediate (no grace period)
|
||||||
// counterpart to the age-based sweep above -- see its doc comment.
|
// counterpart to the age-based sweep above -- see its doc comment.
|
||||||
OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)
|
OrphanDocumentIfUnreferenced(ctx context.Context, id int64) (bool, error)
|
||||||
|
|
||||||
|
// -- "hard" retention mode (TELESRV_STORAGE_RETENTION_MODE=hard) --
|
||||||
|
// Candidate selection ignores media_references entirely: a document/
|
||||||
|
// photo still referenced by a live message is exactly as eligible as an
|
||||||
|
// orphaned one once it's old enough. The delete methods physically
|
||||||
|
// remove only the file_blobs row(s)/bytes, never the document/photo
|
||||||
|
// metadata row -- see DeleteFileBlobsForDocument's doc comment.
|
||||||
|
ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||||
|
ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error)
|
||||||
|
DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||||
|
DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
// DeleteOrphanedOlderThan implements maintenance.OrphanedMediaRetentionStore:
|
||||||
|
|
@ -68,6 +79,63 @@ func (s *Service) DeleteOrphanedOlderThan(ctx context.Context, cutoff time.Time,
|
||||||
return deleted, nil
|
return deleted, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DeleteBlobBytesForMediaOlderThan implements
|
||||||
|
// maintenance.HardMediaRetentionStore ("hard" retention mode): for
|
||||||
|
// documents/photos whose upload/created_at is older than cutoff, physically
|
||||||
|
// deletes their blob bytes (main body + thumbnail/rendition variants) from
|
||||||
|
// the backend and removes their file_blobs rows -- REGARDLESS of whether a
|
||||||
|
// live message/profile-photo/sticker-set still references them. It
|
||||||
|
// deliberately never touches the documents/photos metadata row itself: a
|
||||||
|
// message must still be able to render "here was a photo/document"
|
||||||
|
// (dimensions, mime type, filename) after its bytes are gone, rather than
|
||||||
|
// the message breaking outright. A subsequent upload.getFile for the same
|
||||||
|
// location key finds no file_blobs row and returns LOCATION_INVALID, which
|
||||||
|
// stock clients already render as a "media unavailable" placeholder.
|
||||||
|
func (s *Service) DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error) {
|
||||||
|
store, ok := s.media.(mediaRetentionStore)
|
||||||
|
if !ok || limit <= 0 {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
purged := 0
|
||||||
|
docIDs, err := store.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, limit)
|
||||||
|
if err != nil {
|
||||||
|
return purged, fmt.Errorf("list documents for hard retention: %w", err)
|
||||||
|
}
|
||||||
|
for _, id := range docIDs {
|
||||||
|
blobs, err := store.DeleteFileBlobsForDocument(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("hard-delete document blob bytes failed", zap.Int64("document_id", id), zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(blobs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
purged++
|
||||||
|
}
|
||||||
|
remaining := limit - len(docIDs)
|
||||||
|
if remaining <= 0 {
|
||||||
|
return purged, nil
|
||||||
|
}
|
||||||
|
photoIDs, err := store.ListPhotoIDsForHardRetentionOlderThan(ctx, cutoff, remaining)
|
||||||
|
if err != nil {
|
||||||
|
return purged, fmt.Errorf("list photos for hard retention: %w", err)
|
||||||
|
}
|
||||||
|
for _, id := range photoIDs {
|
||||||
|
blobs, err := store.DeleteFileBlobsForPhoto(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
s.log.Warn("hard-delete photo blob bytes failed", zap.Int64("photo_id", id), zap.Error(err))
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(blobs) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
s.deleteOrphanedBlobs(ctx, store, blobs)
|
||||||
|
purged++
|
||||||
|
}
|
||||||
|
return purged, nil
|
||||||
|
}
|
||||||
|
|
||||||
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
|
// deleteDocumentNowIfUnreferenced is the immediate counterpart to the
|
||||||
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
|
// age-based sweep DeleteOrphanedOlderThan runs in the background: orphans
|
||||||
// id right now (skipping the grace period) and, only if that succeeds --
|
// id right now (skipping the grace period) and, only if that succeeds --
|
||||||
|
|
@ -106,6 +174,17 @@ func (s *Service) deleteDocumentNowIfUnreferenced(ctx context.Context, id int64)
|
||||||
// delete it from.
|
// delete it from.
|
||||||
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
|
func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionStore, blobs []domain.FileBlob) {
|
||||||
for _, b := range blobs {
|
for _, b := range blobs {
|
||||||
|
// The file_blobs row for this exact location_key is already gone
|
||||||
|
// (the caller deleted it in the same transaction that produced this
|
||||||
|
// blob list) -- so any cached "found" metadata for it is now wrong
|
||||||
|
// regardless of whether the underlying bytes turn out to still be
|
||||||
|
// shared by another row below. Without this, a hot GetFile path that
|
||||||
|
// had this location_key's metadata cached would keep trying to read
|
||||||
|
// bytes that may no longer exist (hard retention mode purges blobs
|
||||||
|
// for actively-referenced, potentially still-hot media), producing
|
||||||
|
// an internal error instead of the graceful LOCATION_INVALID a stock
|
||||||
|
// client knows how to render.
|
||||||
|
s.blobCache.delete(b.LocationKey)
|
||||||
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
|
refs, err := store.CountFileBlobRefs(ctx, string(b.Backend), b.ObjectKey)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
s.log.Warn("count file blob refs failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||||
|
|
@ -123,5 +202,6 @@ func (s *Service) deleteOrphanedBlobs(ctx context.Context, store mediaRetentionS
|
||||||
if err := backend.Delete(ctx, b.ObjectKey); err != nil {
|
if err := backend.Delete(ctx, b.ObjectKey); err != nil {
|
||||||
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
s.log.Warn("delete orphaned blob failed", zap.String("object_key", b.ObjectKey), zap.Error(err))
|
||||||
}
|
}
|
||||||
|
s.byteCache.delete(b.ObjectKey)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -72,6 +72,10 @@ type Service struct {
|
||||||
stickerSetNegCache *stickerSetNegativeCache
|
stickerSetNegCache *stickerSetNegativeCache
|
||||||
uploadQuota domain.UploadPartQuota
|
uploadQuota domain.UploadPartQuota
|
||||||
spaceGuard SpaceGuard
|
spaceGuard SpaceGuard
|
||||||
|
// maxUploadFileBytes caps a single upload's total assembled size (sum of
|
||||||
|
// all parts). <=0 means unlimited (still bounded by the protocol's own
|
||||||
|
// MaxUploadPartBytes*MaxUploadParts ceiling). See WithMaxUploadFileBytes.
|
||||||
|
maxUploadFileBytes int64
|
||||||
mapTiles *mapTileProxy
|
mapTiles *mapTileProxy
|
||||||
externalMedia *externalMediaFetcher
|
externalMedia *externalMediaFetcher
|
||||||
webpage *webpageFetcher
|
webpage *webpageFetcher
|
||||||
|
|
@ -143,6 +147,15 @@ func WithSpaceGuard(guard SpaceGuard) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WithMaxUploadFileBytes caps a single uploaded file's total assembled size
|
||||||
|
// (sum of all parts, not any one part) -- TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES.
|
||||||
|
// <=0 leaves it unlimited (still bounded by MaxUploadPartBytes*MaxUploadParts).
|
||||||
|
func WithMaxUploadFileBytes(maxBytes int64) Option {
|
||||||
|
return func(s *Service) {
|
||||||
|
s.maxUploadFileBytes = maxBytes
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithAdditionalBlobBackend registers a non-active blob backend purely for
|
// WithAdditionalBlobBackend registers a non-active blob backend purely for
|
||||||
// reading/deleting rows written while it used to be active. Register the
|
// reading/deleting rows written while it used to be active. Register the
|
||||||
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
|
// previous backend here whenever TELESRV_BLOB_BACKEND changes and the old
|
||||||
|
|
@ -753,6 +766,11 @@ func (s *Service) loadAndValidateUploadParts(ctx context.Context, ownerUserID, f
|
||||||
return nil, 0, domain.ErrFilePartsInvalid
|
return nil, 0, domain.ErrFilePartsInvalid
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Checked against the total assembled size, not any single part --
|
||||||
|
// MaxUploadPartBytes above already bounds each part individually.
|
||||||
|
if s.maxUploadFileBytes > 0 && total > s.maxUploadFileBytes {
|
||||||
|
return nil, 0, domain.ErrFileTooLarge
|
||||||
|
}
|
||||||
return parts, total, nil
|
return parts, total, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -171,6 +171,89 @@ func TestCreatePhotoFromUploadReceiptReplaysAfterPartCleanup(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestMaxUploadFileBytesRejectsOversizedAssembledDocument covers
|
||||||
|
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES enforcement (WithMaxUploadFileBytes):
|
||||||
|
// the check must fire against the TOTAL assembled size (sum of every part),
|
||||||
|
// not any single part -- each individual part here is well under the limit,
|
||||||
|
// only their sum exceeds it.
|
||||||
|
func TestMaxUploadFileBytesRejectsOversizedAssembledDocument(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
media := newFakeMediaStore()
|
||||||
|
local, err := NewLocalFS(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewLocalFS: %v", err)
|
||||||
|
}
|
||||||
|
const partSize = 1024
|
||||||
|
const maxUploadFileBytes = 2 * partSize // exactly 2 parts' worth; 3 parts must be rejected
|
||||||
|
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
|
||||||
|
|
||||||
|
parts := []string{
|
||||||
|
strings.Repeat("a", partSize),
|
||||||
|
strings.Repeat("b", partSize),
|
||||||
|
strings.Repeat("c", partSize),
|
||||||
|
}
|
||||||
|
for i, part := range parts {
|
||||||
|
if _, err := svc.SaveBigFilePart(ctx, 10, 300, i, len(parts), []byte(part)); err != nil {
|
||||||
|
t.Fatalf("SaveBigFilePart %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
_, err = svc.CreateDocumentFromUpload(ctx,
|
||||||
|
domain.UploadedFileRef{OwnerUserID: 10, FileID: 300, Parts: len(parts), Name: "big.bin", Big: true},
|
||||||
|
domain.DocumentSpec{MimeType: "application/octet-stream"},
|
||||||
|
)
|
||||||
|
if !errors.Is(err, domain.ErrFileTooLarge) {
|
||||||
|
t.Fatalf("CreateDocumentFromUpload over max size err = %v, want ErrFileTooLarge", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit ensures the check
|
||||||
|
// is an upper bound, not an off-by-one trap: a total exactly at the
|
||||||
|
// configured ceiling must still succeed.
|
||||||
|
func TestMaxUploadFileBytesAllowsAssembledSizeAtOrUnderLimit(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
media := newFakeMediaStore()
|
||||||
|
local, err := NewLocalFS(t.TempDir())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewLocalFS: %v", err)
|
||||||
|
}
|
||||||
|
const partSize = 1024
|
||||||
|
const maxUploadFileBytes = 2 * partSize
|
||||||
|
svc := NewService(media, local, 2, WithVideoThumbnailer(nil), WithMaxUploadFileBytes(maxUploadFileBytes))
|
||||||
|
|
||||||
|
parts := []string{strings.Repeat("a", partSize), strings.Repeat("b", partSize)}
|
||||||
|
for i, part := range parts {
|
||||||
|
if _, err := svc.SaveBigFilePart(ctx, 10, 301, i, len(parts), []byte(part)); err != nil {
|
||||||
|
t.Fatalf("SaveBigFilePart %d: %v", i, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
doc, err := svc.CreateDocumentFromUpload(ctx,
|
||||||
|
domain.UploadedFileRef{OwnerUserID: 10, FileID: 301, Parts: len(parts), Name: "exact.bin", Big: true},
|
||||||
|
domain.DocumentSpec{MimeType: "application/octet-stream"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CreateDocumentFromUpload at exact max size: %v", err)
|
||||||
|
}
|
||||||
|
if doc.Size != int64(maxUploadFileBytes) {
|
||||||
|
t.Fatalf("doc size = %d, want %d", doc.Size, maxUploadFileBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestMaxUploadFileBytesUnlimitedByDefault ensures leaving
|
||||||
|
// WithMaxUploadFileBytes unset (or 0) never rejects on size -- only the
|
||||||
|
// protocol's own MaxUploadPartBytes/MaxUploadParts ceiling still applies.
|
||||||
|
func TestMaxUploadFileBytesUnlimitedByDefault(t *testing.T) {
|
||||||
|
ctx := context.Background()
|
||||||
|
media := newFakeMediaStore()
|
||||||
|
svc, _ := newUploadPartTestService(t, media, domain.UploadPartQuota{})
|
||||||
|
file := domain.UploadedFileRef{OwnerUserID: 10, FileID: 302, Parts: 1, Name: "photo.jpg"}
|
||||||
|
if _, err := svc.SaveFilePart(ctx, file.OwnerUserID, file.FileID, 0, []byte(strings.Repeat("z", 4096))); err != nil {
|
||||||
|
t.Fatalf("SaveFilePart: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := svc.CreatePhotoFromUpload(ctx, file); err != nil {
|
||||||
|
t.Fatalf("CreatePhotoFromUpload with no configured max size: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
type countingUploadPartBackend struct {
|
type countingUploadPartBackend struct {
|
||||||
*LocalFS
|
*LocalFS
|
||||||
getUploadPartCalls int
|
getUploadPartCalls int
|
||||||
|
|
|
||||||
|
|
@ -19,9 +19,38 @@ type OrphanedMediaRetentionStore interface {
|
||||||
// how long a document/photo must have been orphaned before it's actually
|
// how long a document/photo must have been orphaned before it's actually
|
||||||
// deleted (not how old the media itself is) -- <=0 leaves the sweep
|
// deleted (not how old the media itself is) -- <=0 leaves the sweep
|
||||||
// disabled even if a store is provided, matching
|
// disabled even if a store is provided, matching
|
||||||
// TELESRV_STORAGE_RETENTION_ENABLE=false being the safe default.
|
// TELESRV_STORAGE_RETENTION_MODE=off/orphan-with-no-age being the safe
|
||||||
|
// default. Mutually exclusive with WithHardMediaRetention -- the resolved
|
||||||
|
// TELESRV_STORAGE_RETENTION_MODE is one of "off"/"orphan"/"hard", never
|
||||||
|
// both orphan and hard sweeps at once.
|
||||||
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
func (w *RetentionWorker) WithOrphanedMediaRetention(store OrphanedMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
||||||
w.orphanedMedia = store
|
w.orphanedMedia = store
|
||||||
w.orphanedMediaMaxAge = maxAge
|
w.orphanedMediaMaxAge = maxAge
|
||||||
return w
|
return w
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// HardMediaRetentionStore physically deletes a document/photo's blob bytes
|
||||||
|
// once the media itself (its upload/created time, not how long it has been
|
||||||
|
// orphaned) is older than the configured age, REGARDLESS of whether a live
|
||||||
|
// message/profile-photo/sticker-set still references it. Unlike
|
||||||
|
// OrphanedMediaRetentionStore, it must never delete the document/photo
|
||||||
|
// metadata row -- only the underlying file_blobs row(s) and, once confirming
|
||||||
|
// no other row still needs the same content-addressed object, the physical
|
||||||
|
// bytes. This keeps a message rendering its media placeholder (dimensions,
|
||||||
|
// mime type, filename) after the bytes are gone; a subsequent download
|
||||||
|
// resolves to LOCATION_INVALID instead of the message breaking outright.
|
||||||
|
type HardMediaRetentionStore interface {
|
||||||
|
DeleteBlobBytesForMediaOlderThan(ctx context.Context, cutoff time.Time, limit int) (int, error)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithHardMediaRetention enables the aggressive storage retention sweep
|
||||||
|
// (TELESRV_STORAGE_RETENTION_MODE=hard). maxAge is how old the media itself
|
||||||
|
// must be (its created_at, not an orphaned_at grace period) before its blob
|
||||||
|
// bytes are purged -- <=0 leaves the sweep disabled even if a store is
|
||||||
|
// provided. Mutually exclusive with WithOrphanedMediaRetention -- call at
|
||||||
|
// most one of the two, matching the config's single 3-way retention mode.
|
||||||
|
func (w *RetentionWorker) WithHardMediaRetention(store HardMediaRetentionStore, maxAge time.Duration) *RetentionWorker {
|
||||||
|
w.hardMedia = store
|
||||||
|
w.hardMediaMaxAge = maxAge
|
||||||
|
return w
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -95,6 +95,10 @@ const (
|
||||||
// 继续保留,在线漏推由正常 difference 路径补偿。
|
// 继续保留,在线漏推由正常 difference 路径补偿。
|
||||||
defaultOutboxPoisonRetention = time.Minute
|
defaultOutboxPoisonRetention = time.Minute
|
||||||
defaultOutboxPoisonInterval = 15 * time.Second
|
defaultOutboxPoisonInterval = 15 * time.Second
|
||||||
|
// minMediaRetentionInterval floors the derived media-sweep cadence below
|
||||||
|
// so a very short (e.g. test-only) TELESRV_STORAGE_RETENTION_MAX_AGE
|
||||||
|
// can't spin the sweep query in a tight loop.
|
||||||
|
minMediaRetentionInterval = 15 * time.Second
|
||||||
)
|
)
|
||||||
|
|
||||||
// RetentionWorker 周期性回收存储中的死数据。
|
// RetentionWorker 周期性回收存储中的死数据。
|
||||||
|
|
@ -119,6 +123,7 @@ type RetentionWorker struct {
|
||||||
activeAuthKeys ActiveRawAuthKeyProvider
|
activeAuthKeys ActiveRawAuthKeyProvider
|
||||||
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
activeAuthKeyHeartbeat ActiveAuthKeyHeartbeatStore
|
||||||
orphanedMedia OrphanedMediaRetentionStore
|
orphanedMedia OrphanedMediaRetentionStore
|
||||||
|
hardMedia HardMediaRetentionStore
|
||||||
logger *zap.Logger
|
logger *zap.Logger
|
||||||
retention time.Duration
|
retention time.Duration
|
||||||
botAPIRetention time.Duration
|
botAPIRetention time.Duration
|
||||||
|
|
@ -128,6 +133,7 @@ type RetentionWorker struct {
|
||||||
outboxPoisonRetention time.Duration
|
outboxPoisonRetention time.Duration
|
||||||
outboxPoisonInterval time.Duration
|
outboxPoisonInterval time.Duration
|
||||||
orphanedMediaMaxAge time.Duration
|
orphanedMediaMaxAge time.Duration
|
||||||
|
hardMediaMaxAge time.Duration
|
||||||
interval time.Duration
|
interval time.Duration
|
||||||
batch int
|
batch int
|
||||||
}
|
}
|
||||||
|
|
@ -259,6 +265,15 @@ func (w *RetentionWorker) Run(ctx context.Context) {
|
||||||
heartbeatC = heartbeatTicker.C
|
heartbeatC = heartbeatTicker.C
|
||||||
defer heartbeatTicker.Stop()
|
defer heartbeatTicker.Stop()
|
||||||
}
|
}
|
||||||
|
var (
|
||||||
|
mediaTicker *time.Ticker
|
||||||
|
mediaC <-chan time.Time
|
||||||
|
)
|
||||||
|
if interval := w.mediaRetentionInterval(); interval > 0 {
|
||||||
|
mediaTicker = time.NewTicker(interval)
|
||||||
|
mediaC = mediaTicker.C
|
||||||
|
defer mediaTicker.Stop()
|
||||||
|
}
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-ctx.Done():
|
case <-ctx.Done():
|
||||||
|
|
@ -269,6 +284,8 @@ func (w *RetentionWorker) Run(ctx context.Context) {
|
||||||
w.runOutboxPoisonOnce(ctx)
|
w.runOutboxPoisonOnce(ctx)
|
||||||
case <-heartbeatC:
|
case <-heartbeatC:
|
||||||
w.heartbeatActiveAuthKeys(ctx)
|
w.heartbeatActiveAuthKeys(ctx)
|
||||||
|
case <-mediaC:
|
||||||
|
w.runMediaRetentionOnce(ctx)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
@ -276,6 +293,63 @@ func (w *RetentionWorker) Run(ctx context.Context) {
|
||||||
func (w *RetentionWorker) runOnce(ctx context.Context) {
|
func (w *RetentionWorker) runOnce(ctx context.Context) {
|
||||||
w.runOutboxPoisonOnce(ctx)
|
w.runOutboxPoisonOnce(ctx)
|
||||||
w.runRetentionOnce(ctx)
|
w.runRetentionOnce(ctx)
|
||||||
|
w.runMediaRetentionOnce(ctx)
|
||||||
|
}
|
||||||
|
|
||||||
|
// mediaRetentionInterval derives how often the storage media sweep
|
||||||
|
// (orphan/hard) runs, independent of the shared housekeeping w.interval.
|
||||||
|
// A short TELESRV_STORAGE_RETENTION_MAX_AGE (e.g. 30m) configured under a
|
||||||
|
// much longer TELESRV_RETENTION_INTERVAL (default 1h) would otherwise let
|
||||||
|
// media sit for up to age+interval past its cutoff before actually being
|
||||||
|
// swept -- capping the sweep cadence at the retention age itself bounds
|
||||||
|
// that worst case to at most 2x the configured age instead.
|
||||||
|
func (w *RetentionWorker) mediaRetentionInterval() time.Duration {
|
||||||
|
var maxAge time.Duration
|
||||||
|
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||||
|
maxAge = w.orphanedMediaMaxAge
|
||||||
|
}
|
||||||
|
if w.hardMedia != nil && w.hardMediaMaxAge > 0 && (maxAge == 0 || w.hardMediaMaxAge < maxAge) {
|
||||||
|
maxAge = w.hardMediaMaxAge
|
||||||
|
}
|
||||||
|
if maxAge <= 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
interval := w.interval
|
||||||
|
if interval <= 0 || maxAge < interval {
|
||||||
|
interval = maxAge
|
||||||
|
}
|
||||||
|
if interval < minMediaRetentionInterval {
|
||||||
|
interval = minMediaRetentionInterval
|
||||||
|
}
|
||||||
|
return interval
|
||||||
|
}
|
||||||
|
|
||||||
|
func (w *RetentionWorker) runMediaRetentionOnce(ctx context.Context) {
|
||||||
|
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
||||||
|
// Only ever touches documents/photos already marked orphaned (no live
|
||||||
|
// message/profile-photo/sticker-set reference remains) -- media still
|
||||||
|
// visible in a conversation is never a candidate, regardless of age.
|
||||||
|
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
||||||
|
} else if mediaDeleted > 0 {
|
||||||
|
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if w.hardMedia != nil && w.hardMediaMaxAge > 0 {
|
||||||
|
// "Hard" mode: purges blob bytes for documents/photos older than
|
||||||
|
// hardMediaMaxAge regardless of whether a live reference remains --
|
||||||
|
// the store is required to keep the document/photo metadata row
|
||||||
|
// intact, only removing file_blobs rows/bytes, so a message still
|
||||||
|
// renders its media placeholder (LOCATION_INVALID on download)
|
||||||
|
// instead of breaking outright.
|
||||||
|
hardDeleted, err := w.hardMedia.DeleteBlobBytesForMediaOlderThan(ctx, time.Now().Add(-w.hardMediaMaxAge), w.batch)
|
||||||
|
if err != nil {
|
||||||
|
w.logger.Warn("hard media storage retention sweep failed", zap.Error(err))
|
||||||
|
} else if hardDeleted > 0 {
|
||||||
|
w.logger.Info("hard media storage retention sweep complete", zap.Int("deleted", hardDeleted))
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
|
func (w *RetentionWorker) runOutboxPoisonOnce(ctx context.Context) {
|
||||||
|
|
@ -414,17 +488,6 @@ func (w *RetentionWorker) runRetentionOnce(ctx context.Context) {
|
||||||
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
|
w.logger.Info("expired channel_update_events contiguous-prefix cleanup complete", zap.Int("deleted", channelDeleted))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if w.orphanedMedia != nil && w.orphanedMediaMaxAge > 0 {
|
|
||||||
// Only ever touches documents/photos already marked orphaned (no live
|
|
||||||
// message/profile-photo/sticker-set reference remains) -- media still
|
|
||||||
// visible in a conversation is never a candidate, regardless of age.
|
|
||||||
mediaDeleted, err := w.orphanedMedia.DeleteOrphanedOlderThan(ctx, time.Now().Add(-w.orphanedMediaMaxAge), w.batch)
|
|
||||||
if err != nil {
|
|
||||||
w.logger.Warn("orphaned media storage retention sweep failed", zap.Error(err))
|
|
||||||
} else if mediaDeleted > 0 {
|
|
||||||
w.logger.Info("orphaned media storage retention sweep complete", zap.Int("deleted", mediaDeleted))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {
|
func (w *RetentionWorker) orphanHeartbeatInterval() time.Duration {
|
||||||
|
|
|
||||||
|
|
@ -393,3 +393,67 @@ func TestRetentionWorkerSkipsOrphanDeleteWhenHeartbeatFails(t *testing.T) {
|
||||||
t.Fatalf("heartbeat failure signals = %+v", entries)
|
t.Fatalf("heartbeat failure signals = %+v", entries)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeHardMediaRetention struct {
|
||||||
|
calls int
|
||||||
|
cutoff time.Time
|
||||||
|
limit int
|
||||||
|
deleted int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *fakeHardMediaRetention) DeleteBlobBytesForMediaOlderThan(_ context.Context, cutoff time.Time, limit int) (int, error) {
|
||||||
|
f.calls++
|
||||||
|
f.cutoff = cutoff
|
||||||
|
f.limit = limit
|
||||||
|
return f.deleted, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge guards the fix for
|
||||||
|
// a real gotcha reported live: with a short storage retention age (e.g. 30m)
|
||||||
|
// under the default 1h TELESRV_RETENTION_INTERVAL housekeeping cadence,
|
||||||
|
// media eligible for hard/orphan deletion used to sit for up to age+interval
|
||||||
|
// before the shared ticker actually got around to sweeping it (a 30m age
|
||||||
|
// could take up to 1h30m to actually disappear). The sweep now runs on its
|
||||||
|
// own ticker capped at the configured age, so it never lags by more than
|
||||||
|
// roughly one age-window.
|
||||||
|
func TestRetentionWorkerMediaSweepIntervalNeverExceedsMaxAge(t *testing.T) {
|
||||||
|
hard := &fakeHardMediaRetention{}
|
||||||
|
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
|
||||||
|
WithHardMediaRetention(hard, 30*time.Minute)
|
||||||
|
|
||||||
|
if got := w.mediaRetentionInterval(); got != 30*time.Minute {
|
||||||
|
t.Fatalf("media retention interval = %v, want min(interval=1h, maxAge=30m) = 30m", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetentionWorkerMediaSweepIntervalFloorsAtMinimum(t *testing.T) {
|
||||||
|
hard := &fakeHardMediaRetention{}
|
||||||
|
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
|
||||||
|
WithHardMediaRetention(hard, time.Second)
|
||||||
|
|
||||||
|
if got := w.mediaRetentionInterval(); got != minMediaRetentionInterval {
|
||||||
|
t.Fatalf("media retention interval = %v, want floor %v", got, minMediaRetentionInterval)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRetentionWorkerMediaSweepIntervalZeroWhenNoModeConfigured(t *testing.T) {
|
||||||
|
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50)
|
||||||
|
if got := w.mediaRetentionInterval(); got != 0 {
|
||||||
|
t.Fatalf("media retention interval = %v, want 0 (no separate ticker) when retention is off", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// TestRetentionWorkerRunsHardMediaSweepOnStartup confirms runOnce (called
|
||||||
|
// once immediately when Run starts, matching every other retention check)
|
||||||
|
// still fires the media sweep too, not just the new dedicated ticker.
|
||||||
|
func TestRetentionWorkerRunsHardMediaSweepOnStartup(t *testing.T) {
|
||||||
|
hard := &fakeHardMediaRetention{deleted: 4}
|
||||||
|
w := NewRetentionWorker(&fakeOutboxRetention{}, nil, zap.NewNop(), time.Hour, time.Hour, 50).
|
||||||
|
WithHardMediaRetention(hard, 30*time.Minute)
|
||||||
|
|
||||||
|
w.runOnce(context.Background())
|
||||||
|
|
||||||
|
if hard.calls != 1 {
|
||||||
|
t.Fatalf("hard media sweep calls = %d, want 1", hard.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -13,11 +13,20 @@ import (
|
||||||
|
|
||||||
"golang.org/x/text/language"
|
"golang.org/x/text/language"
|
||||||
|
|
||||||
|
"telesrv/internal/app/files"
|
||||||
"telesrv/internal/branding"
|
"telesrv/internal/branding"
|
||||||
"telesrv/internal/domain"
|
"telesrv/internal/domain"
|
||||||
"telesrv/internal/links"
|
"telesrv/internal/links"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// Storage retention modes for TELESRV_STORAGE_RETENTION_MODE. See
|
||||||
|
// Config.StorageRetentionMode for what each one does.
|
||||||
|
const (
|
||||||
|
StorageRetentionModeOff = "off"
|
||||||
|
StorageRetentionModeOrphan = "orphan"
|
||||||
|
StorageRetentionModeHard = "hard"
|
||||||
|
)
|
||||||
|
|
||||||
const (
|
const (
|
||||||
defaultConfigFile = ".env"
|
defaultConfigFile = ".env"
|
||||||
defaultCountryCode = "CN"
|
defaultCountryCode = "CN"
|
||||||
|
|
@ -311,19 +320,41 @@ type Config struct {
|
||||||
// the s3 backend (no OS-level free space concept), and usable as an
|
// the s3 backend (no OS-level free space concept), and usable as an
|
||||||
// optional soft budget cap on localfs too. <=0 disables this check.
|
// optional soft budget cap on localfs too. <=0 disables this check.
|
||||||
StorageMaxTotalBytes int64
|
StorageMaxTotalBytes int64
|
||||||
|
// StorageMaxUploadFileBytes caps the total assembled size of a single
|
||||||
|
// uploaded file (sum of all its parts, not any one part) -- rejected at
|
||||||
|
// upload-finalize time with a FILE_TOO_BIG rpc error. <=0 disables this
|
||||||
|
// check (the only remaining ceiling is then the protocol's own part-count
|
||||||
|
// limit, files.MaxUploadPartBytes*files.MaxUploadParts, ~4GB). Validated
|
||||||
|
// at config-load time to never exceed that protocol ceiling -- a larger
|
||||||
|
// value could never actually be reached, so it's refused as a
|
||||||
|
// startup-time misconfiguration rather than silently accepted as a no-op.
|
||||||
|
StorageMaxUploadFileBytes int64
|
||||||
// StorageUsageRefreshInterval is how often the cached free-space/budget
|
// StorageUsageRefreshInterval is how often the cached free-space/budget
|
||||||
// usage gauge refreshes; <=0 uses a 1 minute default.
|
// usage gauge refreshes; <=0 uses a 1 minute default.
|
||||||
StorageUsageRefreshInterval time.Duration
|
StorageUsageRefreshInterval time.Duration
|
||||||
// StorageRetentionEnable turns on the orphaned-media age sweep. Off by
|
// StorageRetentionMode selects the storage retention sweep behavior:
|
||||||
// default: orphaned media (no live message/profile-photo/sticker-set
|
// - "off" (default): no sweep at all.
|
||||||
// reference) is tracked and visible in the admin panel, but nothing is
|
// - "orphan": safe mode -- deletes a document/photo's blob only once it
|
||||||
// auto-deleted until the operator explicitly opts in.
|
// has had no live message/profile-photo/sticker-set reference for at
|
||||||
StorageRetentionEnable bool
|
// least StorageRetentionMaxAge. Never touches media still referenced,
|
||||||
// StorageRetentionMaxAge is how long a document/photo must have been
|
// regardless of age.
|
||||||
// orphaned (not how old the media itself is) before the sweep deletes
|
// - "hard": aggressive mode -- deletes a document/photo's blob bytes
|
||||||
// it. Never deletes media that still has a live reference, regardless
|
// once the media itself (its upload/created time, not how long it's
|
||||||
// of age. <=0 uses a 30 day default. The sweep itself runs on the
|
// been orphaned) is older than StorageRetentionMaxAge, REGARDLESS of
|
||||||
// shared RetentionInterval/RetentionBatch cadence alongside every other
|
// whether it's still referenced by a live message. Only the blob
|
||||||
|
// bytes are removed; the document/photo metadata row is kept so the
|
||||||
|
// message still renders (dimensions, mime type, filename) --
|
||||||
|
// subsequent downloads get LOCATION_INVALID ("media no longer
|
||||||
|
// available") instead of the message breaking outright. This is
|
||||||
|
// irreversible and can surprise users if misunderstood: old media in
|
||||||
|
// active conversations WILL disappear.
|
||||||
|
// Any other value fails config validation.
|
||||||
|
StorageRetentionMode string
|
||||||
|
// StorageRetentionMaxAge is the shared age threshold for both "orphan"
|
||||||
|
// and "hard" modes -- see StorageRetentionMode for how its meaning
|
||||||
|
// differs between them. Ignored when StorageRetentionMode is "off". <=0
|
||||||
|
// uses a 30 day default. The sweep itself runs on the shared
|
||||||
|
// RetentionInterval/RetentionBatch cadence alongside every other
|
||||||
// retention check (see maintenance.RetentionWorker).
|
// retention check (see maintenance.RetentionWorker).
|
||||||
StorageRetentionMaxAge time.Duration
|
StorageRetentionMaxAge time.Duration
|
||||||
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob)。
|
// StickerSeedDir 是 reaction / sticker 资源种子目录(导入到 documents/sticker_sets + blob)。
|
||||||
|
|
@ -945,8 +976,9 @@ func Load() (Config, error) {
|
||||||
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
|
StorageLowSpaceGuardEnable: envBoolOr("TELESRV_STORAGE_LOW_SPACE_GUARD_ENABLE", true),
|
||||||
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
|
StorageMinFreeBytes: envInt64Or("TELESRV_STORAGE_MIN_FREE_BYTES", 1<<30),
|
||||||
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
|
StorageMaxTotalBytes: envInt64Or("TELESRV_STORAGE_MAX_TOTAL_BYTES", 0),
|
||||||
|
StorageMaxUploadFileBytes: envInt64Or("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", 0),
|
||||||
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
StorageUsageRefreshInterval: envDurationOr("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL", time.Minute),
|
||||||
StorageRetentionEnable: envBoolOr("TELESRV_STORAGE_RETENTION_ENABLE", false),
|
StorageRetentionMode: strings.ToLower(strings.TrimSpace(envOr("TELESRV_STORAGE_RETENTION_MODE", StorageRetentionModeOff))),
|
||||||
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
StorageRetentionMaxAge: envDurationOr("TELESRV_STORAGE_RETENTION_MAX_AGE", 30*24*time.Hour),
|
||||||
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
StickerSeedDir: envOr("TELESRV_STICKER_SEED_DIR", "data/sticker-seed"),
|
||||||
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
StickerSeedMaxSets: envIntOr("TELESRV_STICKER_SEED_MAX_SETS", 300),
|
||||||
|
|
@ -1285,6 +1317,9 @@ func validateBlobStorageConfig(cfg Config) error {
|
||||||
if cfg.StorageMaxTotalBytes < 0 {
|
if cfg.StorageMaxTotalBytes < 0 {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
||||||
}
|
}
|
||||||
|
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 {
|
if cfg.StorageLowSpaceGuardEnable && cfg.StorageUsageRefreshInterval <= 0 {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled")
|
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be positive when storage capacity guard is enabled")
|
||||||
}
|
}
|
||||||
|
|
@ -1474,14 +1509,39 @@ func validateStorageConfig(cfg Config) error {
|
||||||
if cfg.StorageMaxTotalBytes < 0 {
|
if cfg.StorageMaxTotalBytes < 0 {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
return fmt.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES must be non-negative")
|
||||||
}
|
}
|
||||||
|
if err := validateMaxUploadFileBytes(cfg.StorageMaxUploadFileBytes); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if cfg.StorageUsageRefreshInterval < 0 {
|
if cfg.StorageUsageRefreshInterval < 0 {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
|
return fmt.Errorf("TELESRV_STORAGE_USAGE_REFRESH_INTERVAL must be non-negative")
|
||||||
}
|
}
|
||||||
if cfg.StorageRetentionMaxAge < 0 {
|
if cfg.StorageRetentionMaxAge < 0 {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be non-negative")
|
||||||
}
|
}
|
||||||
if cfg.StorageRetentionEnable && cfg.StorageRetentionMaxAge <= 0 {
|
switch cfg.StorageRetentionMode {
|
||||||
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_ENABLE is true")
|
case StorageRetentionModeOff:
|
||||||
|
// no sweep; StorageRetentionMaxAge is ignored.
|
||||||
|
case StorageRetentionModeOrphan, StorageRetentionModeHard:
|
||||||
|
if cfg.StorageRetentionMaxAge <= 0 {
|
||||||
|
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MAX_AGE must be positive when TELESRV_STORAGE_RETENTION_MODE is %q", cfg.StorageRetentionMode)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("TELESRV_STORAGE_RETENTION_MODE must be \"off\", \"orphan\", or \"hard\", got %q", cfg.StorageRetentionMode)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// validateMaxUploadFileBytes checks TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: it
|
||||||
|
// must be non-negative, and if set (>0) must not exceed the protocol's own
|
||||||
|
// part-count upload ceiling -- a larger configured value could never actually
|
||||||
|
// be hit, so it's refused as a startup-time misconfiguration.
|
||||||
|
func validateMaxUploadFileBytes(v int64) error {
|
||||||
|
if v < 0 {
|
||||||
|
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES must be non-negative")
|
||||||
|
}
|
||||||
|
const protocolCeiling = int64(files.MaxUploadPartBytes) * int64(files.MaxUploadParts)
|
||||||
|
if v > protocolCeiling {
|
||||||
|
return fmt.Errorf("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES (%d) must not exceed the protocol upload ceiling of %d bytes (%d parts x %d bytes)", v, protocolCeiling, files.MaxUploadParts, files.MaxUploadPartBytes)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -1,6 +1,7 @@
|
||||||
package config
|
package config
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
@ -467,6 +468,101 @@ func TestLoadRejectsInvalidStorageCapacityConfig(t *testing.T) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLoadMaxUploadFileBytesDefaultsToUnlimited(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
cfg, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.StorageMaxUploadFileBytes != 0 {
|
||||||
|
t.Fatalf("expected default StorageMaxUploadFileBytes=0 (unlimited), got %d", cfg.StorageMaxUploadFileBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadAcceptsMaxUploadFileBytesAtProtocolCeiling(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
const ceiling = int64(524288) * 8000 // files.MaxUploadPartBytes * files.MaxUploadParts
|
||||||
|
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", fmt.Sprint(ceiling))
|
||||||
|
cfg, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.StorageMaxUploadFileBytes != ceiling {
|
||||||
|
t.Fatalf("expected StorageMaxUploadFileBytes=%d, got %d", ceiling, cfg.StorageMaxUploadFileBytes)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidMaxUploadFileBytes(t *testing.T) {
|
||||||
|
const ceiling = int64(524288) * 8000
|
||||||
|
for _, item := range []struct {
|
||||||
|
name string
|
||||||
|
value string
|
||||||
|
}{
|
||||||
|
{"negative", "-1"},
|
||||||
|
{"exceeds protocol ceiling", fmt.Sprint(ceiling + 1)},
|
||||||
|
} {
|
||||||
|
t.Run(item.name, func(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
t.Setenv("TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES", item.value)
|
||||||
|
if _, err := Load(); err == nil {
|
||||||
|
t.Fatalf("Load accepted invalid TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES=%s", item.value)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadStorageRetentionModeDefaultsToOff(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
cfg, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.StorageRetentionMode != StorageRetentionModeOff {
|
||||||
|
t.Fatalf("expected default StorageRetentionMode=%q, got %q", StorageRetentionModeOff, cfg.StorageRetentionMode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadStorageRetentionModeOrphanAndHard(t *testing.T) {
|
||||||
|
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
|
||||||
|
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "48h")
|
||||||
|
cfg, err := Load()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("Load: %v", err)
|
||||||
|
}
|
||||||
|
if cfg.StorageRetentionMode != mode {
|
||||||
|
t.Fatalf("expected StorageRetentionMode=%q, got %q", mode, cfg.StorageRetentionMode)
|
||||||
|
}
|
||||||
|
if cfg.StorageRetentionMaxAge != 48*time.Hour {
|
||||||
|
t.Fatalf("expected StorageRetentionMaxAge=48h, got %v", cfg.StorageRetentionMaxAge)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsInvalidStorageRetentionMode(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", "sometimes")
|
||||||
|
if _, err := Load(); err == nil {
|
||||||
|
t.Fatal("Load accepted an unknown TELESRV_STORAGE_RETENTION_MODE")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestLoadRejectsStorageRetentionModeWithoutMaxAge(t *testing.T) {
|
||||||
|
for _, mode := range []string{StorageRetentionModeOrphan, StorageRetentionModeHard} {
|
||||||
|
t.Run(mode, func(t *testing.T) {
|
||||||
|
disableDefaultConfigFile(t)
|
||||||
|
t.Setenv("TELESRV_STORAGE_RETENTION_MODE", mode)
|
||||||
|
t.Setenv("TELESRV_STORAGE_RETENTION_MAX_AGE", "0s")
|
||||||
|
if _, err := Load(); err == nil {
|
||||||
|
t.Fatalf("Load accepted TELESRV_STORAGE_RETENTION_MODE=%s with zero max age", mode)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestLoadUpdateServiceConfig(t *testing.T) {
|
func TestLoadUpdateServiceConfig(t *testing.T) {
|
||||||
disableDefaultConfigFile(t)
|
disableDefaultConfigFile(t)
|
||||||
t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/")
|
t.Setenv("TELESRV_UPDATE_PUBLIC_URL", "https://updates.example.test/root/")
|
||||||
|
|
|
||||||
|
|
@ -9,6 +9,11 @@ var (
|
||||||
ErrFilePartsInvalid = errors.New("file parts invalid")
|
ErrFilePartsInvalid = errors.New("file parts invalid")
|
||||||
ErrFilePartTooBig = errors.New("file part too big")
|
ErrFilePartTooBig = errors.New("file part too big")
|
||||||
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
|
ErrUploadQuotaExceeded = errors.New("upload quota exceeded")
|
||||||
|
// ErrFileTooLarge is returned when a file's total assembled size (sum of
|
||||||
|
// all its parts, not any single part) exceeds the configured
|
||||||
|
// TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES ceiling. Distinct from
|
||||||
|
// ErrFilePartTooBig, which is a single oversized part.
|
||||||
|
ErrFileTooLarge = errors.New("file too large")
|
||||||
ErrPhotoInvalid = errors.New("photo invalid")
|
ErrPhotoInvalid = errors.New("photo invalid")
|
||||||
ErrDocumentInvalid = errors.New("document invalid")
|
ErrDocumentInvalid = errors.New("document invalid")
|
||||||
// ErrStorageFull is returned when the configured low-space guard rejects a
|
// ErrStorageFull is returned when the configured low-space guard rejects a
|
||||||
|
|
|
||||||
|
|
@ -133,3 +133,56 @@ func TestWriteEnvValuesRoundTrip(t *testing.T) {
|
||||||
t.Error(".env lost .env.example's comment lines on write")
|
t.Error(".env lost .env.example's comment lines on write")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TestWriteEnvValuesPreservesUnrelatedCustomValues guards against a real
|
||||||
|
// regression: a save that only touches one section's keys used to reset
|
||||||
|
// every OTHER already-customized key (e.g. TELESRV_ADMIN_UI_PASSWORD) back
|
||||||
|
// to .env.example's bare template default, because the old implementation
|
||||||
|
// fell back to the template line instead of the current .env value for any
|
||||||
|
// key missing from that save's payload.
|
||||||
|
func TestWriteEnvValuesPreservesUnrelatedCustomValues(t *testing.T) {
|
||||||
|
root := findRepoRoot(t)
|
||||||
|
tmpl, err := os.ReadFile(filepath.Join(root, ".env.example"))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
dir := t.TempDir()
|
||||||
|
if err := os.WriteFile(filepath.Join(dir, ".env.example"), tmpl, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
m := NewManager(dir)
|
||||||
|
|
||||||
|
// First save sets the admin password, as a one-time setup step would.
|
||||||
|
if err := m.WriteEnvValues(map[string]string{
|
||||||
|
"TELESRV_ADMIN_UI_PASSWORD": "s3cret",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A later, unrelated save (e.g. the Storage page) that never mentions
|
||||||
|
// the password must not disturb it.
|
||||||
|
if err := m.WriteEnvValues(map[string]string{
|
||||||
|
"TELESRV_STORAGE_MAX_TOTAL_BYTES": "209715200",
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
groups, err := m.ReadEnvGroups()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
values := map[string]string{}
|
||||||
|
for _, g := range groups {
|
||||||
|
for _, f := range g.Fields {
|
||||||
|
values[f.Key] = f.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if values["TELESRV_ADMIN_UI_PASSWORD"] != "s3cret" {
|
||||||
|
t.Errorf("TELESRV_ADMIN_UI_PASSWORD = %q, want it preserved as \"s3cret\" after an unrelated save", values["TELESRV_ADMIN_UI_PASSWORD"])
|
||||||
|
}
|
||||||
|
if values["TELESRV_STORAGE_MAX_TOTAL_BYTES"] != "209715200" {
|
||||||
|
t.Errorf("TELESRV_STORAGE_MAX_TOTAL_BYTES = %q, want 209715200", values["TELESRV_STORAGE_MAX_TOTAL_BYTES"])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -637,15 +637,25 @@ func (m *Manager) readEnvFile() (map[string]string, error) {
|
||||||
// WriteEnvValues rewrites .env from .env.example's exact text, substituting
|
// WriteEnvValues rewrites .env from .env.example's exact text, substituting
|
||||||
// each known key's value in place -- see save_env()'s docstring in
|
// each known key's value in place -- see save_env()'s docstring in
|
||||||
// server-panel.py for why this (not a fresh key=value dump) is what
|
// server-panel.py for why this (not a fresh key=value dump) is what
|
||||||
// preserves comments/layout. Only keys present in values are touched; a
|
// preserves comments/layout. Only keys present in values are set to a new
|
||||||
// template-commented optional field is uncommented when given a non-empty
|
// value; every other key keeps whatever is already in the current .env
|
||||||
// value and left as-is when given an empty one.
|
// (falling back to the template's own default only for a key .env never
|
||||||
|
// set) -- previously this fell straight back to the template default for
|
||||||
|
// any key not in this particular save's payload, silently wiping out every
|
||||||
|
// other customized setting (e.g. TELESRV_ADMIN_UI_PASSWORD) on every save
|
||||||
|
// that only touches one section's keys. A template-commented optional field
|
||||||
|
// is uncommented when given a non-empty value and left as-is when given an
|
||||||
|
// empty one.
|
||||||
func (m *Manager) WriteEnvValues(values map[string]string) error {
|
func (m *Manager) WriteEnvValues(values map[string]string) error {
|
||||||
tmplPath := filepath.Join(m.Root, ".env.example")
|
tmplPath := filepath.Join(m.Root, ".env.example")
|
||||||
tmplData, err := os.ReadFile(tmplPath)
|
tmplData, err := os.ReadFile(tmplPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("read .env.example: %w", err)
|
return fmt.Errorf("read .env.example: %w", err)
|
||||||
}
|
}
|
||||||
|
existing, err := m.readEnvFile()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
lines := strings.Split(string(tmplData), "\n")
|
lines := strings.Split(string(tmplData), "\n")
|
||||||
// Split() on a trailing "\n" leaves one empty trailing element; drop it
|
// Split() on a trailing "\n" leaves one empty trailing element; drop it
|
||||||
// so the join below doesn't add a spurious blank line before the final
|
// so the join below doesn't add a spurious blank line before the final
|
||||||
|
|
@ -657,15 +667,20 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
for _, raw := range lines {
|
for _, raw := range lines {
|
||||||
line := strings.TrimSpace(raw)
|
line := strings.TrimSpace(raw)
|
||||||
if a := activeFieldRe.FindStringSubmatch(line); a != nil {
|
if a := activeFieldRe.FindStringSubmatch(line); a != nil && !seen[a[1]] {
|
||||||
if v, ok := values[a[1]]; ok && !seen[a[1]] {
|
if v, ok := values[a[1]]; ok {
|
||||||
|
seen[a[1]] = true
|
||||||
|
out = append(out, a[1]+"="+v)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if v, ok := existing[a[1]]; ok {
|
||||||
seen[a[1]] = true
|
seen[a[1]] = true
|
||||||
out = append(out, a[1]+"="+v)
|
out = append(out, a[1]+"="+v)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if c := commentedFieldRe.FindStringSubmatch(line); c != nil {
|
if c := commentedFieldRe.FindStringSubmatch(line); c != nil && !seen[c[1]] {
|
||||||
if v, ok := values[c[1]]; ok && !seen[c[1]] {
|
if v, ok := values[c[1]]; ok {
|
||||||
seen[c[1]] = true
|
seen[c[1]] = true
|
||||||
if v != "" {
|
if v != "" {
|
||||||
out = append(out, c[1]+"="+v)
|
out = append(out, c[1]+"="+v)
|
||||||
|
|
@ -674,6 +689,14 @@ func (m *Manager) WriteEnvValues(values map[string]string) error {
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
// A previously-enabled optional field shows up in the current
|
||||||
|
// .env as an active line even though the template still has it
|
||||||
|
// commented out -- keep it enabled with its existing value.
|
||||||
|
if v, ok := existing[c[1]]; ok {
|
||||||
|
seen[c[1]] = true
|
||||||
|
out = append(out, c[1]+"="+v)
|
||||||
|
continue
|
||||||
|
}
|
||||||
}
|
}
|
||||||
out = append(out, raw)
|
out = append(out, raw)
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -107,6 +107,15 @@ func locationInvalidErr() error { return tgerr.New(400, "LOCATION_INVALID")
|
||||||
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
|
func fileIDInvalidErr() error { return tgerr.New(400, "FILE_ID_INVALID") }
|
||||||
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
|
func documentInvalidErr() error { return tgerr.New(400, "DOCUMENT_INVALID") }
|
||||||
|
|
||||||
|
// fileTooBigErr surfaces TELESRV_STORAGE_MAX_UPLOAD_FILE_BYTES: the upload's
|
||||||
|
// total assembled size (not any single part) exceeds the configured
|
||||||
|
// per-file ceiling. FILE_TOO_BIG is not an official Telegram desktop-RPC
|
||||||
|
// error string, but it's already this codebase's own convention for the
|
||||||
|
// same concept on the Bot API surface (see internal/botapi/server.go) --
|
||||||
|
// reused here for consistency rather than inventing a second name for the
|
||||||
|
// same condition.
|
||||||
|
func fileTooBigErr() error { return tgerr.New(400, "FILE_TOO_BIG") }
|
||||||
|
|
||||||
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
|
// storageFullErr surfaces the low-disk-space upload guard. Deliberately not
|
||||||
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
|
// a flood-wait: a full disk won't resolve itself in 60 seconds, and telling
|
||||||
// the client to retry shortly would be misleading.
|
// the client to retry shortly would be misleading.
|
||||||
|
|
|
||||||
|
|
@ -1152,6 +1152,8 @@ func mediaUploadErr(err error) error {
|
||||||
switch {
|
switch {
|
||||||
case errors.Is(err, domain.ErrFilePartsInvalid):
|
case errors.Is(err, domain.ErrFilePartsInvalid):
|
||||||
return filePartsInvalidErr()
|
return filePartsInvalidErr()
|
||||||
|
case errors.Is(err, domain.ErrFileTooLarge):
|
||||||
|
return fileTooBigErr()
|
||||||
case errors.Is(err, domain.ErrPhotoInvalid):
|
case errors.Is(err, domain.ErrPhotoInvalid):
|
||||||
return photoInvalidErr()
|
return photoInvalidErr()
|
||||||
case errors.Is(err, domain.ErrDocumentInvalid):
|
case errors.Is(err, domain.ErrDocumentInvalid):
|
||||||
|
|
|
||||||
|
|
@ -348,6 +348,8 @@ func fileSaveErr(err error) error {
|
||||||
return filePartsInvalidErr()
|
return filePartsInvalidErr()
|
||||||
case errors.Is(err, domain.ErrFilePartTooBig):
|
case errors.Is(err, domain.ErrFilePartTooBig):
|
||||||
return filePartTooBigErr()
|
return filePartTooBigErr()
|
||||||
|
case errors.Is(err, domain.ErrFileTooLarge):
|
||||||
|
return fileTooBigErr()
|
||||||
case errors.Is(err, domain.ErrUploadQuotaExceeded):
|
case errors.Is(err, domain.ErrUploadQuotaExceeded):
|
||||||
return floodWaitErr(60)
|
return floodWaitErr(60)
|
||||||
case errors.Is(err, domain.ErrStorageFull):
|
case errors.Is(err, domain.ErrStorageFull):
|
||||||
|
|
|
||||||
105
internal/store/postgres/media_hard_retention_integration_test.go
Normal file
105
internal/store/postgres/media_hard_retention_integration_test.go
Normal file
|
|
@ -0,0 +1,105 @@
|
||||||
|
package postgres
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"strconv"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"telesrv/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestHardRetentionPurgesBlobBytesButKeepsMetadataRow exercises the "hard"
|
||||||
|
// storage retention mode's store methods end-to-end against a real
|
||||||
|
// Postgres: a document old enough (by created_at, not orphaned_at) is a
|
||||||
|
// candidate for ListDocumentIDsForHardRetentionOlderThan REGARDLESS of
|
||||||
|
// still having a live media_references row, DeleteFileBlobsForDocument
|
||||||
|
// removes only its file_blobs row (never the documents row itself), and a
|
||||||
|
// second sweep pass no longer finds it a candidate since it no longer owns
|
||||||
|
// any file_blobs row. This is the correctness property the whole "hard"
|
||||||
|
// mode design hinges on: a message must still be able to render its media
|
||||||
|
// placeholder after the bytes are gone.
|
||||||
|
func TestHardRetentionPurgesBlobBytesButKeepsMetadataRow(t *testing.T) {
|
||||||
|
pool := testPool(t)
|
||||||
|
ctx := context.Background()
|
||||||
|
media := NewMediaStore(pool)
|
||||||
|
|
||||||
|
docID := time.Now().UnixNano()
|
||||||
|
locationKey := "doc:" + strconv.FormatInt(docID, 10)
|
||||||
|
|
||||||
|
if err := media.PutDocument(ctx, domain.Document{
|
||||||
|
ID: docID,
|
||||||
|
MimeType: "application/octet-stream",
|
||||||
|
Size: 1024,
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("PutDocument: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM documents WHERE id = $1", docID)
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE media_kind = 'document' AND media_id = $1", docID)
|
||||||
|
})
|
||||||
|
|
||||||
|
// Backdate created_at well past any plausible test cutoff -- PutDocument
|
||||||
|
// always stamps now() and has no created_at parameter.
|
||||||
|
if _, err := pool.Exec(ctx, "UPDATE documents SET created_at = now() - interval '100 days' WHERE id = $1", docID); err != nil {
|
||||||
|
t.Fatalf("backdate document: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
blob := postgresTestBlob(locationKey, "hard-retention-doc", 1024, "application/octet-stream")
|
||||||
|
if err := media.PutFileBlob(ctx, blob); err != nil {
|
||||||
|
t.Fatalf("PutFileBlob: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// A LIVE reference (as if a message still embeds this document) must not
|
||||||
|
// exempt it from "hard" mode -- that's the entire point of the mode,
|
||||||
|
// unlike the orphan-only sweep.
|
||||||
|
if err := addMediaReferencesTx(ctx, pool, &domain.MessageMedia{
|
||||||
|
Kind: domain.MessageMediaKindDocument,
|
||||||
|
Document: &domain.Document{ID: docID},
|
||||||
|
}, domain.MediaRefKindMessageBox, "hard-retention-test:1"); err != nil {
|
||||||
|
t.Fatalf("add media reference: %v", err)
|
||||||
|
}
|
||||||
|
t.Cleanup(func() {
|
||||||
|
_, _ = pool.Exec(context.Background(), "DELETE FROM media_references WHERE ref_key = 'hard-retention-test:1'")
|
||||||
|
})
|
||||||
|
|
||||||
|
cutoff := time.Now().Add(-24 * time.Hour)
|
||||||
|
ids, err := media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan: %v", err)
|
||||||
|
}
|
||||||
|
if !containsInt64(ids, docID) {
|
||||||
|
t.Fatalf("hard retention candidates = %v, want to include still-referenced but old document %d", ids, docID)
|
||||||
|
}
|
||||||
|
|
||||||
|
blobs, err := media.DeleteFileBlobsForDocument(ctx, docID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DeleteFileBlobsForDocument: %v", err)
|
||||||
|
}
|
||||||
|
if len(blobs) != 1 || blobs[0].LocationKey != locationKey {
|
||||||
|
t.Fatalf("deleted blobs = %+v, want exactly one for %q", blobs, locationKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The documents row itself must survive -- a message referencing it
|
||||||
|
// still needs to render its placeholder (mime type, size, filename).
|
||||||
|
if _, found, err := media.GetDocument(ctx, docID); err != nil || !found {
|
||||||
|
t.Fatalf("document metadata row missing after hard blob purge: found=%v err=%v", found, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// The file_blobs row is gone: a subsequent download attempt resolves via
|
||||||
|
// GetFileBlob (files.Service.GetFile's path) to not-found, which the rpc
|
||||||
|
// layer already maps to LOCATION_INVALID.
|
||||||
|
if _, found, err := media.GetFileBlob(ctx, locationKey); err != nil || found {
|
||||||
|
t.Fatalf("file_blobs row still present after hard purge: found=%v err=%v", found, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Idempotent / self-terminating: a second sweep pass no longer selects
|
||||||
|
// this document, since it no longer owns any file_blobs row.
|
||||||
|
ids, err = media.ListDocumentIDsForHardRetentionOlderThan(ctx, cutoff, 1000)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ListDocumentIDsForHardRetentionOlderThan (2nd pass): %v", err)
|
||||||
|
}
|
||||||
|
if containsInt64(ids, docID) {
|
||||||
|
t.Fatalf("hard retention re-selected already-purged document %d", docID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -197,6 +197,101 @@ func (s *MediaStore) DeleteDocumentAndBlobs(ctx context.Context, id int64) ([]do
|
||||||
return blobs, nil
|
return blobs, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ListDocumentIDsForHardRetentionOlderThan returns document ids older than
|
||||||
|
// cutoff (by upload/created_at) that still own at least one file_blobs row,
|
||||||
|
// oldest first, up to limit -- the "hard" retention sweep's candidate list.
|
||||||
|
// Unlike ListOrphanedDocumentIDsOlderThan, this ignores media_references
|
||||||
|
// entirely: a document still referenced by a live message is exactly as
|
||||||
|
// eligible as an orphaned one.
|
||||||
|
func (s *MediaStore) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return s.q.ListDocumentIDsForHardRetentionOlderThan(ctx, sqlcgen.ListDocumentIDsForHardRetentionOlderThanParams{
|
||||||
|
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||||
|
BatchLimit: int32(limit),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// ListPhotoIDsForHardRetentionOlderThan is the photo counterpart of
|
||||||
|
// ListDocumentIDsForHardRetentionOlderThan.
|
||||||
|
func (s *MediaStore) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, cutoff time.Time, limit int) ([]int64, error) {
|
||||||
|
if limit <= 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return s.q.ListPhotoIDsForHardRetentionOlderThan(ctx, sqlcgen.ListPhotoIDsForHardRetentionOlderThanParams{
|
||||||
|
Cutoff: pgtype.Timestamptz{Time: cutoff, Valid: true},
|
||||||
|
BatchLimit: int32(limit),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteFileBlobsForDocument deletes every file_blobs row a document owns
|
||||||
|
// (main body + thumbnail variants), returning what was deleted so the caller
|
||||||
|
// can physically remove each object from its backend once confirming (via
|
||||||
|
// CountFileBlobRefs, after this call) no other row still needs it. Unlike
|
||||||
|
// DeleteDocumentAndBlobs, this deliberately does NOT delete the documents
|
||||||
|
// row itself -- "hard" retention mode keeps the metadata (dimensions, mime
|
||||||
|
// type, filename) so a message can still render "this document is no longer
|
||||||
|
// available" instead of disappearing outright. A subsequent
|
||||||
|
// upload.getFile/GetFileBlob lookup for a location key this call removed
|
||||||
|
// correctly finds nothing and reports not-found.
|
||||||
|
func (s *MediaStore) DeleteFileBlobsForDocument(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||||
|
var blobs []domain.FileBlob
|
||||||
|
err := withTx(ctx, s.db, "delete document blob bytes (hard retention)", func(tx pgx.Tx) error {
|
||||||
|
qtx := s.q.WithTx(tx)
|
||||||
|
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||||
|
ExactKey: fmt.Sprintf("doc:%d", id),
|
||||||
|
PrefixPattern: fmt.Sprintf("doc:%d:%%", id),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list document blobs: %w", err)
|
||||||
|
}
|
||||||
|
for _, r := range rows {
|
||||||
|
blobs = append(blobs, domain.FileBlob{
|
||||||
|
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||||
|
})
|
||||||
|
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||||
|
return fmt.Errorf("delete file blob row: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return blobs, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteFileBlobsForPhoto is the photo counterpart of
|
||||||
|
// DeleteFileBlobsForDocument -- see its doc comment. Deliberately does not
|
||||||
|
// delete the photos row.
|
||||||
|
func (s *MediaStore) DeleteFileBlobsForPhoto(ctx context.Context, id int64) ([]domain.FileBlob, error) {
|
||||||
|
var blobs []domain.FileBlob
|
||||||
|
err := withTx(ctx, s.db, "delete photo blob bytes (hard retention)", func(tx pgx.Tx) error {
|
||||||
|
qtx := s.q.WithTx(tx)
|
||||||
|
rows, err := qtx.ListFileBlobsByLocationPrefix(ctx, sqlcgen.ListFileBlobsByLocationPrefixParams{
|
||||||
|
ExactKey: fmt.Sprintf("photo:%d", id),
|
||||||
|
PrefixPattern: fmt.Sprintf("photo:%d:%%", id),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("list photo blobs: %w", err)
|
||||||
|
}
|
||||||
|
for _, r := range rows {
|
||||||
|
blobs = append(blobs, domain.FileBlob{
|
||||||
|
LocationKey: r.LocationKey, Backend: domain.MediaBackend(r.Backend), ObjectKey: r.ObjectKey, Size: r.Size,
|
||||||
|
})
|
||||||
|
if err := qtx.DeleteFileBlobRow(ctx, r.LocationKey); err != nil {
|
||||||
|
return fmt.Errorf("delete file blob row: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return blobs, nil
|
||||||
|
}
|
||||||
|
|
||||||
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
// DeletePhotoAndBlobs deletes a photo row and every file_blobs row it owns
|
||||||
// (one per rendition size), returning what was deleted so the caller can
|
// (one per rendition size), returning what was deleted so the caller can
|
||||||
// physically remove each object from its backend once confirming (via
|
// physically remove each object from its backend once confirming (via
|
||||||
|
|
|
||||||
|
|
@ -237,6 +237,37 @@ WHERE orphaned_at IS NOT NULL AND orphaned_at < sqlc.arg(cutoff)::timestamptz
|
||||||
ORDER BY orphaned_at ASC
|
ORDER BY orphaned_at ASC
|
||||||
LIMIT sqlc.arg(batch_limit)::int;
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
|
-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||||
|
-- "Hard" retention mode: candidates are documents older than cutoff (by
|
||||||
|
-- upload/created_at, NOT orphaned_at -- a live reference does not exempt
|
||||||
|
-- them) that still own at least one file_blobs row. The EXISTS check is what
|
||||||
|
-- keeps this sweep from re-selecting the same document forever: once its
|
||||||
|
-- blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||||
|
-- rows but deliberately leaves this documents row in place), it naturally
|
||||||
|
-- drops out of this query on the next pass.
|
||||||
|
SELECT d.id FROM documents d
|
||||||
|
WHERE d.created_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'doc:' || d.id::text
|
||||||
|
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY d.created_at ASC
|
||||||
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
|
-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||||
|
-- See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||||
|
-- candidate selection, for photos.
|
||||||
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.created_at < sqlc.arg(cutoff)::timestamptz
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY p.created_at ASC
|
||||||
|
LIMIT sqlc.arg(batch_limit)::int;
|
||||||
|
|
||||||
-- name: CountFileBlobRefs :one
|
-- name: CountFileBlobRefs :one
|
||||||
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
SELECT COUNT(*)::int FROM file_blobs WHERE backend = sqlc.arg(backend)::text AND object_key = sqlc.arg(object_key)::text;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -825,6 +825,50 @@ func (q *Queries) ListAvailableReactions(ctx context.Context) ([]AvailableReacti
|
||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const listDocumentIDsForHardRetentionOlderThan = `-- name: ListDocumentIDsForHardRetentionOlderThan :many
|
||||||
|
SELECT d.id FROM documents d
|
||||||
|
WHERE d.created_at < $1::timestamptz
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'doc:' || d.id::text
|
||||||
|
OR fb.location_key LIKE 'doc:' || d.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY d.created_at ASC
|
||||||
|
LIMIT $2::int
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListDocumentIDsForHardRetentionOlderThanParams struct {
|
||||||
|
Cutoff pgtype.Timestamptz
|
||||||
|
BatchLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// "Hard" retention mode: candidates are documents older than cutoff (by
|
||||||
|
// upload/created_at, NOT orphaned_at -- a live reference does not exempt
|
||||||
|
// them) that still own at least one file_blobs row. The EXISTS check is what
|
||||||
|
// keeps this sweep from re-selecting the same document forever: once its
|
||||||
|
// blob bytes are purged (DeleteFileBlobsForDocument removes the file_blobs
|
||||||
|
// rows but deliberately leaves this documents row in place), it naturally
|
||||||
|
// drops out of this query on the next pass.
|
||||||
|
func (q *Queries) ListDocumentIDsForHardRetentionOlderThan(ctx context.Context, arg ListDocumentIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listDocumentIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
|
const listFileBlobsByLocationPrefix = `-- name: ListFileBlobsByLocationPrefix :many
|
||||||
SELECT location_key, backend, object_key, size
|
SELECT location_key, backend, object_key, size
|
||||||
FROM file_blobs
|
FROM file_blobs
|
||||||
|
|
@ -937,6 +981,45 @@ func (q *Queries) ListOrphanedPhotoIDsOlderThan(ctx context.Context, arg ListOrp
|
||||||
return items, nil
|
return items, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const listPhotoIDsForHardRetentionOlderThan = `-- name: ListPhotoIDsForHardRetentionOlderThan :many
|
||||||
|
SELECT p.id FROM photos p
|
||||||
|
WHERE p.created_at < $1::timestamptz
|
||||||
|
AND EXISTS (
|
||||||
|
SELECT 1 FROM file_blobs fb
|
||||||
|
WHERE fb.location_key = 'photo:' || p.id::text
|
||||||
|
OR fb.location_key LIKE 'photo:' || p.id::text || ':%'
|
||||||
|
)
|
||||||
|
ORDER BY p.created_at ASC
|
||||||
|
LIMIT $2::int
|
||||||
|
`
|
||||||
|
|
||||||
|
type ListPhotoIDsForHardRetentionOlderThanParams struct {
|
||||||
|
Cutoff pgtype.Timestamptz
|
||||||
|
BatchLimit int32
|
||||||
|
}
|
||||||
|
|
||||||
|
// See ListDocumentIDsForHardRetentionOlderThan -- same "hard" retention
|
||||||
|
// candidate selection, for photos.
|
||||||
|
func (q *Queries) ListPhotoIDsForHardRetentionOlderThan(ctx context.Context, arg ListPhotoIDsForHardRetentionOlderThanParams) ([]int64, error) {
|
||||||
|
rows, err := q.db.Query(ctx, listPhotoIDsForHardRetentionOlderThan, arg.Cutoff, arg.BatchLimit)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer rows.Close()
|
||||||
|
var items []int64
|
||||||
|
for rows.Next() {
|
||||||
|
var id int64
|
||||||
|
if err := rows.Scan(&id); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
items = append(items, id)
|
||||||
|
}
|
||||||
|
if err := rows.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return items, nil
|
||||||
|
}
|
||||||
|
|
||||||
const listProfilePhotos = `-- name: ListProfilePhotos :many
|
const listProfilePhotos = `-- name: ListProfilePhotos :many
|
||||||
SELECT photo_id
|
SELECT photo_id
|
||||||
FROM profile_photos
|
FROM profile_photos
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue